GKaszewski/k-os
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: GKaszewski:2e665fd66edf57da94c2da2646ebc8286f4169f6
Branches Tags
GKaszewski:master GKaszewski:kernel-hardening
...
compare: GKaszewski:4049ba500720876b3cc5748025a3cb75eda2864c
Branches Tags
GKaszewski:kernel-hardening GKaszewski:master

14 Commits
2e665fd66e ... 4049ba5007

Author SHA1 Message Date
Gabriel Kaszewski
4049ba5007 feat(kernel): per-process budgets (pages, endpoints, notifs, regions) carved from creator, returned on death; init owns pool 2026-10-27 15:08:58 +01:00
Gabriel Kaszewski
7415bab7f5 feat(kernel): console cap gates DebugPutchar, no uart fast path; spawn start arg in a0; delegate returns slot 2026-10-27 14:54:36 +01:00
Gabriel Kaszewski
33e1c849d1 feat(kernel): ipc cap transfer drops Delegate; one owner per irq line, freed when owner dies 2026-10-27 14:42:28 +01:00
Gabriel Kaszewski
37c482c36c chore: fix build warnings (unused vars/imports, dead ext4 param/field/fn) 2026-10-27 14:12:50 +01:00
Gabriel Kaszewski
b4390b7b6d refactor(userspace): split syscall/fat16/ext4 by concern; trim comments 2026-10-27 13:21:53 +01:00
Gabriel Kaszewski
99273db6d3 refactor(adapter): split trap/page_table/smp/entry by concern; trim comments 2026-10-27 13:21:53 +01:00
Gabriel Kaszewski
95dbe7b9d5 refactor(kernel): split ipc/memory/lib/thread by concern; trim comments 2026-10-27 13:21:53 +01:00
Gabriel Kaszewski
b814bd9570 test: on-hardware SMP self-test (revoke/kill must stop every hart) + make test-qemu 2026-10-27 12:54:42 +01:00
Gabriel Kaszewski
909b3f0503 feat(storage): full file protocol over per-client shared region (badged clients, owned handles, clamped lengths, ReadOnly on unwritable disk); init loads via libkos::fs; drop boot shared buffer 2026-10-27 12:42:37 +01:00
Gabriel Kaszewski
694c675a40 feat: devices via caps (BootInfo, init assigns devices, drivers map MMIO/DMA themselves, IRQ-driven input); drop boot device wiring and info pages; headless display fallback 2026-10-27 12:28:39 +01:00
Gabriel Kaszewski
8bc5402a29 fix(userspace): clip display rects, bounds-check session app ids and fat16 handles, reject bad fat16 geometry, session kills via process cap 2026-10-27 12:11:13 +01:00
Gabriel Kaszewski
e8a2963b24 feat(userspace): Call/ReplyRecv IPC, malformed-request replies, servers exit on dead endpoint; drop send/receive compat, select!, async_syscall 2026-10-27 12:06:10 +01:00
Gabriel Kaszewski
2a8a5b0c27 perf(kernel): linked cap tree, translate-based page lookup, next-fit frames/heap, targeted dequeue; host benches 2026-10-27 01:33:26 +01:00
Gabriel Kaszewski
19aeabd86d feat: SMP stage 2, lock-free fast paths (GetTime, GetProcessId, DebugPutchar, no-op Yield), storm checks yield equivalence 2026-10-27 01:07:57 +01:00
184 changed files with 7348 additions and 4280 deletions
Expand all files Collapse all files

16
Cargo.lock generated
View File

@@ -55,7 +55,6 @@ name = "display-qemu"
version = "0.1.0"
dependencies = [
"display-server",
"driver-framebuffer-mock",
"driver-framebuffer-virtio",
"libkos",
"port-framebuffer",
@@ -85,6 +84,7 @@ name = "driver-block-virtio"
version = "0.1.0"
dependencies = [
"hw-virtio",
"libkos",
"port-block-device",
"types",
]
@@ -102,6 +102,7 @@ name = "driver-framebuffer-virtio"
version = "0.1.0"
dependencies = [
"hw-virtio",
"libkos",
"port-framebuffer",
"types",
]
@@ -110,7 +111,9 @@ dependencies = [
name = "driver-input-uart"
version = "0.1.0"
dependencies = [
"libkos",
"port-input-device",
"types",
]
[[package]]
@@ -118,6 +121,7 @@ name = "driver-input-virtio"
version = "0.1.0"
dependencies = [
"hw-virtio",
"libkos",
"port-input-device",
"types",
]
@@ -161,6 +165,7 @@ name = "init"
version = "0.1.0"
dependencies = [
"elf",
"hw-virtio",
"libkos",
"types",
]
@@ -173,6 +178,7 @@ dependencies = [
"driver-input-virtio",
"input-server",
"libkos",
"port-input-device",
"server-framework",
"types",
]
@@ -288,6 +294,14 @@ dependencies = [
"types",
]
[[package]]
name = "smptest"
version = "0.1.0"
dependencies = [
"libkos",
"types",
]
[[package]]
name = "snake"
version = "0.1.0"

1
Cargo.toml
View File

@@ -3,6 +3,7 @@ members = [
"crates/adapter-riscv",
"crates/apps/hello",
"crates/apps/echo",
"crates/apps/smptest",
"crates/apps/snake",
"crates/domain",
"crates/elf",

10
Makefile
View File

@@ -1,4 +1,4 @@
.PHONY: check fmt clippy test kani build all image run ci kernel-fmt kernel-clippy kernel-test
.PHONY: check fmt clippy test kani build all image run ci kernel-fmt kernel-clippy kernel-test test-qemu
RISCV_TARGET = riscv64gc-unknown-none-elf
KERNEL_BIN = target/$(RISCV_TARGET)/release/adapter-riscv
@@ -52,11 +52,17 @@ build:
cargo build --target $(RISCV_TARGET) --release \
-p adapter-riscv -p init -p echo -p snake \
-p display-qemu -p input-qemu -p storage-qemu \
-p log-server -p permission-manager -p session-server
-p log-server -p permission-manager -p session-server -p smptest
image: build
scripts/make-disk-image.sh
# Boots a test image (normal image + smptest.elf) on $(SMP) harts; init runs
# the SMP self-test and halts. Passes only on "SMPTEST PASS".
test-qemu: build
IMG=test.img SMPTEST=1 scripts/make-disk-image.sh > /dev/null
SMP=$(SMP) scripts/test-qemu.sh test.img
run: build
qemu-system-riscv64 -machine virt -smp $(SMP) -nographic -bios default \
-kernel $(KERNEL_BIN)

13
crates/adapter-riscv/src/board.rs
View File

@@ -15,9 +15,18 @@ const VIRTIO_IRQ_BASE: u16 = 1;
const VIRTIO_SLOT_SIZE: usize = 0x1000;
const VIRTIO_SLOTS: usize = VIRTIO_SIZE / VIRTIO_SLOT_SIZE;
pub const BOARD_DEVICES: usize = 1 + VIRTIO_SLOTS;
/// What each entry of `devices()` is, for init's BootInfo.
pub fn kinds() -> [u64; BOARD_DEVICES] {
let mut kinds = [types::bootinfo::KIND_VIRTIO; BOARD_DEVICES];
kinds[0] = types::bootinfo::KIND_UART_16550;
kinds
}
/// Every device userspace may drive: the UART and each VirtIO MMIO slot.
/// The PLIC, CLINT and RAM stay the kernel's.
pub fn devices() -> Option<[kernel::BoardDevice; 1 + VIRTIO_SLOTS]> {
pub fn devices() -> Option<[kernel::BoardDevice; BOARD_DEVICES]> {
use types::{address::PhysicalAddress, bounded::PageCount, id::IrqId};
let one = PageCount::new(1)?;
let device = |base: usize, irq: u16| kernel::BoardDevice {
@@ -25,7 +34,7 @@ pub fn devices() -> Option<[kernel::BoardDevice; 1 + VIRTIO_SLOTS]> {
pages: one,
irq: Some(IrqId::from_raw(irq)),
};
let mut all = [device(UART_BASE, UART_IRQ); 1 + VIRTIO_SLOTS];
let mut all = [device(UART_BASE, UART_IRQ); BOARD_DEVICES];
for (i, d) in all.iter_mut().skip(1).enumerate() {
*d = device(
VIRTIO_BASE + i * VIRTIO_SLOT_SIZE,

15
crates/adapter-riscv/src/boot/caps.rs
View File

@@ -105,3 +105,18 @@ pub fn grant_all(
uart.write_str("k-os: capabilities granted\n");
}
/// Until the log server owns the console, the servers loaded here print
/// through a console cap of their own, its slot passed in `a0`.
pub fn grant_console(uart: &Uart, servers: &[super::loader::LoadedProcess]) {
let write =
types::permission::PermissionSet::empty().with(types::permission::Permission::Write);
for server in servers {
let slot = kern()
.grant_capability(server.pid, domain::resource::Resource::Console, write)
.unwrap_or_else(|_| crate::halt(uart, "console grant failed"));
if let Some(ctx) = kern().thread_context_mut(server.tid) {
ctx.registers_mut()[10] = types::start_arg::encode(Some(slot), None);
}
}
}

35
crates/adapter-riscv/src/boot/loader.rs
View File

@@ -2,6 +2,7 @@ use crate::hw::uart::Uart;
use crate::hw::virtio_blk::VirtioBlk;
use crate::{boot_fs, mmu};
use types::address::VirtualAddress;
use types::budget::Limits;
use types::id::{ProcessId, ThreadId};
use types::mapping::PagePerms;
@@ -11,8 +12,10 @@ const USER_VA_BASE: usize = 0x10000;
const USER_STACK_PAGES: usize = USER_VA_BASE / PAGE_SIZE - 1;
const MAX_FILE_SIZE: usize = 64 * 1024;
#[derive(Clone, Copy)]
pub struct LoadedProcess {
pub pid: ProcessId,
pub tid: ThreadId,
}
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
@@ -35,7 +38,7 @@ fn elf_buffer() -> &'static mut [u8] {
}
}
pub fn load_init(uart: &Uart, blk: &mut VirtioBlk) -> (LoadedProcess, ThreadId) {
pub fn load_init(uart: &Uart, blk: &mut VirtioBlk) -> LoadedProcess {
let (pid, tid) = kern()
.boot(
types::bounded::Priority::DEFAULT,
@@ -47,17 +50,41 @@ pub fn load_init(uart: &Uart, blk: &mut VirtioBlk) -> (LoadedProcess, ThreadId)
setup_thread(tid, entry);
uart.write_str("k-os: init loaded\n");
(LoadedProcess { pid }, tid)
LoadedProcess { pid, tid }
}
/// The servers boot loads, their budgets carved from init's.
pub const STORAGE: Limits = Limits {
pages: 256,
endpoints: 2,
notifications: 2,
// A buffer per client (8) and the disk's DMA.
regions: 12,
};
/// The framebuffer alone is ~1000 pages at 1280x800.
pub const DISPLAY: Limits = Limits {
pages: 1280,
endpoints: 2,
notifications: 2,
regions: 4,
};
pub const INPUT: Limits = Limits {
pages: 64,
endpoints: 2,
notifications: 2,
regions: 4,
};
pub fn load_server(
uart: &Uart,
blk: &mut VirtioBlk,
init: ProcessId,
filename: &[u8],
label: &str,
limits: Limits,
) -> LoadedProcess {
let pid = kern()
.create_process_internal()
.create_process_internal(init, limits)
.unwrap_or_else(|| crate::halt(uart, "process create failed"));
let entry = load_elf(uart, blk, pid, filename);
let tid = kern()
@@ -68,7 +95,7 @@ pub fn load_server(
uart.write_str(label);
uart.write_str(" loaded\n");
LoadedProcess { pid }
LoadedProcess { pid, tid }
}
fn setup_thread(tid: ThreadId, entry: usize) {

3
crates/adapter-riscv/src/boot_fs.rs
View File

@@ -67,7 +67,6 @@ pub fn read_file_from_disk(
) -> Option<usize> {
let mut sector_buf = [0u8; SECTOR_SIZE];
// Read boot sector
if !blk.read_sector(0, &mut sector_buf) {
return None;
}
@@ -75,7 +74,6 @@ pub fn read_file_from_disk(
let fat_name = to_fat_name(filename);
// Search root directory
let mut first_cluster: u16 = 0;
let mut file_size: u32 = 0;
let mut found = false;
@@ -132,7 +130,6 @@ pub fn read_file_from_disk(
read += chunk;
}
// Read next cluster from FAT
let fat_offset = (cluster as u32) * 2;
let fat_sector = fs.reserved_sectors as u32 + fat_offset / SECTOR_SIZE as u32;
let fat_byte = (fat_offset % SECTOR_SIZE as u32) as usize;

81
crates/adapter-riscv/src/devices/gpu_setup.rs
View File

@@ -1,81 +0,0 @@
use super::{dma_buffer, info_page, kern, map_mmio};
use crate::hw::uart::Uart;
use crate::hw::virtio_gpu::VirtioGpu;
use types::display::Dimension;
use types::id::ProcessId;
const GPU_VA: usize = 0x20000;
const VQUEUE_VA: usize = 0x21000;
const CMDBUF_VA: usize = 0x23000;
const PHYSINFO_VA: usize = 0x24000;
const FB_VA: usize = 0x100000;
const DEFAULT_WIDTH: Dimension = Dimension::new(640);
const DEFAULT_HEIGHT: Dimension = Dimension::new(480);
/// The framebuffer alone is ~1000 pages at 1280x800.
const DISPLAY_PAGE_LIMIT: usize = 2048;
pub fn configure(uart: &Uart, display_pid: ProcessId) {
let gpu = match VirtioGpu::probe() {
Some(g) => g,
None => {
uart.write_str("k-os: no GPU (headless)\n");
return;
}
};
kern()
.set_page_limit(display_pid, DISPLAY_PAGE_LIMIT)
.unwrap_or_else(|_| crate::halt(uart, "display quota"));
map_mmio(uart, display_pid, GPU_VA, gpu.base(), 1);
let vq_phys = dma_buffer(uart, display_pid, VQUEUE_VA, 2);
let cmd_phys = dma_buffer(uart, display_pid, CMDBUF_VA, 1);
gpu.init_device(vq_phys);
let queue = hw_virtio::queue::VirtioQueue::new(hw_virtio::queue::QueueAddrs::from_base(
crate::mmu::phys_to_virt(vq_phys),
));
let dma = hw_virtio::gpu::GpuDmaAddrs {
cmd_pa: types::address::PhysicalAddress::from_raw(cmd_phys),
cmd_va: types::address::VirtualAddress::from_raw(crate::mmu::phys_to_virt(cmd_phys)),
resp_pa: types::address::PhysicalAddress::from_raw(cmd_phys + 512),
};
let (width, height) = match hw_virtio::gpu::get_display_info(gpu.mmio(), &queue, &dma) {
Some((w, h)) => {
uart.write_str("k-os: GPU display: ");
uart.write_dec(w as usize);
uart.put_char(b'x');
uart.write_dec(h as usize);
uart.put_char(b'\n');
(Dimension::new(w as u16), Dimension::new(h as u16))
}
None => {
uart.write_str("k-os: GPU display: using default 640x480\n");
(DEFAULT_WIDTH, DEFAULT_HEIGHT)
}
};
let fb_pages = (width.raw() as usize * height.raw() as usize * 4).div_ceil(4096);
let fb_phys = dma_buffer(uart, display_pid, FB_VA, fb_pages);
info_page(
uart,
display_pid,
PHYSINFO_VA,
&[
cmd_phys as u64,
(cmd_phys + 512) as u64,
fb_phys as u64,
width.raw() as u64,
height.raw() as u64,
],
);
if gpu.setup_framebuffer(cmd_phys, vq_phys, fb_phys, width, height) {
uart.write_str("k-os: VirtIO GPU configured\n");
} else {
uart.write_str("k-os: VirtIO GPU setup failed (display will be headless)\n");
}
}

35
crates/adapter-riscv/src/devices/input_setup.rs
View File

@@ -1,35 +0,0 @@
use super::{dma_buffer, info_page, map_mmio};
use crate::board;
use crate::hw::uart::Uart;
use types::id::ProcessId;
const UART_VA: usize = 0x20000;
const INPUT_MMIO_VA: usize = 0x25000;
const INPUT_VQUEUE_VA: usize = 0x26000;
const INPUT_EVBUF_VA: usize = 0x28000;
const INPUT_PHYSINFO_VA: usize = 0x29000;
pub fn configure(uart: &Uart, input_pid: ProcessId) {
// Always map UART for fallback
map_mmio(uart, input_pid, UART_VA, board::UART_BASE, 1);
// Try to find VirtIO input device
let first = crate::mmu::phys_to_virt(hw_virtio::mmio::MMIO_FIRST);
if let Some(mmio) = hw_virtio::mmio::scan_for_device(first, hw_virtio::mmio::DEVICE_ID_INPUT) {
let base = crate::mmu::virt_to_phys(mmio.base());
map_mmio(uart, input_pid, INPUT_MMIO_VA, base, 1);
let vq_phys = dma_buffer(uart, input_pid, INPUT_VQUEUE_VA, 2);
let evbuf_phys = dma_buffer(uart, input_pid, INPUT_EVBUF_VA, 1);
info_page(
uart,
input_pid,
INPUT_PHYSINFO_VA,
&[evbuf_phys as u64, vq_phys as u64],
);
hw_virtio::mmio::init_legacy_device(&mmio, vq_phys, 32);
uart.write_str("k-os: VirtIO keyboard configured\n");
} else {
uart.write_str("k-os: UART mapped for input\n");
}
}

129
crates/adapter-riscv/src/devices/mod.rs
View File

@@ -1,91 +1,80 @@
pub mod gpu_setup;
pub mod input_setup;
pub mod storage_setup;
// Boot-time userspace setup: init's BootInfo page.
use crate::hw::uart::Uart;
use types::address::{PhysicalAddress, VirtualAddress};
use types::bounded::PageCount;
use types::address::VirtualAddress;
use types::bootinfo::{BOOT_INFO_VA, BootDevice, BootInfo, MAGIC, MAX_BOOT_DEVICES, NO_SLOT};
use types::id::ProcessId;
use types::mapping::PagePerms;
// Boot-time device wiring. Everything here goes through the kernel's
// mapping API so the kernel knows about every page a driver can touch.
// Device regions come from the board table and belong to init, which
// also holds capabilities to them; moving drivers onto those caps is
// userspace work.
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
unsafe { &mut *crate::kernel_ptr() }
}
fn pages(uart: &Uart, n: usize) -> PageCount {
u16::try_from(n)
.ok()
.and_then(PageCount::new)
.unwrap_or_else(|| crate::halt(uart, "bad page count"))
}
/// Maps the board device region at `pa` into `pid` at `va`. The region
/// belongs to init (see `kernel::Kernel::register_board`); `n` must match
/// its size.
pub(crate) fn map_mmio(uart: &Uart, pid: ProcessId, va: usize, pa: usize, n: usize) {
let id = kern()
.device_region_at(PhysicalAddress::from_raw(pa))
.unwrap_or_else(|| crate::halt(uart, "no such board device"));
if kern().region_pages(id).map(|p| p.value() as usize) != Some(n) {
crate::halt(uart, "device size mismatch");
/// Tells init, in a read-only page at BOOT_INFO_VA, which of its slots hold
/// each board device's capabilities.
pub(crate) fn boot_info(
uart: &Uart,
init: ProcessId,
board: &[kernel::BoardDevice],
kinds: &[u64],
placed: &[Option<kernel::BoardCaps>],
boot_caps: kernel::BootCaps,
) {
let empty = BootDevice {
kind: 0,
phys_base: 0,
pages: 0,
irq: 0,
region_slot: NO_SLOT,
irq_slot: NO_SLOT,
};
let mut info = BootInfo {
magic: MAGIC,
device_count: 0,
system_control_slot: boot_caps.system_control.value() as u64,
console_slot: boot_caps.console.value() as u64,
devices: [empty; MAX_BOOT_DEVICES],
};
for (((d, kind), caps), out) in board
.iter()
.zip(kinds)
.zip(placed)
.zip(info.devices.iter_mut())
{
let Some(caps) = caps else { continue };
*out = BootDevice {
kind: *kind,
phys_base: d.base.raw() as u64,
pages: d.pages.value() as u64,
irq: d.irq.map_or(0, |i| i.raw() as u64),
region_slot: caps.region.value() as u64,
irq_slot: caps.irq.map_or(NO_SLOT, |s| s.value() as u64),
};
info.device_count += 1;
}
kern()
.map_region(id, pid, VirtualAddress::from_raw(va), PagePerms::ReadWrite)
.unwrap_or_else(|_| crate::halt(uart, "device map failed"));
}
/// Physically contiguous, zeroed DMA memory for `owner`, mapped read-write
/// at `va`. Returns its physical base for programming the device.
pub(crate) fn dma_buffer(uart: &Uart, owner: ProcessId, va: usize, n: usize) -> usize {
let id = kern()
.create_pool_region(owner, pages(uart, n))
.unwrap_or_else(|_| crate::halt(uart, "DMA region failed"));
kern()
.map_region(
id,
owner,
VirtualAddress::from_raw(va),
PagePerms::ReadWrite,
// SAFETY: BootInfo is repr(C) and made only of u64s.
let words = unsafe {
core::slice::from_raw_parts(
&info as *const BootInfo as *const u64,
core::mem::size_of::<BootInfo>() / 8,
)
.unwrap_or_else(|_| crate::halt(uart, "DMA map failed"));
kern()
.region_base(id)
.unwrap_or_else(|| crate::halt(uart, "DMA region vanished"))
.raw()
};
write_page(uart, init, BOOT_INFO_VA, words);
}
/// Shares a fresh region between `owner` and `other` at the same `va`.
pub(crate) fn shared_buffer(uart: &Uart, owner: ProcessId, other: ProcessId, va: usize, n: usize) {
let id = kern()
.create_pool_region(owner, pages(uart, n))
.unwrap_or_else(|_| crate::halt(uart, "shared region failed"));
for pid in [owner, other] {
kern()
.map_region(id, pid, VirtualAddress::from_raw(va), PagePerms::ReadWrite)
.unwrap_or_else(|_| crate::halt(uart, "shared map failed"));
}
}
/// Writes `words` into a fresh page mapped at `va` in `pid`, so a driver
/// can learn the physical addresses of its DMA buffers.
pub(crate) fn info_page(uart: &Uart, pid: ProcessId, va: usize, words: &[u64]) {
/// Writes `words` into a fresh read-only page mapped at `va` in `pid`.
fn write_page(uart: &Uart, pid: ProcessId, va: usize, words: &[u64]) {
let va = VirtualAddress::from_raw(va);
kern()
.map_fresh_range(pid, va, 1, PagePerms::ReadOnly)
.unwrap_or_else(|_| crate::halt(uart, "info page failed"));
.unwrap_or_else(|_| crate::halt(uart, "boot info page failed"));
let pa = kern()
.user_page_frame(pid, va)
.unwrap_or_else(|| crate::halt(uart, "info page vanished"));
let info = crate::mmu::phys_to_virt(pa.raw()) as *mut u64;
for (i, &w) in words.iter().enumerate() {
.unwrap_or_else(|| crate::halt(uart, "boot info page vanished"));
let page = crate::mmu::phys_to_virt(pa.raw()) as *mut u64;
for (i, &w) in words.iter().take(crate::mmu::PAGE_SIZE / 8).enumerate() {
// SAFETY: `pa` is a pool frame just mapped for `pid`, reachable at
// `phys_to_virt`, and `words` fits in one page.
unsafe { core::ptr::write_volatile(info.add(i), w) };
// `phys_to_virt`, and at most one page is written.
unsafe { core::ptr::write_volatile(page.add(i), w) };
}
}

36
crates/adapter-riscv/src/devices/storage_setup.rs
View File

@@ -1,36 +0,0 @@
use super::{dma_buffer, info_page, map_mmio, shared_buffer};
use crate::hw::uart::Uart;
use types::id::ProcessId;
const VIRTIO_VA: usize = 0x20000;
const VQUEUE_VA: usize = 0x21000;
const REQHDR_VA: usize = 0x23000;
const PHYSINFO_VA: usize = 0x24000;
const SHARED_BUF_VA: usize = 0x30000;
const SHARED_BUF_PAGES: usize = 16;
pub fn configure(uart: &Uart, init_pid: ProcessId, storage_pid: ProcessId, virtio_base: usize) {
map_mmio(uart, storage_pid, VIRTIO_VA, virtio_base, 1);
let vqueue_phys = dma_buffer(uart, storage_pid, VQUEUE_VA, 2);
let reqhdr_phys = dma_buffer(uart, storage_pid, REQHDR_VA, 1);
info_page(
uart,
storage_pid,
PHYSINFO_VA,
&[
reqhdr_phys as u64,
(reqhdr_phys + 16) as u64,
(reqhdr_phys + 32) as u64,
],
);
reinit_device(virtio_base, vqueue_phys);
uart.write_str("k-os: VirtIO configured for storage\n");
shared_buffer(uart, init_pid, storage_pid, SHARED_BUF_VA, SHARED_BUF_PAGES);
uart.write_str("k-os: shared buffer mapped\n");
}
fn reinit_device(base: usize, vqueue_phys: usize) {
let mmio = hw_virtio::mmio::DirectMmio::new(crate::mmu::phys_to_virt(base));
hw_virtio::mmio::init_legacy_device(&mmio, vqueue_phys, hw_virtio::queue::QUEUE_SIZE as u32);
}

73
crates/adapter-riscv/src/entry.rs Normal file
View File

@@ -0,0 +1,73 @@
use core::arch::global_asm;
// Entered with the MMU off at the physical address; the kernel is linked high.
// PC-relative code only: enable `boot_page_table` (RAM mapped low for the next
// instruction and high, MMIO high), then jump high and take the hart's stack.
global_asm!(
r#"
.section .text.entry
.globl _start
_start:
lla t2, start_high_address
j 1f
.globl _secondary_start
_secondary_start:
lla t2, secondary_high_address
1:
li t0, 8 # smp::MAX_HARTS
bgeu a0, t0, spin
mv s0, a0
lla t0, boot_page_table
srli t0, t0, 12
li t1, 8 << 60
or t0, t0, t1
csrw satp, t0
sfence.vma
ld t0, 0(t2)
jr t0
# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT
start_high:
call hart_stack
mv a0, s0
call kmain
j spin
secondary_high:
call hart_stack
mv a0, s0
call kmain_secondary
j spin
hart_stack:
lla sp, _stacks_start
addi t0, s0, 1
li t1, 0x81000
mul t0, t0, t1
add sp, sp, t0
ret
spin:
wfi
j spin
.pushsection .rodata
.balign 8
start_high_address:
.quad start_high
secondary_high_address:
.quad secondary_high
.popsection
# Sv39 root of 1 GiB leaves; flags V|R|W|A|D = 0xc7, plus X = 0xcf.
.pushsection .data
.balign 4096
boot_page_table:
.quad 0
.quad 0
.quad 0x200000cf # [2] 0x8000_0000 -> RAM, for the jump only
.fill 253, 8, 0
.quad 0x000000c7 # [256] PHYS_OFFSET + 0 -> MMIO
.quad 0
.quad 0x200000cf # [258] PHYS_OFFSET + 0x8000_0000 -> RAM
.fill 253, 8, 0
.popsection
"#
);

1
crates/adapter-riscv/src/hw/mod.rs
View File

@@ -2,4 +2,3 @@ pub mod plic;
pub mod timer;
pub mod uart;
pub mod virtio_blk;
pub mod virtio_gpu;

5
crates/adapter-riscv/src/hw/virtio_blk.rs
View File

@@ -57,11 +57,6 @@ impl VirtioBlk {
Some(Self { mmio, queue, dma })
}
/// Physical address of the device's MMIO window.
pub fn base(&self) -> usize {
virt_to_phys(self.mmio.base())
}
pub fn read_sector(&mut self, sector: u32, buf: &mut [u8; SECTOR_SIZE]) -> bool {
block::read_sector(&self.mmio, &self.queue, &self.dma, sector, buf)
}

57
crates/adapter-riscv/src/hw/virtio_gpu.rs
View File

@@ -1,57 +0,0 @@
use crate::mmu::{phys_to_virt, virt_to_phys};
use hw_virtio::gpu::{self, GpuDmaAddrs};
use hw_virtio::mmio::{self, DirectMmio};
use hw_virtio::queue::{QUEUE_SIZE, QueueAddrs, VirtioQueue};
use types::address::{PhysicalAddress, VirtualAddress};
use types::display::Dimension;
pub struct VirtioGpu {
mmio: DirectMmio,
}
impl VirtioGpu {
pub fn probe() -> Option<Self> {
let mmio = mmio::scan_for_device(phys_to_virt(mmio::MMIO_FIRST), mmio::DEVICE_ID_GPU)?;
Some(Self { mmio })
}
/// Physical address of the device's MMIO window.
pub fn base(&self) -> usize {
virt_to_phys(self.mmio.base())
}
pub fn mmio(&self) -> &DirectMmio {
&self.mmio
}
pub fn init_device(&self, vqueue_phys: usize) {
mmio::init_legacy_device(&self.mmio, vqueue_phys, QUEUE_SIZE as u32);
}
pub fn setup_framebuffer(
&self,
cmd_phys: usize,
vq_phys: usize,
fb_phys: usize,
width: Dimension,
height: Dimension,
) -> bool {
let queue = VirtioQueue::new(QueueAddrs::from_base(phys_to_virt(vq_phys)));
let dma = GpuDmaAddrs {
cmd_pa: PhysicalAddress::from_raw(cmd_phys),
cmd_va: VirtualAddress::from_raw(phys_to_virt(cmd_phys)),
resp_pa: PhysicalAddress::from_raw(cmd_phys + 512),
};
let ok1 = gpu::create_2d(&self.mmio, &queue, &dma, 1, width, height);
let ok2 = gpu::attach_backing(
&self.mmio,
&queue,
&dma,
1,
PhysicalAddress::from_raw(fb_phys),
width.raw() as u32 * height.raw() as u32 * 4,
);
let ok3 = gpu::set_scanout(&self.mmio, &queue, &dma, 0, 1, width, height);
ok1 && ok2 && ok3
}
}

9
crates/adapter-riscv/src/kernel_cell.rs
View File

@@ -7,7 +7,7 @@ pub struct KernelCell<T> {
inner: UnsafeCell<MaybeUninit<T>>,
}
// Single-core, interrupts disabled during kernel access.
// Access is serialized by the kernel lock.
unsafe impl<T> Sync for KernelCell<T> {}
impl<T> KernelCell<T> {
@@ -18,8 +18,7 @@ impl<T> KernelCell<T> {
}
}
/// Initializes the cell by letting `build` write the value straight
/// into its storage, so a large `T` never lives on the stack.
/// Initializes in place, so a large `T` never lives on the stack.
///
/// # Safety
/// `build` must fully initialize the `T` behind the pointer.
@@ -28,9 +27,7 @@ impl<T> KernelCell<T> {
self.initialized.store(true, Ordering::Release);
}
/// Returns a raw pointer to the kernel. Callers must ensure
/// no aliasing `&mut` references exist. Safe on single-core
/// with interrupts disabled during kernel calls.
/// Raw pointer to the kernel; callers must avoid aliasing `&mut`.
pub fn as_ptr(&self) -> *mut T {
debug_assert!(self.initialized.load(Ordering::Acquire));
unsafe { (*self.inner.get()).as_mut_ptr() }

125
crates/adapter-riscv/src/main.rs
View File

@@ -5,6 +5,7 @@ mod board;
mod boot;
mod boot_fs;
mod devices;
mod entry;
mod hw;
mod kernel_cell;
mod mmu;
@@ -12,7 +13,6 @@ mod platform;
mod smp;
mod trap;
use core::arch::global_asm;
use hw::uart::Uart;
use kernel_cell::KernelCell;
@@ -28,119 +28,50 @@ pub fn kernel_ptr() -> *mut kernel::Kernel<platform::RiscVPlatform> {
KERNEL.as_ptr()
}
// Firmware enters `_start` (and SBI starts other harts at
// `_secondary_start`) at the physical load address with the MMU off and
// the hart id in a0, but the kernel is linked in the upper half. Using
// only PC-relative code, the stub turns on `boot_page_table`, which maps
// RAM twice (at its physical address, just so the next instruction still
// exists, and in the upper half) plus the MMIO window high, then jumps to
// its own high address and takes the hart's kernel stack. The kernel then
// switches every hart to the real table, which maps nothing low.
global_asm!(
r#"
.section .text.entry
.globl _start
_start:
lla t2, start_high_address
j 1f
.globl _secondary_start
_secondary_start:
lla t2, secondary_high_address
1:
li t0, 8 # smp::MAX_HARTS
bgeu a0, t0, spin
mv s0, a0
lla t0, boot_page_table
srli t0, t0, 12
li t1, 8 << 60
or t0, t0, t1
csrw satp, t0
sfence.vma
ld t0, 0(t2)
jr t0
# sp = _stacks_start + (hart + 1) * smp::STACK_SLOT
start_high:
call hart_stack
mv a0, s0
call kmain
j spin
secondary_high:
call hart_stack
mv a0, s0
call kmain_secondary
j spin
hart_stack:
lla sp, _stacks_start
addi t0, s0, 1
li t1, 0x81000
mul t0, t0, t1
add sp, sp, t0
ret
spin:
wfi
j spin
.pushsection .rodata
.balign 8
start_high_address:
.quad start_high
secondary_high_address:
.quad secondary_high
.popsection
# Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with
# flags V|R|W|A|D = 0xc7, plus X = 0xcf.
.pushsection .data
.balign 4096
boot_page_table:
.quad 0
.quad 0
.quad 0x200000cf # [2] 0x8000_0000 -> RAM, for the jump only
.fill 253, 8, 0
.quad 0x000000c7 # [256] PHYS_OFFSET + 0 -> MMIO
.quad 0
.quad 0x200000cf # [258] PHYS_OFFSET + 0x8000_0000 -> RAM
.fill 253, 8, 0
.popsection
"#
);
const SHARED_BUF_VA: usize = 0x30000;
#[unsafe(no_mangle)]
extern "C" fn kmain(hart: usize) -> ! {
let uart = Uart::new();
uart.write_str("k-os: booted on RISC-V\n");
boot::hardware::init(&uart, hart);
// Held until the boot hart first enters userspace; harts started
// below wait for it. Not before init: zeroing .bss resets the lock.
// Not before init: zeroing .bss resets the lock.
smp::lock();
let mut blk =
boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device"));
let (init, init_thread) = boot::loader::load_init(&uart, &mut blk);
if let Some(ctx) = kern_mut().thread_context_mut(init_thread) {
ctx.registers_mut()[10] = SHARED_BUF_VA;
}
let storage = boot::loader::load_server(&uart, &mut blk, b"storage.elf", "storage");
let display = boot::loader::load_server(&uart, &mut blk, b"display.elf", "display");
let input = boot::loader::load_server(&uart, &mut blk, b"input.elf", "input");
let virtio_base = blk.base();
let init = boot::loader::load_init(&uart, &mut blk);
use boot::loader::{DISPLAY, INPUT, STORAGE, load_server};
let storage = load_server(
&uart,
&mut blk,
init.pid,
b"storage.elf",
"storage",
STORAGE,
);
let display = load_server(
&uart,
&mut blk,
init.pid,
b"display.elf",
"display",
DISPLAY,
);
let input = load_server(&uart, &mut blk, init.pid, b"input.elf", "input", INPUT);
let board = board::devices().unwrap_or_else(|| halt(&uart, "bad board table"));
kern_mut()
.register_board(init.pid, &board)
.unwrap_or_else(|_| halt(&uart, "board registration failed"));
devices::storage_setup::configure(&uart, init.pid, storage.pid, virtio_base);
devices::gpu_setup::configure(&uart, display.pid);
devices::input_setup::configure(&uart, input.pid);
boot::caps::grant_all(&uart, init.pid, storage.pid, display.pid, input.pid);
// After grant_all, so the slots init already relies on stay put.
kern_mut()
.grant_boot_authority(init.pid, &board)
// Last, so the servers' fixed slots stay where they expect them.
boot::caps::grant_console(&uart, &[storage, display, input]);
// After grant_all: init already relies on those slots.
let mut placed = [None; board::BOARD_DEVICES];
let boot_caps = kern_mut()
.grant_boot_authority(init.pid, &board, &mut placed)
.unwrap_or_else(|_| halt(&uart, "boot authority failed"));
devices::boot_info(&uart, init.pid, &board, &board::kinds(), &placed, boot_caps);
start_other_harts(&uart, hart);
uart.write_str("k-os: entering userspace\n");

4
crates/adapter-riscv/src/mmu/allocator.rs
View File

@@ -1,9 +1,7 @@
use super::page_table::{PAGE_SIZE, PageTable};
use crate::board;
// Boot-time bump allocator for kernel-only memory: the kernel page table
// and boot scratch buffers. User memory and user page tables come from the
// kernel's frame table instead. It must never reach the user pool.
// Boot bump allocator for kernel-only memory; must never reach the user pool.
static mut NEXT_FREE: usize = 0;
pub fn init_allocator(start: usize) {

11
crates/adapter-riscv/src/mmu/mod.rs
View File

@@ -4,13 +4,11 @@ pub mod page_table;
pub use allocator::init_allocator;
pub use page_table::{
PAGE_SIZE, PageTable, Sv39Flags, USER_VA_END, USER_VA_START, activate, active_root, check_wx,
create_user_root, init_kernel_page_table, map_user_page, unmap_user_page,
create_user_root, init_kernel_page_table, map_user_page, translate_user, unmap_user_page,
};
/// The kernel lives in the upper half of the address space: all of RAM and
/// the MMIO windows appear at their physical address plus this offset (the
/// start of Sv39's upper half), and the kernel image is linked there too,
/// so one offset converts any kernel address. Must match linker.ld.
/// Direct-map offset: RAM, MMIO and the kernel image sit at physical + this
/// (start of Sv39's upper half). Must match linker.ld.
pub const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000;
/// Where the kernel sees physical address `pa`.
@@ -18,8 +16,7 @@ pub const fn phys_to_virt(pa: usize) -> usize {
pa + PHYS_OFFSET
}
/// The physical address behind kernel address `va` (a static, the boot
/// heap, a page table). Only valid for addresses in the direct map.
/// The physical address behind a direct-map kernel address `va`.
pub const fn virt_to_phys(va: usize) -> usize {
va - PHYS_OFFSET
}

234
crates/adapter-riscv/src/mmu/page_table.rs → crates/adapter-riscv/src/mmu/page_table/kernel.rs
View File

@@ -1,139 +1,12 @@
use super::allocator::alloc_page_table;
use super::{phys_to_virt, virt_to_phys};
use super::pte::{
Mapping, PAGE_SIZE, PTE_W, PTE_X, PageTable, PageTableEntry, Sv39Flags, map, table_at,
table_ppn,
};
use crate::board;
use crate::mmu::allocator::alloc_page_table;
use crate::mmu::{phys_to_virt, virt_to_phys};
use crate::smp;
use platform::{FrameSource, MapError};
use types::address::{PhysicalAddress, VirtualAddress};
use types::mapping::PagePerms;
pub use types::mapping::Mapping;
pub const PAGE_SIZE: usize = 4096;
const PTE_V: u64 = 1 << 0;
const PTE_R: u64 = 1 << 1;
const PTE_W: u64 = 1 << 2;
const PTE_X: u64 = 1 << 3;
const PTE_U: u64 = 1 << 4;
const PTE_G: u64 = 1 << 5;
const PTE_A: u64 = 1 << 6;
const PTE_D: u64 = 1 << 7;
#[derive(Clone, Copy)]
pub struct Sv39Flags {
bits: u64,
}
impl Sv39Flags {
pub fn kernel_rx() -> Self {
Self {
bits: PTE_R | PTE_X | PTE_G,
}
}
pub fn kernel_ro() -> Self {
Self {
bits: PTE_R | PTE_G,
}
}
pub fn kernel_rw() -> Self {
Self {
bits: PTE_R | PTE_W | PTE_G,
}
}
pub fn user(perms: PagePerms) -> Self {
let bits = match perms {
PagePerms::ReadOnly => PTE_R | PTE_U,
PagePerms::ReadWrite => PTE_R | PTE_W | PTE_U,
PagePerms::ReadExecute => PTE_R | PTE_X | PTE_U,
};
Self { bits }
}
fn to_pte_flags(self) -> u64 {
PTE_V | PTE_A | PTE_D | self.bits
}
}
#[derive(Clone, Copy)]
#[repr(transparent)]
pub struct PageTableEntry(u64);
impl PageTableEntry {
pub fn empty() -> Self {
Self(0)
}
fn new_leaf(ppn: u64, flags: Sv39Flags) -> Self {
Self(flags.to_pte_flags() | (ppn << 10))
}
fn new_branch(ppn: u64) -> Self {
Self(PTE_V | (ppn << 10))
}
pub fn is_valid(&self) -> bool {
self.0 & PTE_V != 0
}
pub fn ppn(&self) -> u64 {
(self.0 >> 10) & 0xFFF_FFFF_FFFF
}
}
#[repr(C, align(4096))]
pub struct PageTable {
pub entries: [PageTableEntry; 512],
}
impl PageTable {
pub fn zero(&mut self) {
for entry in &mut self.entries {
*entry = PageTableEntry::empty();
}
}
}
fn vpn(va: usize, level: usize) -> usize {
(va >> (12 + 9 * level)) & 0x1FF
}
fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) {
let vpn2 = vpn(va, 2);
let vpn1 = vpn(va, 1);
let vpn0 = vpn(va, 0);
if !root.entries[vpn2].is_valid() {
root.entries[vpn2] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
}
let l1 = table_at((root.entries[vpn2].ppn() << 12) as usize);
if !l1.entries[vpn1].is_valid() {
l1.entries[vpn1] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
}
let l0 = table_at((l1.entries[vpn1].ppn() << 12) as usize);
let ppn = (pa as u64) >> 12;
l0.entries[vpn0] = PageTableEntry::new_leaf(ppn, flags);
}
/// The physical page number of a page table the kernel holds a reference to.
fn table_ppn(table: &PageTable) -> u64 {
(virt_to_phys(table as *const PageTable as usize) as u64) >> 12
}
pub fn map(root: &mut PageTable, mapping: Mapping<Sv39Flags>) {
for i in 0..mapping.pages {
let offset = i * PAGE_SIZE;
map_page(
root,
mapping.va.raw() + offset,
mapping.pa.raw() + offset,
mapping.flags,
);
}
}
pub fn init_kernel_page_table() -> &'static mut PageTable {
let root = alloc_page_table();
@@ -325,101 +198,6 @@ pub fn activate(root: &PageTable) {
}
}
// ---- user address spaces ----
//
// User mappings live in [USER_VA_START, USER_VA_END), which is root slot 0,
// the L1 slots below USER_VA_END. Every address space gets its own root and its
// own L1 for slot 0, and the kernel maps nothing in the lower half, so all
// page tables below the user window are private to one process. The
// kernel's upper-half entries are shared by copying them.
pub const USER_VA_START: usize = 0x1000;
pub const USER_VA_END: usize = 0x0200_0000;
/// The page table at physical address `pa`, through the direct map.
pub fn table_at(pa: usize) -> &'static mut PageTable {
// SAFETY: every page table lives in RAM, which the kernel maps at
// `phys_to_virt`.
unsafe { &mut *(phys_to_virt(pa) as *mut PageTable) }
}
fn fresh_table(frames: &mut dyn FrameSource) -> Result<usize, MapError> {
let pa = frames
.alloc_table_frame()
.ok_or(MapError::OutOfFrames)?
.raw();
table_at(pa).zero();
Ok(pa)
}
fn in_user_window(va: usize) -> bool {
va.is_multiple_of(PAGE_SIZE) && (USER_VA_START..USER_VA_END).contains(&va)
}
/// Builds a root table sharing the kernel's mappings, with a private L1 for
/// the user window. Returns the root's physical address.
pub fn create_user_root(
kernel_root: &PageTable,
frames: &mut dyn FrameSource,
) -> Result<usize, MapError> {
let root_pa = fresh_table(frames)?;
let l1_pa = fresh_table(frames)?;
let root = table_at(root_pa);
for (mine, kernels) in root.entries.iter_mut().zip(&kernel_root.entries).skip(256) {
*mine = *kernels;
}
root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12);
Ok(root_pa)
}
fn user_leaf_table(
root_pa: usize,
va: usize,
frames: Option<&mut dyn FrameSource>,
) -> Result<&'static mut PageTable, MapError> {
let root = table_at(root_pa);
let l1 = table_at((root.entries[0].ppn() << 12) as usize);
let slot = &mut l1.entries[vpn(va, 1)];
if !slot.is_valid() {
let frames = frames.ok_or(MapError::NoAddressSpace)?;
let l0_pa = fresh_table(frames)?;
*slot = PageTableEntry::new_branch((l0_pa as u64) >> 12);
}
Ok(table_at((slot.ppn() << 12) as usize))
}
pub fn map_user_page(
root_pa: usize,
va: usize,
pa: usize,
flags: Sv39Flags,
frames: &mut dyn FrameSource,
) -> Result<(), MapError> {
if !in_user_window(va) || !pa.is_multiple_of(PAGE_SIZE) {
return Err(MapError::InvalidAddress);
}
let l0 = user_leaf_table(root_pa, va, Some(frames))?;
l0.entries[vpn(va, 0)] = PageTableEntry::new_leaf((pa as u64) >> 12, flags);
flush_tlb(va);
Ok(())
}
pub fn unmap_user_page(root_pa: usize, va: usize) {
if !in_user_window(va) {
return;
}
if let Ok(l0) = user_leaf_table(root_pa, va, None) {
l0.entries[vpn(va, 0)] = PageTableEntry::empty();
flush_tlb(va);
}
}
fn flush_tlb(va: usize) {
// SAFETY: sfence.vma only invalidates translation caches.
unsafe { core::arch::asm!("sfence.vma {0}, zero", in(reg) va) };
}
/// Physical address of the root table the hart is translating with.
pub fn active_root() -> usize {
let satp: usize;

11
crates/adapter-riscv/src/mmu/page_table/mod.rs Normal file
View File

@@ -0,0 +1,11 @@
//! Sv39 page tables: entries and mapping, the kernel table, user address spaces.
mod kernel;
mod pte;
mod user;
pub use kernel::{WxViolation, activate, active_root, check_wx, init_kernel_page_table};
pub use pte::{PAGE_SIZE, PageTable, Sv39Flags, table_at};
pub use user::{
USER_VA_END, USER_VA_START, create_user_root, map_user_page, translate_user, unmap_user_page,
};

139
crates/adapter-riscv/src/mmu/page_table/pte.rs Normal file
View File

@@ -0,0 +1,139 @@
use crate::mmu::allocator::alloc_page_table;
use crate::mmu::{phys_to_virt, virt_to_phys};
use types::mapping::PagePerms;
pub use types::mapping::Mapping;
pub const PAGE_SIZE: usize = 4096;
const PTE_V: u64 = 1 << 0;
const PTE_R: u64 = 1 << 1;
pub(super) const PTE_W: u64 = 1 << 2;
pub(super) const PTE_X: u64 = 1 << 3;
const PTE_U: u64 = 1 << 4;
const PTE_G: u64 = 1 << 5;
const PTE_A: u64 = 1 << 6;
const PTE_D: u64 = 1 << 7;
#[derive(Clone, Copy)]
pub struct Sv39Flags {
bits: u64,
}
impl Sv39Flags {
pub fn kernel_rx() -> Self {
Self {
bits: PTE_R | PTE_X | PTE_G,
}
}
pub fn kernel_ro() -> Self {
Self {
bits: PTE_R | PTE_G,
}
}
pub fn kernel_rw() -> Self {
Self {
bits: PTE_R | PTE_W | PTE_G,
}
}
pub fn user(perms: PagePerms) -> Self {
let bits = match perms {
PagePerms::ReadOnly => PTE_R | PTE_U,
PagePerms::ReadWrite => PTE_R | PTE_W | PTE_U,
PagePerms::ReadExecute => PTE_R | PTE_X | PTE_U,
};
Self { bits }
}
fn to_pte_flags(self) -> u64 {
PTE_V | PTE_A | PTE_D | self.bits
}
}
#[derive(Clone, Copy)]
#[repr(transparent)]
pub struct PageTableEntry(pub(super) u64);
impl PageTableEntry {
pub fn empty() -> Self {
Self(0)
}
pub(super) fn new_leaf(ppn: u64, flags: Sv39Flags) -> Self {
Self(flags.to_pte_flags() | (ppn << 10))
}
pub(super) fn new_branch(ppn: u64) -> Self {
Self(PTE_V | (ppn << 10))
}
pub fn is_valid(&self) -> bool {
self.0 & PTE_V != 0
}
pub fn ppn(&self) -> u64 {
(self.0 >> 10) & 0xFFF_FFFF_FFFF
}
}
#[repr(C, align(4096))]
pub struct PageTable {
pub entries: [PageTableEntry; 512],
}
impl PageTable {
pub fn zero(&mut self) {
for entry in &mut self.entries {
*entry = PageTableEntry::empty();
}
}
}
pub(super) fn vpn(va: usize, level: usize) -> usize {
(va >> (12 + 9 * level)) & 0x1FF
}
fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) {
let vpn2 = vpn(va, 2);
let vpn1 = vpn(va, 1);
let vpn0 = vpn(va, 0);
if !root.entries[vpn2].is_valid() {
root.entries[vpn2] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
}
let l1 = table_at((root.entries[vpn2].ppn() << 12) as usize);
if !l1.entries[vpn1].is_valid() {
l1.entries[vpn1] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
}
let l0 = table_at((l1.entries[vpn1].ppn() << 12) as usize);
let ppn = (pa as u64) >> 12;
l0.entries[vpn0] = PageTableEntry::new_leaf(ppn, flags);
}
/// The physical page number of a page table the kernel holds a reference to.
pub(super) fn table_ppn(table: &PageTable) -> u64 {
(virt_to_phys(table as *const PageTable as usize) as u64) >> 12
}
pub fn map(root: &mut PageTable, mapping: Mapping<Sv39Flags>) {
for i in 0..mapping.pages {
let offset = i * PAGE_SIZE;
map_page(
root,
mapping.va.raw() + offset,
mapping.pa.raw() + offset,
mapping.flags,
);
}
}
/// The page table at physical address `pa`, through the direct map.
pub fn table_at(pa: usize) -> &'static mut PageTable {
// SAFETY: every page table lives in RAM, which the kernel maps at
// `phys_to_virt`.
unsafe { &mut *(phys_to_virt(pa) as *mut PageTable) }
}

93
crates/adapter-riscv/src/mmu/page_table/user.rs Normal file
View File

@@ -0,0 +1,93 @@
use super::pte::{PAGE_SIZE, PageTable, PageTableEntry, Sv39Flags, table_at, vpn};
use platform::{FrameSource, MapError};
/// User mappings live in root slot 0, in a private L1 per address space.
pub const USER_VA_START: usize = 0x1000;
pub const USER_VA_END: usize = 0x0200_0000;
fn fresh_table(frames: &mut dyn FrameSource) -> Result<usize, MapError> {
let pa = frames
.alloc_table_frame()
.ok_or(MapError::OutOfFrames)?
.raw();
table_at(pa).zero();
Ok(pa)
}
fn in_user_window(va: usize) -> bool {
va.is_multiple_of(PAGE_SIZE) && (USER_VA_START..USER_VA_END).contains(&va)
}
/// Builds a root table sharing the kernel's mappings, with a private L1 for
/// the user window. Returns the root's physical address.
pub fn create_user_root(
kernel_root: &PageTable,
frames: &mut dyn FrameSource,
) -> Result<usize, MapError> {
let root_pa = fresh_table(frames)?;
let l1_pa = fresh_table(frames)?;
let root = table_at(root_pa);
for (mine, kernels) in root.entries.iter_mut().zip(&kernel_root.entries).skip(256) {
*mine = *kernels;
}
root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12);
Ok(root_pa)
}
fn user_leaf_table(
root_pa: usize,
va: usize,
frames: Option<&mut dyn FrameSource>,
) -> Result<&'static mut PageTable, MapError> {
let root = table_at(root_pa);
let l1 = table_at((root.entries[0].ppn() << 12) as usize);
let slot = &mut l1.entries[vpn(va, 1)];
if !slot.is_valid() {
let frames = frames.ok_or(MapError::NoAddressSpace)?;
let l0_pa = fresh_table(frames)?;
*slot = PageTableEntry::new_branch((l0_pa as u64) >> 12);
}
Ok(table_at((slot.ppn() << 12) as usize))
}
pub fn map_user_page(
root_pa: usize,
va: usize,
pa: usize,
flags: Sv39Flags,
frames: &mut dyn FrameSource,
) -> Result<(), MapError> {
if !in_user_window(va) || !pa.is_multiple_of(PAGE_SIZE) {
return Err(MapError::InvalidAddress);
}
let l0 = user_leaf_table(root_pa, va, Some(frames))?;
l0.entries[vpn(va, 0)] = PageTableEntry::new_leaf((pa as u64) >> 12, flags);
flush_tlb(va);
Ok(())
}
/// The physical page `va` maps to in the address space at `root_pa`.
pub fn translate_user(root_pa: usize, va: usize) -> Option<usize> {
if !in_user_window(va) {
return None;
}
let l0 = user_leaf_table(root_pa, va, None).ok()?;
let entry = l0.entries[vpn(va, 0)];
entry.is_valid().then(|| (entry.ppn() << 12) as usize)
}
pub fn unmap_user_page(root_pa: usize, va: usize) {
if !in_user_window(va) {
return;
}
if let Ok(l0) = user_leaf_table(root_pa, va, None) {
l0.entries[vpn(va, 0)] = PageTableEntry::empty();
flush_tlb(va);
}
}
fn flush_tlb(va: usize) {
// SAFETY: sfence.vma only invalidates translation caches.
unsafe { core::arch::asm!("sfence.vma {0}, zero", in(reg) va) };
}

10
crates/adapter-riscv/src/platform.rs
View File

@@ -64,9 +64,8 @@ impl Platform for RiscVPlatform {
let Some(root) = self.root(process) else {
return;
};
// No hart may still translate through it once the kernel frees its
// frames: other harts leave it (and wait for nothing but that), and
// the caller may be tearing down the space we are running on.
// No hart may translate through it once its frames are freed; that
// includes this one, which may be running on it.
smp::shoot_down(root);
if mmu::active_root() == root {
// SAFETY: the kernel page table lives in the boot heap forever.
@@ -103,6 +102,11 @@ impl Platform for RiscVPlatform {
}
}
fn translate(&self, process: ProcessId, va: VirtualAddress) -> Option<PhysicalAddress> {
let root = self.root(process)?;
mmu::translate_user(root, va.raw()).map(PhysicalAddress::from_raw)
}
fn zero_frame(&mut self, pa: PhysicalAddress) {
// SAFETY: pool frames are RAM, mapped at `phys_to_virt`, and the
// kernel only zeroes frames it has just allocated.

220
crates/adapter-riscv/src/smp.rs
View File

@@ -1,220 +0,0 @@
//! Harts. Each hart has its own kernel stack, and its hart id doubles as
//! its kernel `CoreId`. The kernel is entered under one lock for now (stage
//! 1 of multi-core; the kernel itself stays a sequential program), so
//! userspace runs in parallel and the kernel does not.
//!
//! Harts talk through IPIs carrying two kinds of request:
//! - reschedule: the kernel made work for this hart, or took its thread
//! away; handled under the lock on the way out of the kernel.
//! - leave root R: a page table this hart may be using is changing or about
//! to be freed; the hart switches to the kernel's root and flushes its
//! TLB. This needs no lock, so it is serviced even while a hart spins
//! waiting for the lock or idles, and the sender (which holds the lock)
//! can wait for the acknowledgement without deadlocking.
use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering};
use crate::mmu;
/// Must match linker.ld.
pub const MAX_HARTS: usize = 8;
/// A guard page and a 512 KiB stack (boot builds big tables on it). Must
/// match linker.ld.
pub const STACK_SLOT: usize = 4096 + 0x8_0000;
const SIP_SSIP: usize = 1 << 1;
/// The hart running this code, from which kernel stack it is on.
pub fn hart_id() -> usize {
let sp: usize;
// SAFETY: reading sp has no side effects.
unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) };
(sp - stacks_start()) / STACK_SLOT
}
pub fn stacks_start() -> usize {
unsafe extern "C" {
static _stacks_start: u8;
}
// SAFETY: only the symbol's address is taken.
unsafe { &_stacks_start as *const u8 as usize }
}
static BOOT_HART: AtomicUsize = AtomicUsize::new(0);
static NUM_HARTS: AtomicUsize = AtomicUsize::new(1);
/// Physical address of the kernel's own root table.
static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0);
/// The root table each hart translates with.
static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS];
static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
pub fn boot_hart() -> usize {
BOOT_HART.load(Ordering::Relaxed)
}
pub fn num_harts() -> usize {
NUM_HARTS.load(Ordering::Relaxed)
}
pub fn kernel_root() -> usize {
KERNEL_ROOT.load(Ordering::Relaxed)
}
/// Records which hart booted and how many there are, and where the
/// kernel's root table is.
pub fn init(boot: usize, harts: usize, kernel_root: usize) {
BOOT_HART.store(boot, Ordering::Relaxed);
NUM_HARTS.store(harts, Ordering::Relaxed);
KERNEL_ROOT.store(kernel_root, Ordering::Relaxed);
}
/// Called by `mmu::activate`.
pub fn note_active_root(root: usize) {
if let Some(r) = ACTIVE_ROOT.get(hart_id()) {
r.store(root, Ordering::SeqCst);
}
}
// ---- the kernel lock ----
static NEXT_TICKET: AtomicU32 = AtomicU32::new(0);
static SERVING: AtomicU32 = AtomicU32::new(0);
/// Takes the kernel lock, first come first served. Leave-root requests
/// are serviced while waiting.
pub fn lock() {
let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
while SERVING.load(Ordering::Acquire) != ticket {
service_ipi();
core::hint::spin_loop();
}
}
pub fn unlock() {
SERVING.fetch_add(1, Ordering::Release);
}
// ---- IPIs ----
/// The lock-free half of an IPI: acknowledges it and, if asked, leaves the
/// root being changed. Safe to call at any time, any number of times.
pub fn service_ipi() {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & SIP_SSIP == 0 {
return;
}
// Clear first: a request posted after this raises SSIP again.
// SAFETY: clearing our own pending software interrupt.
unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) };
let me = hart_id();
let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else {
return;
};
let root = leave.swap(0, Ordering::SeqCst);
if root != 0 {
if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) {
mmu::activate(mmu::page_table::table_at(kernel_root()));
}
// SAFETY: flushing translations has no other effect.
unsafe { core::arch::asm!("sfence.vma zero, zero") };
left.fetch_add(1, Ordering::SeqCst);
}
}
/// Asks `hart` to look at its run queue again, on its way out of the
/// kernel.
pub fn kick(hart: usize) {
if let Some(r) = RESCHEDULE.get(hart) {
r.store(true, Ordering::SeqCst);
sbi_send_ipi(hart);
}
}
/// Consumes this hart's reschedule request.
pub fn take_reschedule() -> bool {
RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.swap(false, Ordering::SeqCst))
}
/// Makes every other hart that translates through `root` leave it and
/// flush, and waits until each has. Called with the kernel lock held, so
/// no hart can start using `root` meanwhile.
pub fn shoot_down(root: usize) {
let me = hart_id();
for hart in 0..num_harts() {
if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root {
continue;
}
let seen = LEFT[hart].load(Ordering::SeqCst);
LEAVE_ROOT[hart].store(root, Ordering::SeqCst);
sbi_send_ipi(hart);
while LEFT[hart].load(Ordering::SeqCst) == seen {
core::hint::spin_loop();
}
}
}
// ---- SBI ----
const SBI_EXT_IPI: usize = 0x735049;
const SBI_EXT_HSM: usize = 0x48534D;
fn sbi_send_ipi(hart: usize) {
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") 1usize => _,
inlateout("a1") hart => _,
in("a6") 0usize,
in("a7") SBI_EXT_IPI,
);
}
}
/// Whether `hart` exists, per SBI HSM `hart_get_status`.
fn sbi_hart_exists(hart: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
lateout("a1") _,
in("a6") 2usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}
/// The number of harts, counting up from hart 0 until one is missing.
pub fn probe_harts() -> usize {
(0..MAX_HARTS)
.take_while(|h| sbi_hart_exists(*h))
.count()
.max(1)
}
/// Starts `hart` at the physical address `entry` with the MMU off.
pub fn sbi_hart_start(hart: usize, entry: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
inlateout("a1") entry => _,
in("a2") 0usize,
in("a6") 0usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}

42
crates/adapter-riscv/src/smp/hints.rs Normal file
View File

@@ -0,0 +1,42 @@
//! State published under the kernel lock for lock-free fast paths to read.
use core::sync::atomic::{AtomicUsize, Ordering};
use super::{MAX_HARTS, hart_id};
/// Per core: highest waiting priority + 1, 0 if its queue is empty.
static WAITING: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
/// Per hart: the running thread's process id + 1 and priority + 1.
static RUNNING_PID: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RUNNING_PRIORITY: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
pub fn publish_waiting(core: usize, highest: Option<u8>) {
if let Some(w) = WAITING.get(core) {
w.store(highest.map_or(0, |p| p as usize + 1), Ordering::Release);
}
}
pub fn publish_running(pid: Option<u16>, priority: Option<u8>) {
let me = hart_id();
if let (Some(p), Some(q)) = (RUNNING_PID.get(me), RUNNING_PRIORITY.get(me)) {
p.store(pid.map_or(0, |x| x as usize + 1), Ordering::Release);
q.store(priority.map_or(0, |x| x as usize + 1), Ordering::Release);
}
}
/// The process the thread running on this hart belongs to.
pub fn running_pid() -> Option<usize> {
let v = RUNNING_PID.get(hart_id())?.load(Ordering::Acquire);
v.checked_sub(1)
}
/// Whether a Yield by the running thread would change nothing: no thread
/// waits on this core at its priority or above.
pub fn yield_is_noop() -> bool {
let me = hart_id();
let (Some(w), Some(p)) = (WAITING.get(me), RUNNING_PRIORITY.get(me)) else {
return false;
};
let mine = p.load(Ordering::Acquire);
mine != 0 && w.load(Ordering::Acquire) < mine
}

96
crates/adapter-riscv/src/smp/ipi.rs Normal file
View File

@@ -0,0 +1,96 @@
//! Reschedule and leave-root IPIs, serviced without the kernel lock.
use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use super::sbi::sbi_send_ipi;
use super::{MAX_HARTS, hart_id, kernel_root, num_harts};
use crate::mmu;
const SIP_SSIP: usize = 1 << 1;
/// The root table each hart translates with.
static ACTIVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static RESCHEDULE: [AtomicBool; MAX_HARTS] = [const { AtomicBool::new(false) }; MAX_HARTS];
static LEAVE_ROOT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
static LEFT: [AtomicUsize; MAX_HARTS] = [const { AtomicUsize::new(0) }; MAX_HARTS];
/// Called by `mmu::activate`.
pub fn note_active_root(root: usize) {
if let Some(r) = ACTIVE_ROOT.get(hart_id()) {
r.store(root, Ordering::SeqCst);
}
}
/// Whether this hart has an IPI or a reschedule request it has not acted
/// on; fast paths must then take the slow path.
pub fn has_pending_request() -> bool {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
sip & SIP_SSIP != 0
|| RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.load(Ordering::SeqCst))
}
// ---- IPIs ----
/// The lock-free half of an IPI: acknowledges it and leaves the root being
/// changed, if asked. Safe to call at any time.
pub fn service_ipi() {
let sip: usize;
// SAFETY: reading sip has no side effects.
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & SIP_SSIP == 0 {
return;
}
// Clear first: a request posted after this raises SSIP again.
// SAFETY: clearing our own pending software interrupt.
unsafe { core::arch::asm!("csrc sip, {0}", in(reg) SIP_SSIP) };
let me = hart_id();
let (Some(leave), Some(left)) = (LEAVE_ROOT.get(me), LEFT.get(me)) else {
return;
};
let root = leave.swap(0, Ordering::SeqCst);
if root != 0 {
if ACTIVE_ROOT.get(me).map(|r| r.load(Ordering::SeqCst)) == Some(root) {
mmu::activate(mmu::page_table::table_at(kernel_root()));
}
// SAFETY: flushing translations has no other effect.
unsafe { core::arch::asm!("sfence.vma zero, zero") };
left.fetch_add(1, Ordering::SeqCst);
}
}
/// Asks `hart` to look at its run queue again, on its way out of the
/// kernel.
pub fn kick(hart: usize) {
if let Some(r) = RESCHEDULE.get(hart) {
r.store(true, Ordering::SeqCst);
sbi_send_ipi(hart);
}
}
/// Consumes this hart's reschedule request.
pub fn take_reschedule() -> bool {
RESCHEDULE
.get(hart_id())
.is_some_and(|r| r.swap(false, Ordering::SeqCst))
}
/// Makes every other hart leave `root` and flush, waiting for each.
/// Kernel lock held, so no hart can start using `root` meanwhile.
pub fn shoot_down(root: usize) {
let me = hart_id();
for hart in 0..num_harts() {
if hart == me || ACTIVE_ROOT[hart].load(Ordering::SeqCst) != root {
continue;
}
let seen = LEFT[hart].load(Ordering::SeqCst);
LEAVE_ROOT[hart].store(root, Ordering::SeqCst);
sbi_send_ipi(hart);
while LEFT[hart].load(Ordering::SeqCst) == seen {
core::hint::spin_loop();
}
}
}

22
crates/adapter-riscv/src/smp/lock.rs Normal file
View File

@@ -0,0 +1,22 @@
//! The kernel lock: a ticket lock that services IPIs while waiting.
use core::sync::atomic::{AtomicU32, Ordering};
use super::service_ipi;
static NEXT_TICKET: AtomicU32 = AtomicU32::new(0);
static SERVING: AtomicU32 = AtomicU32::new(0);
/// Takes the kernel lock, first come first served. Leave-root requests
/// are serviced while waiting.
pub fn lock() {
let ticket = NEXT_TICKET.fetch_add(1, Ordering::Relaxed);
while SERVING.load(Ordering::Acquire) != ticket {
service_ipi();
core::hint::spin_loop();
}
}
pub fn unlock() {
SERVING.fetch_add(1, Ordering::Release);
}

61
crates/adapter-riscv/src/smp/mod.rs Normal file
View File

@@ -0,0 +1,61 @@
//! Harts: per-hart kernel stacks (the hart id doubles as the kernel `CoreId`),
//! the kernel lock, and IPIs. The kernel runs under one lock; userspace runs in parallel.
mod hints;
mod ipi;
mod lock;
mod sbi;
pub use hints::{publish_running, publish_waiting, running_pid, yield_is_noop};
pub use ipi::{
has_pending_request, kick, note_active_root, service_ipi, shoot_down, take_reschedule,
};
pub use lock::{lock, unlock};
pub use sbi::{probe_harts, sbi_hart_start};
use core::sync::atomic::{AtomicUsize, Ordering};
/// Must match linker.ld.
pub const MAX_HARTS: usize = 8;
/// A guard page and a 512 KiB stack. Must match linker.ld.
pub const STACK_SLOT: usize = 4096 + 0x8_0000;
/// The hart running this code, from which kernel stack it is on.
pub fn hart_id() -> usize {
let sp: usize;
// SAFETY: reading sp has no side effects.
unsafe { core::arch::asm!("mv {0}, sp", out(reg) sp) };
(sp - stacks_start()) / STACK_SLOT
}
pub fn stacks_start() -> usize {
unsafe extern "C" {
static _stacks_start: u8;
}
// SAFETY: only the symbol's address is taken.
unsafe { &_stacks_start as *const u8 as usize }
}
static BOOT_HART: AtomicUsize = AtomicUsize::new(0);
static NUM_HARTS: AtomicUsize = AtomicUsize::new(1);
/// Physical address of the kernel's own root table.
static KERNEL_ROOT: AtomicUsize = AtomicUsize::new(0);
pub fn boot_hart() -> usize {
BOOT_HART.load(Ordering::Relaxed)
}
pub fn num_harts() -> usize {
NUM_HARTS.load(Ordering::Relaxed)
}
pub fn kernel_root() -> usize {
KERNEL_ROOT.load(Ordering::Relaxed)
}
/// Records the boot hart, the hart count and the kernel root table.
pub fn init(boot: usize, harts: usize, kernel_root: usize) {
BOOT_HART.store(boot, Ordering::Relaxed);
NUM_HARTS.store(harts, Ordering::Relaxed);
KERNEL_ROOT.store(kernel_root, Ordering::Relaxed);
}

60
crates/adapter-riscv/src/smp/sbi.rs Normal file
View File

@@ -0,0 +1,60 @@
//! SBI calls: IPIs and hart state management.
use super::MAX_HARTS;
const SBI_EXT_IPI: usize = 0x735049;
const SBI_EXT_HSM: usize = 0x48534D;
pub(super) fn sbi_send_ipi(hart: usize) {
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") 1usize => _,
inlateout("a1") hart => _,
in("a6") 0usize,
in("a7") SBI_EXT_IPI,
);
}
}
/// Whether `hart` exists, per SBI HSM `hart_get_status`.
fn sbi_hart_exists(hart: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
lateout("a1") _,
in("a6") 2usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}
/// The number of harts, counting up from hart 0 until one is missing.
pub fn probe_harts() -> usize {
(0..MAX_HARTS)
.take_while(|h| sbi_hart_exists(*h))
.count()
.max(1)
}
/// Starts `hart` at the physical address `entry` with the MMU off.
pub fn sbi_hart_start(hart: usize, entry: usize) -> bool {
let error: isize;
// SAFETY: an SBI call; a0 and a1 come back clobbered.
unsafe {
core::arch::asm!(
"ecall",
inlateout("a0") hart => error,
inlateout("a1") entry => _,
in("a2") 0usize,
in("a6") 0usize,
in("a7") SBI_EXT_HSM,
);
}
error == 0
}

22
crates/adapter-riscv/src/trap/fast.rs Normal file
View File

@@ -0,0 +1,22 @@
use super::{TrapFrame, write_result};
/// Syscalls answered without the kernel lock because they touch nothing it
/// guards. A pending IPI or reschedule request forces the slow path.
pub(super) fn fast_syscall(frame: &mut TrapFrame) -> bool {
use types::syscall_abi::{SYS_GET_PROCESS_ID, SYS_GET_TIME, SYS_YIELD};
if crate::smp::has_pending_request() {
return false;
}
let result = match frame.regs[17] {
SYS_GET_TIME => kernel::SyscallResult::Value(crate::hw::timer::read_time() as usize),
SYS_GET_PROCESS_ID => match crate::smp::running_pid() {
Some(pid) => kernel::SyscallResult::Value(pid),
None => return false,
},
SYS_YIELD if crate::smp::yield_is_noop() => kernel::SyscallResult::Ok,
_ => return false,
};
frame.sepc += 4;
write_result(frame, &result);
true
}

272
crates/adapter-riscv/src/trap/mod.rs
View File

@@ -1,4 +1,10 @@
use core::arch::global_asm;
mod fast;
mod sched;
mod vector;
use fast::fast_syscall;
pub use sched::enter_first;
use sched::{finish, release, switch_to_next, switch_to_thread};
#[repr(C)]
pub struct TrapFrame {
@@ -7,126 +13,6 @@ pub struct TrapFrame {
pub sstatus: usize,
}
global_asm!(
r#"
.section .text
.align 4
.globl trap_vector
trap_vector:
csrrw sp, sscratch, sp
bnez sp, _from_user
csrrw sp, sscratch, sp
_from_user:
addi sp, sp, -272
sd x1, 8(sp)
sd x3, 24(sp)
sd x4, 32(sp)
sd x5, 40(sp)
sd x6, 48(sp)
sd x7, 56(sp)
sd x8, 64(sp)
sd x9, 72(sp)
sd x10, 80(sp)
sd x11, 88(sp)
sd x12, 96(sp)
sd x13, 104(sp)
sd x14, 112(sp)
sd x15, 120(sp)
sd x16, 128(sp)
sd x17, 136(sp)
sd x18, 144(sp)
sd x19, 152(sp)
sd x20, 160(sp)
sd x21, 168(sp)
sd x22, 176(sp)
sd x23, 184(sp)
sd x24, 192(sp)
sd x25, 200(sp)
sd x26, 208(sp)
sd x27, 216(sp)
sd x28, 224(sp)
sd x29, 232(sp)
sd x30, 240(sp)
sd x31, 248(sp)
csrr t0, sscratch
sd t0, 16(sp)
csrr t0, sepc
sd t0, 256(sp)
csrr t0, sstatus
sd t0, 264(sp)
mv a0, sp
call trap_handler_rust
j _restore_frame
# trap_return(frame): resume whatever `frame` describes, as if returning
# from a trap that saved it. Used to enter userspace for the first time.
.globl trap_return
trap_return:
mv sp, a0
_restore_frame:
ld t0, 256(sp)
csrw sepc, t0
ld t0, 264(sp)
csrw sstatus, t0
andi t1, t0, 0x100
bnez t1, _restore_smode
ld t0, 16(sp)
csrw sscratch, t0
_restore_smode:
ld x1, 8(sp)
ld x3, 24(sp)
ld x4, 32(sp)
ld x6, 48(sp)
ld x7, 56(sp)
ld x8, 64(sp)
ld x9, 72(sp)
ld x10, 80(sp)
ld x11, 88(sp)
ld x12, 96(sp)
ld x13, 104(sp)
ld x14, 112(sp)
ld x15, 120(sp)
ld x16, 128(sp)
ld x17, 136(sp)
ld x18, 144(sp)
ld x19, 152(sp)
ld x20, 160(sp)
ld x21, 168(sp)
ld x22, 176(sp)
ld x23, 184(sp)
ld x24, 192(sp)
ld x25, 200(sp)
ld x26, 208(sp)
ld x27, 216(sp)
ld x28, 224(sp)
ld x29, 232(sp)
ld x30, 240(sp)
ld x31, 248(sp)
# Every GPR except t0 (x5) is restored. Decide where we return to while
# t0 is still free, and restore it last on each path; using t0 after
# its restore would corrupt the interrupted program's t0.
csrr t0, sstatus
andi t0, t0, 0x100
bnez t0, _sret_smode
ld x5, 40(sp)
addi sp, sp, 272
csrrw sp, sscratch, sp
sret
_sret_smode:
ld x5, 40(sp)
addi sp, sp, 272
sret
"#
);
const CAUSE_INTERRUPT_BIT: usize = 1 << 63;
const CAUSE_S_SOFTWARE: usize = 1;
const CAUSE_S_TIMER: usize = 5;
@@ -143,7 +29,7 @@ pub fn init() {
let addr = trap_vector as *const () as usize;
core::arch::asm!("csrw stvec, {0}", in(reg) addr);
core::arch::asm!("csrw sscratch, zero");
// SIE stays clear — interrupts are enabled only when entering userspace via sret
// SIE stays clear; interrupts are enabled only by sret into userspace.
let sie_bits = (1usize << 5) | (1usize << 9) | (1usize << 1);
core::arch::asm!("csrs sie, {0}", in(reg) sie_bits);
}
@@ -169,31 +55,6 @@ fn write_result(frame: &mut TrapFrame, result: &kernel::SyscallResult) {
set_results(frame, kernel::abi::encode(result));
}
fn save_frame_to_context(frame: &TrapFrame, ctx: &mut domain::thread::SavedContext) {
ctx.registers_mut().copy_from_slice(&frame.regs);
ctx.set_pc(frame.sepc);
ctx.set_status(frame.sstatus);
}
fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFrame) {
frame.regs.copy_from_slice(ctx.registers());
frame.sepc = ctx.program_counter();
frame.sstatus = ctx.status();
// ensure returning to U-mode
frame.sstatus &= !(1 << 8); // clear SPP
frame.sstatus |= 1 << 5; // set SPIE
}
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
if let Some(result) = kern().take_pending_result(tid) {
write_result(frame, &result);
}
}
unsafe extern "C" {
fn trap_return(frame: *mut TrapFrame) -> !;
}
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
// SAFETY: only called with the kernel lock held.
unsafe { &mut *crate::kernel_ptr() }
@@ -203,63 +64,7 @@ fn this_core() -> types::id::CoreId {
types::id::CoreId::from_raw(crate::smp::hart_id() as u8)
}
/// Saves `frame` into `from` (if it still exists) and loads `to`.
fn switch_to_thread(
frame: &mut TrapFrame,
from: Option<types::id::ThreadId>,
to: types::id::ThreadId,
) {
let kern = kern();
if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
if let Some(ctx) = kern.thread_context(to) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, to);
}
/// Saves `from` and runs the next ready thread, idling until there is one.
fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next = next_runnable();
let _ = kern().set_thread_running(next, this_core());
switch_to_thread(frame, None, next);
}
/// The next thread to run on this hart, idling until one is ready.
///
/// Idling gives up the kernel lock: an idle hart holds nothing, so other
/// harts can make work for it (and kick it) meanwhile. It also leaves any
/// user address space first, so nobody needs to shoot it down. SIE stays
/// clear; `wfi` still wakes on a pending interrupt, serviced by polling.
fn next_runnable() -> types::id::ThreadId {
loop {
if let Some(tid) = kern().pick_next_thread() {
return tid;
}
crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
crate::smp::unlock();
unsafe { core::arch::asm!("wfi") };
crate::smp::service_ipi();
crate::smp::lock();
let _ = crate::smp::take_reschedule();
let sip: usize;
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & (1 << CAUSE_S_TIMER) != 0 {
// No thread is running, so there is no time slice to charge.
crate::hw::timer::handle_interrupt();
}
if sip & (1 << CAUSE_S_EXTERNAL) != 0 {
dispatch_external();
}
}
}
/// Claims every pending PLIC interrupt and hands each to the kernel. Only
/// the boot hart receives external interrupts.
/// Claims every pending PLIC interrupt (boot hart only) and hands it to the kernel.
fn dispatch_external() {
loop {
let irq_num = crate::hw::plic::Plic::claim();
@@ -271,52 +76,6 @@ fn dispatch_external() {
}
}
/// Last step before leaving the kernel to userspace: act on a reschedule
/// request, make sure this hart has a thread (its own may have been taken
/// away by another hart), and translate through that thread's address
/// space.
fn finish(frame: &mut TrapFrame) {
let kern = kern();
let current = kern.current_thread();
if crate::smp::take_reschedule()
&& let Some(cur) = current
&& let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
{
switch_to_thread(frame, Some(cur), next);
}
if kern.current_thread().is_none() {
switch_to_next(frame, None);
}
let root = kern
.current_thread()
.and_then(|t| kern.thread_process(t))
.map(|pid| kern.platform_mut().page_table_for(pid))
.unwrap_or(0);
if root != 0 && crate::mmu::active_root() != root {
crate::mmu::activate(crate::mmu::page_table::table_at(root));
}
}
/// Leaves the kernel for userspace for the first time on this hart, with
/// the kernel lock held: runs the thread the kernel already made current
/// here (init, on the boot hart) or the next one homed here.
pub fn enter_first() -> ! {
let mut frame = TrapFrame {
regs: [0; 32],
sepc: 0,
sstatus: 0,
};
match kern().current_thread() {
Some(tid) => switch_to_thread(&mut frame, None, tid),
None => switch_to_next(&mut frame, None),
}
finish(&mut frame);
crate::smp::unlock();
// SAFETY: `frame` holds a user context; sscratch is set from it on
// the way out, so later traps land at the top of this frame.
unsafe { trap_return(&mut frame) }
}
#[unsafe(no_mangle)]
extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
let scause: usize;
@@ -326,6 +85,9 @@ extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
// Checked before taking the lock: this hart may already hold it.
kernel_fault(frame, scause);
}
if scause == CAUSE_ECALL_U && fast_syscall(frame) {
return;
}
crate::smp::service_ipi();
crate::smp::lock();
if scause & CAUSE_INTERRUPT_BIT != 0 {
@@ -334,7 +96,7 @@ extern "C" fn trap_handler_rust(frame: &mut TrapFrame) {
handle_exception(frame, scause);
}
finish(frame);
crate::smp::unlock();
release();
}
fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
@@ -350,15 +112,12 @@ fn handle_interrupt(frame: &mut TrapFrame, cause: usize) {
}
}
CAUSE_S_EXTERNAL => dispatch_external(),
// Serviced on entry; any reschedule happens in `finish`.
CAUSE_S_SOFTWARE => {}
_ => {}
}
}
/// The kernel itself faulted (a W^X violation, a stack guard, a bad
/// pointer). Its state can no longer be trusted, and no user thread is to
/// blame.
/// The kernel itself faulted; its state is untrusted, so halt.
fn kernel_fault(frame: &TrapFrame, scause: usize) -> ! {
let uart = crate::hw::uart::Uart::new();
uart.write_str("KERNEL FAULT: scause=");
@@ -427,8 +186,7 @@ fn handle_exception(frame: &mut TrapFrame, scause: usize) {
kern.handle_fault(current_tid, fault, frame.sepc);
switch_to_next(frame, Some(current_tid));
}
// With no current thread (another hart just destroyed it),
// `finish` moves on to the next one.
// No current thread (destroyed by another hart): `finish` picks one.
}
}
}

139
crates/adapter-riscv/src/trap/sched.rs Normal file
View File

@@ -0,0 +1,139 @@
use super::{
CAUSE_S_EXTERNAL, CAUSE_S_TIMER, TrapFrame, dispatch_external, kern, this_core, write_result,
};
fn save_frame_to_context(frame: &TrapFrame, ctx: &mut domain::thread::SavedContext) {
ctx.registers_mut().copy_from_slice(&frame.regs);
ctx.set_pc(frame.sepc);
ctx.set_status(frame.sstatus);
}
fn load_context_to_frame(ctx: &domain::thread::SavedContext, frame: &mut TrapFrame) {
frame.regs.copy_from_slice(ctx.registers());
frame.sepc = ctx.program_counter();
frame.sstatus = ctx.status();
frame.sstatus &= !(1 << 8); // SPP: return to U-mode
frame.sstatus |= 1 << 5; // SPIE
}
fn apply_pending_result(frame: &mut TrapFrame, tid: types::id::ThreadId) {
if let Some(result) = kern().take_pending_result(tid) {
write_result(frame, &result);
}
}
unsafe extern "C" {
fn trap_return(frame: *mut TrapFrame) -> !;
}
/// Saves `frame` into `from` (if it still exists) and loads `to`.
pub(super) fn switch_to_thread(
frame: &mut TrapFrame,
from: Option<types::id::ThreadId>,
to: types::id::ThreadId,
) {
let kern = kern();
if let Some(ctx) = from.and_then(|t| kern.thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
if let Some(ctx) = kern.thread_context(to) {
load_context_to_frame(ctx, frame);
}
apply_pending_result(frame, to);
}
/// Saves `from` and runs the next ready thread, idling until there is one.
pub(super) fn switch_to_next(frame: &mut TrapFrame, from: Option<types::id::ThreadId>) {
if let Some(ctx) = from.and_then(|t| kern().thread_context_mut(t)) {
save_frame_to_context(frame, ctx);
}
let next = next_runnable();
let _ = kern().set_thread_running(next, this_core());
switch_to_thread(frame, None, next);
}
/// The next thread to run on this hart, idling (lock released, kernel
/// address space active so no shootdown is needed) until one is ready.
fn next_runnable() -> types::id::ThreadId {
loop {
if let Some(tid) = kern().pick_next_thread() {
return tid;
}
crate::mmu::activate(crate::mmu::page_table::table_at(crate::smp::kernel_root()));
release();
unsafe { core::arch::asm!("wfi") };
crate::smp::service_ipi();
crate::smp::lock();
let _ = crate::smp::take_reschedule();
let sip: usize;
unsafe { core::arch::asm!("csrr {0}, sip", out(reg) sip) };
if sip & (1 << CAUSE_S_TIMER) != 0 {
// No thread is running, so there is no time slice to charge.
crate::hw::timer::handle_interrupt();
}
if sip & (1 << CAUSE_S_EXTERNAL) != 0 {
dispatch_external();
}
}
}
/// Releases the kernel lock after publishing state the lock-free fast paths read.
pub(super) fn release() {
let kern = kern();
for core in 0..crate::smp::num_harts() {
let id = types::id::CoreId::from_raw(core as u8);
crate::smp::publish_waiting(core, kern.highest_waiting(id).map(|p| p.value()));
}
let running = kern.current_thread();
crate::smp::publish_running(
running
.and_then(|t| kern.thread_process(t))
.map(|p| p.raw()),
running
.and_then(|t| kern.thread_priority(t))
.map(|p| p.value()),
);
crate::smp::unlock();
}
/// Last step before returning to userspace: apply a reschedule request,
/// ensure this hart has a thread, and activate its address space.
pub(super) fn finish(frame: &mut TrapFrame) {
let kern = kern();
let current = kern.current_thread();
if crate::smp::take_reschedule()
&& let Some(cur) = current
&& let domain::scheduler::SchedulerDecision::Preempt(next) = kern.reschedule(this_core())
{
switch_to_thread(frame, Some(cur), next);
}
if kern.current_thread().is_none() {
switch_to_next(frame, None);
}
let root = kern
.current_thread()
.and_then(|t| kern.thread_process(t))
.map(|pid| kern.platform_mut().page_table_for(pid))
.unwrap_or(0);
if root != 0 && crate::mmu::active_root() != root {
crate::mmu::activate(crate::mmu::page_table::table_at(root));
}
}
/// Enters userspace for the first time on this hart (kernel lock held).
pub fn enter_first() -> ! {
let mut frame = TrapFrame {
regs: [0; 32],
sepc: 0,
sstatus: 0,
};
match kern().current_thread() {
Some(tid) => switch_to_thread(&mut frame, None, tid),
None => switch_to_next(&mut frame, None),
}
finish(&mut frame);
release();
// SAFETY: `frame` holds a user context; sscratch is set from it on
// the way out, so later traps land at the top of this frame.
unsafe { trap_return(&mut frame) }
}

118
crates/adapter-riscv/src/trap/vector.rs Normal file
View File

@@ -0,0 +1,118 @@
use core::arch::global_asm;
global_asm!(
r#"
.section .text
.align 4
.globl trap_vector
trap_vector:
csrrw sp, sscratch, sp
bnez sp, _from_user
csrrw sp, sscratch, sp
_from_user:
addi sp, sp, -272
sd x1, 8(sp)
sd x3, 24(sp)
sd x4, 32(sp)
sd x5, 40(sp)
sd x6, 48(sp)
sd x7, 56(sp)
sd x8, 64(sp)
sd x9, 72(sp)
sd x10, 80(sp)
sd x11, 88(sp)
sd x12, 96(sp)
sd x13, 104(sp)
sd x14, 112(sp)
sd x15, 120(sp)
sd x16, 128(sp)
sd x17, 136(sp)
sd x18, 144(sp)
sd x19, 152(sp)
sd x20, 160(sp)
sd x21, 168(sp)
sd x22, 176(sp)
sd x23, 184(sp)
sd x24, 192(sp)
sd x25, 200(sp)
sd x26, 208(sp)
sd x27, 216(sp)
sd x28, 224(sp)
sd x29, 232(sp)
sd x30, 240(sp)
sd x31, 248(sp)
csrr t0, sscratch
sd t0, 16(sp)
csrr t0, sepc
sd t0, 256(sp)
csrr t0, sstatus
sd t0, 264(sp)
mv a0, sp
call trap_handler_rust
j _restore_frame
# trap_return(frame): resume `frame` as if returning from the trap that saved it.
.globl trap_return
trap_return:
mv sp, a0
_restore_frame:
ld t0, 256(sp)
csrw sepc, t0
ld t0, 264(sp)
csrw sstatus, t0
andi t1, t0, 0x100
bnez t1, _restore_smode
ld t0, 16(sp)
csrw sscratch, t0
_restore_smode:
ld x1, 8(sp)
ld x3, 24(sp)
ld x4, 32(sp)
ld x6, 48(sp)
ld x7, 56(sp)
ld x8, 64(sp)
ld x9, 72(sp)
ld x10, 80(sp)
ld x11, 88(sp)
ld x12, 96(sp)
ld x13, 104(sp)
ld x14, 112(sp)
ld x15, 120(sp)
ld x16, 128(sp)
ld x17, 136(sp)
ld x18, 144(sp)
ld x19, 152(sp)
ld x20, 160(sp)
ld x21, 168(sp)
ld x22, 176(sp)
ld x23, 184(sp)
ld x24, 192(sp)
ld x25, 200(sp)
ld x26, 208(sp)
ld x27, 216(sp)
ld x28, 224(sp)
ld x29, 232(sp)
ld x30, 240(sp)
ld x31, 248(sp)
# t0 (x5) is restored last on each path; it is the only scratch register.
csrr t0, sstatus
andi t0, t0, 0x100
bnez t0, _sret_smode
ld x5, 40(sp)
addi sp, sp, 272
csrrw sp, sscratch, sp
sret
_sret_smode:
ld x5, 40(sp)
addi sp, sp, 272
sret
"#
);

5
crates/apps/echo/src/main.rs
View File

@@ -10,8 +10,9 @@ use libkos::syscall;
extern "C" fn main() {
let ep = CapSlot::const_new(0);
let (w0, w1, w2, w3) = syscall::receive(ep);
syscall::send(ep, w0.wrapping_add(1), w1, w2, w3);
if let Ok(syscall::Received::Call { words: [w0, w1, w2, w3], .. }) = syscall::receive_msg(ep) {
let _ = syscall::reply([w0.wrapping_add(1), w1, w2, w3], None);
}
syscall::exit();
}

13
crates/apps/smptest/Cargo.toml Normal file
View File

@@ -0,0 +1,13 @@
[package]
name = "smptest"
version = "0.1.0"
edition = "2024"
[[bin]]
name = "smptest"
test = false
bench = false
[dependencies]
types = { workspace = true }
libkos = { workspace = true }

47
crates/apps/smptest/src/main.rs Normal file
View File

@@ -0,0 +1,47 @@
//! Spinner half of init's SMP self-test: one thread per hart, each
//! incrementing its own counter in the page given in slot 0.
#![no_std]
#![no_main]
extern crate libkos;
use core::sync::atomic::{AtomicUsize, Ordering};
use libkos::syscall;
use types::address::{EntryPoint, StackPointer, VirtualAddress};
use types::bounded::{CapSlot, PageCount};
use types::mapping::PagePerms;
/// Counter page layout (words): 0 next free counter index, 1..=THREADS counters.
const COUNTERS_VA: usize = 0x0100_0000;
const THREADS: usize = 4;
#[unsafe(no_mangle)]
extern "C" fn main() {
let page = CapSlot::const_new(0);
if syscall::map_region(page, VirtualAddress::from_raw(COUNTERS_VA), PagePerms::ReadWrite).is_err() {
libkos::println!("smptest: cannot map the counter page");
syscall::exit();
}
for _ in 1..THREADS {
let Some(stack) = PageCount::new(1).and_then(syscall::request_pages) else {
break;
};
let top = StackPointer::from_raw(stack.raw() + 4096);
syscall::spawn_thread(EntryPoint::from_raw(spin as *const () as usize), top, 0);
}
spin();
}
extern "C" fn spin() -> ! {
// SAFETY: COUNTERS_VA maps the counter page; every access is atomic.
let word = |i: usize| unsafe { &*((COUNTERS_VA + i * 8) as *const AtomicUsize) };
let mine = 1 + word(0).fetch_add(1, Ordering::Relaxed) % THREADS;
loop {
// Yield so a spinner sharing init's hart cannot starve it.
for _ in 0..1024 {
word(mine).fetch_add(1, Ordering::Relaxed);
}
syscall::yield_now();
}
}

9
crates/apps/snake/src/main.rs
View File

@@ -18,9 +18,7 @@ const TICK_DELAY: u64 = 5_000_000;
const MAX_RENDER_CMDS: usize = 140;
fn display_send(req: &DisplayRequest) {
let words = req.encode();
syscall::send(DISPLAY_EP, words[0], words[1], words[2], words[3]);
syscall::receive(DISPLAY_EP);
let _ = syscall::request(DISPLAY_EP, req.encode());
}
fn draw_rect(x: u16, y: u16, w: u16, h: u16, color: u32) {
@@ -67,8 +65,9 @@ extern "C" fn main() {
fn handle_input(snake: &mut Snake) {
// Drain all pending events, apply the latest direction
loop {
let _ = syscall::send(INPUT_EP, 0, 0, 0, 0);
let (tag, kind, _, _) = syscall::receive(INPUT_EP);
let Ok([tag, kind, _, _]) = syscall::request(INPUT_EP, [0; 4]) else {
break;
};
if tag == 0 { break; } // NoEvent — queue drained
if tag == 1 {
match kind {

1
crates/bins/display-qemu/Cargo.toml
View File

@@ -15,5 +15,4 @@ libkos = { workspace = true }
server-framework = { workspace = true }
display-server = { workspace = true }
driver-framebuffer-virtio = { workspace = true }
driver-framebuffer-mock = { workspace = true }
port-framebuffer = { workspace = true }

50
crates/bins/display-qemu/src/main.rs
View File

@@ -8,31 +8,51 @@ use types::bounded::CapSlot;
#[unsafe(no_mangle)]
extern "C" fn main() {
let ep = CapSlot::const_new(0);
if let Some(fb) = driver_framebuffer_virtio::VirtioFramebuffer::try_new() {
// Init hands over the GPU, or nothing if the machine has none.
let dev = libkos::device::receive_device(ep);
if let Some(fb) = dev.mmio.and_then(driver_framebuffer_virtio::VirtioFramebuffer::try_new) {
run_virtio(fb, ep);
} else {
run_mock(ep);
}
loop { libkos::syscall::yield_now(); }
libkos::println!("display: no GPU, running headless");
run_headless(ep);
}
#[inline(never)]
fn run_virtio(mut fb: driver_framebuffer_virtio::VirtioFramebuffer, ep: CapSlot) {
fn run_virtio(mut fb: driver_framebuffer_virtio::VirtioFramebuffer, ep: CapSlot) -> ! {
use port_framebuffer::Framebuffer;
fb.clear(types::display::Color::rgb(30, 30, 60));
fb.flush();
let mut srv = display_server::server::DisplayServer::new(fb);
server_framework::serve(&mut srv, ep);
server_framework::serve(&mut srv, ep)
}
/// No GPU: clients still get answers (and the dimensions they draw for);
/// pixels go nowhere.
struct Headless;
impl port_framebuffer::Framebuffer for Headless {
fn width(&self) -> types::display::Dimension {
types::display::Dimension::new(640)
}
fn height(&self) -> types::display::Dimension {
types::display::Dimension::new(480)
}
fn write_pixel(&mut self, _: types::display::ScreenCoord, _: types::display::ScreenCoord, _: types::display::Color) {}
fn fill_rect(
&mut self,
_: types::display::ScreenCoord,
_: types::display::ScreenCoord,
_: types::display::Dimension,
_: types::display::Dimension,
_: types::display::Color,
) {
}
fn clear(&mut self, _: types::display::Color) {}
fn flush(&mut self) {}
}
#[inline(never)]
fn run_mock(ep: CapSlot) {
let fb = driver_framebuffer_mock::MockFramebuffer::new(
types::display::Dimension::new(640),
types::display::Dimension::new(480),
);
let mut srv = display_server::server::DisplayServer::new(fb);
server_framework::serve(&mut srv, ep);
fn run_headless(ep: CapSlot) -> ! {
let mut srv = display_server::server::DisplayServer::new(Headless);
server_framework::serve(&mut srv, ep)
}

1
crates/bins/input-qemu/Cargo.toml
View File

@@ -9,6 +9,7 @@ test = false
bench = false
[dependencies]
port-input-device = { workspace = true }
types = { workspace = true }
libkos = { workspace = true }

42
crates/bins/input-qemu/src/main.rs
View File

@@ -3,30 +3,38 @@
extern crate libkos;
use libkos::device::{self, KIND_UART_16550, KIND_VIRTIO};
use port_input_device::InputDevice;
use types::bounded::CapSlot;
#[unsafe(no_mangle)]
extern "C" fn main() {
let ep = CapSlot::new(0).unwrap();
if let Some(dev) = driver_input_virtio::VirtioInputDevice::try_new() {
run_input(dev, ep);
} else {
run_uart(ep);
// Init hands over the keyboard (virtio) or, failing that, the UART.
let dev = device::receive_device(ep);
match (dev.kind, dev.mmio) {
(KIND_VIRTIO, Some(mmio)) => {
if let Some(d) = driver_input_virtio::VirtioInputDevice::try_new(mmio) {
run(d, dev.irq, ep);
}
}
(KIND_UART_16550, Some(mmio)) => {
if let Some(d) = driver_input_uart::UartInputDevice::new(mmio) {
run(d, dev.irq, ep);
}
}
_ => {}
}
loop { libkos::syscall::yield_now(); }
libkos::println!("input: no usable input device, exiting");
libkos::syscall::exit();
}
/// Serves `device`, woken by its interrupt when it has one.
#[inline(never)]
fn run_input(device: driver_input_virtio::VirtioInputDevice, ep: CapSlot) {
let mut srv = input_server::server::InputServer::new(device);
server_framework::serve(&mut srv, ep);
}
#[inline(never)]
fn run_uart(ep: CapSlot) {
let device = driver_input_uart::UartInputDevice::new();
let mut srv = input_server::server::InputServer::new(device);
server_framework::serve(&mut srv, ep);
fn run<D: InputDevice>(device: D, irq: Option<CapSlot>, ep: CapSlot) -> ! {
let mut srv = match irq.filter(|irq| device::bind_irq(*irq).is_ok()) {
Some(irq) => input_server::server::InputServer::with_irq(device, irq),
None => input_server::server::InputServer::new(device),
};
server_framework::serve(&mut srv, ep)
}

80
crates/domain/src/budget.rs Normal file
View File

@@ -0,0 +1,80 @@
//! What a process may hold: frames and kernel objects. A child's limits
//! are carved from its creator's unused budget and go back when it dies,
//! so the limits of all live processes never add up to more than init got.
use crate::memory::quota::Quota;
pub use types::budget::Limits;
#[cfg(test)]
#[path = "budget_tests.rs"]
mod tests;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Budget {
pub pages: Quota,
pub endpoints: Quota,
pub notifications: Quota,
pub regions: Quota,
}
impl Budget {
pub const EMPTY: Self = Self::new(Limits {
pages: 0,
endpoints: 0,
notifications: 0,
regions: 0,
});
pub const fn new(limits: Limits) -> Self {
Self {
pages: Quota::new(limits.pages),
endpoints: Quota::new(limits.endpoints),
notifications: Quota::new(limits.notifications),
regions: Quota::new(limits.regions),
}
}
pub fn limits(&self) -> Limits {
Limits {
pages: self.pages.limit(),
endpoints: self.endpoints.limit(),
notifications: self.notifications.limit(),
regions: self.regions.limit(),
}
}
fn parts(&mut self) -> [&mut Quota; 4] {
[
&mut self.pages,
&mut self.endpoints,
&mut self.notifications,
&mut self.regions,
]
}
/// Carves `child` out of the unused budget: all of it or nothing.
pub fn carve(&mut self, child: Limits) -> bool {
let amounts = amounts(child);
let fits = self.parts().iter().zip(amounts).all(|(q, n)| {
q.limit()
.checked_sub(n)
.is_some_and(|rest| rest >= q.used())
});
if fits {
for (q, n) in self.parts().into_iter().zip(amounts) {
q.carve(n);
}
}
fits
}
pub fn refund(&mut self, child: Limits) {
for (q, n) in self.parts().into_iter().zip(amounts(child)) {
q.refund(n);
}
}
}
fn amounts(l: Limits) -> [usize; 4] {
[l.pages, l.endpoints, l.notifications, l.regions]
}

35
crates/domain/src/budget_tests.rs Normal file
View File

@@ -0,0 +1,35 @@
use super::{Budget, Limits};
const L: Limits = Limits {
pages: 10,
endpoints: 2,
notifications: 2,
regions: 1,
};
#[test]
fn a_carve_takes_all_parts_or_none() {
let mut b = Budget::new(L);
b.endpoints.charge(2);
let child = Limits {
pages: 1,
endpoints: 1,
..Limits::default()
};
assert!(!b.carve(child));
assert_eq!(b.limits(), L);
assert!(b.carve(Limits {
endpoints: 0,
..child
}));
assert_eq!(b.limits().pages, 9);
}
#[test]
fn a_refund_restores_what_was_carved() {
let mut b = Budget::new(L);
assert!(b.carve(L));
assert_eq!(b.limits(), Limits::default());
b.refund(L);
assert_eq!(b.limits(), L);
}

9
crates/domain/src/capability/derivation.rs
View File

@@ -2,10 +2,7 @@
//! capability is a node with at most one parent. Kept free of kernel
//! types so the revocation rule can be checked on its own.
/// `result[i]` is true iff `root` is `i` or an ancestor of `i`.
///
/// `parents` must be acyclic, which holds because a capability's parent
/// always exists before it does.
/// `result[i]` is true iff `root` is `i` or an ancestor of `i`. `parents` must be acyclic.
pub fn descendants<const N: usize>(parents: &[Option<u16>; N], root: usize) -> [bool; N] {
let mut marked = [false; N];
let Some(r) = marked.get_mut(root) else {
@@ -57,9 +54,7 @@ mod proofs {
false
}
/// On every acyclic forest of `N` capabilities, `descendants` marks
/// exactly the nodes `root` is an ancestor of: revocation removes the
/// whole subtree and nothing else.
/// On every acyclic forest, `descendants` marks exactly the subtree under `root`.
#[kani::proof]
#[kani::unwind(6)]
fn marks_exactly_the_subtree() {

9
crates/domain/src/capability/mod.rs
View File

@@ -5,11 +5,16 @@ use types::{
pub mod authority;
pub mod derivation;
pub mod tree;
#[cfg(test)]
#[path = "derivation_tests.rs"]
mod derivation_tests;
#[cfg(test)]
#[path = "tree_tests.rs"]
mod tree_tests;
use crate::{error::KernelError, generations::SlotGenerations, resource::Resource};
#[cfg(test)]
@@ -19,9 +24,7 @@ mod tests;
// Sized so that every representable `CapSlot` indexes a real slot.
pub const MAX_CAPS_PER_PROCESS: usize = CapSlot::MAX as usize + 1;
/// Names one capability slot at one point in time. Both generations must
/// still match for the reference to resolve, so a reused pid or slot is
/// never mistaken for the capability that used to live there.
/// One capability slot at one point in time; both generations must match to resolve.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CapRef {
pub process: ProcessId,

206
crates/domain/src/capability/tree.rs Normal file
View File

@@ -0,0 +1,206 @@
//! Derivation tree with explicit links: delete costs its children, revoke its subtree.
//! Nodes are `0..N`; `derivation::descendants` is the reference model it is checked against.
/// Longest walk any operation may take: every node visited once, plus the
/// climb back up. Corrupted links can therefore never hang the kernel.
const fn budget(n: usize) -> usize {
2 * n + 1
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct Node {
live: bool,
parent: Option<u16>,
first_child: Option<u16>,
prev: Option<u16>,
next: Option<u16>,
}
pub struct CapTree<const N: usize> {
nodes: [Node; N],
}
impl<const N: usize> Default for CapTree<N> {
fn default() -> Self {
Self::new()
}
}
fn id(i: usize) -> Option<u16> {
u16::try_from(i).ok()
}
impl<const N: usize> CapTree<N> {
pub fn new() -> Self {
Self {
nodes: [Node::default(); N],
}
}
fn node(&self, i: usize) -> Option<&Node> {
self.nodes.get(i).filter(|n| n.live)
}
fn node_mut(&mut self, i: usize) -> Option<&mut Node> {
self.nodes.get_mut(i).filter(|n| n.live)
}
pub fn is_live(&self, i: usize) -> bool {
self.node(i).is_some()
}
pub fn parent(&self, i: usize) -> Option<usize> {
self.node(i)?.parent.map(usize::from)
}
/// Adds `i` under `parent` (or as a root); fails without change if `i` is live or out of range, or `parent` is not live.
pub fn attach(&mut self, i: usize, parent: Option<usize>) -> bool {
let (Some(me), Some(node)) = (id(i), self.nodes.get(i)) else {
return false;
};
if node.live || parent.is_some_and(|p| p == i || !self.is_live(p)) {
return false;
}
let next = parent
.and_then(|p| self.node(p))
.and_then(|p| p.first_child);
if let Some(n) = next.and_then(|n| self.node_mut(n.into())) {
n.prev = Some(me);
}
if let Some(p) = parent.and_then(|p| self.node_mut(p)) {
p.first_child = Some(me);
}
if let Some(node) = self.nodes.get_mut(i) {
*node = Node {
live: true,
parent: parent.and_then(id),
first_child: None,
prev: None,
next,
};
}
true
}
/// Unlinks `i` from its parent's child list, leaving its own children.
fn unlink(&mut self, i: usize) {
let Some(node) = self.node(i).copied() else {
return;
};
match node.prev {
Some(p) => {
if let Some(p) = self.node_mut(p.into()) {
p.next = node.next;
}
}
None => {
if let Some(p) = node.parent.and_then(|p| self.node_mut(p.into())) {
p.first_child = node.next;
}
}
}
if let Some(n) = node.next.and_then(|n| self.node_mut(n.into())) {
n.prev = node.prev;
}
}
/// Removes `i`; its children are adopted by its parent (or become
/// roots), so no descendant escapes a revocation further up.
pub fn detach(&mut self, i: usize) {
let Some(node) = self.node(i).copied() else {
return;
};
self.unlink(i);
let mut child = node.first_child;
for _ in 0..budget(N) {
let Some(c) = child else { break };
let next = self.node(c.into()).and_then(|n| n.next);
if let Some(n) = self.node_mut(c.into()) {
n.parent = None;
n.prev = None;
n.next = None;
}
// Re-attach under the grandparent; `attach` needs a dead node.
if let Some(n) = self.nodes.get_mut(usize::from(c)) {
n.live = false;
let kids = n.first_child;
if self.attach(c.into(), node.parent.map(usize::from))
&& let Some(n) = self.nodes.get_mut(usize::from(c))
{
n.first_child = kids;
}
}
child = next;
}
if let Some(n) = self.nodes.get_mut(i) {
*n = Node::default();
}
}
/// The direct children of `i`, written into `out`; returns how many.
pub fn children(&self, i: usize, out: &mut [u16; N]) -> usize {
let mut count = 0;
let mut cur = self.node(i).and_then(|n| n.first_child);
for _ in 0..budget(N) {
let Some(c) = cur else { break };
if let Some(slot) = out.get_mut(count) {
*slot = c;
count += 1;
}
cur = self.node(c.into()).and_then(|n| n.next);
}
count
}
/// Every node strictly below `root`, written into `out` in pre-order;
/// returns how many.
pub fn descendants(&self, root: usize, out: &mut [u16; N]) -> usize {
let mut count = 0;
let Some(start) = self.node(root) else {
return 0;
};
let mut cur = start.first_child;
for _ in 0..budget(N) {
let Some(c) = cur else { break };
if let Some(slot) = out.get_mut(count) {
*slot = c;
count += 1;
}
cur = self.next_in_subtree(root, c.into());
}
count
}
/// The node after `cur` in a pre-order walk of `root`'s subtree.
fn next_in_subtree(&self, root: usize, cur: usize) -> Option<u16> {
if let Some(child) = self.node(cur)?.first_child {
return Some(child);
}
let mut at = cur;
for _ in 0..budget(N) {
if at == root {
return None;
}
let node = self.node(at)?;
if let Some(next) = node.next {
return Some(next);
}
at = node.parent?.into();
}
None
}
/// Removes every node strictly below `root`, keeping `root`.
pub fn remove_descendants(&mut self, root: usize) {
let mut doomed = [0u16; N];
let n = self.descendants(root, &mut doomed);
for d in doomed.iter().take(n) {
if let Some(node) = self.nodes.get_mut(usize::from(*d)) {
*node = Node::default();
}
}
if let Some(r) = self.node_mut(root) {
r.first_child = None;
}
}
}

120
crates/domain/src/capability/tree_tests.rs Normal file
View File

@@ -0,0 +1,120 @@
use super::derivation::descendants;
use super::tree::CapTree;
const N: usize = 12;
/// The tree as a plain parent array, updated by the rules the tree must
/// follow; `descendants` from derivation.rs answers subtree questions.
struct Model {
live: [bool; N],
parents: [Option<u16>; N],
}
impl Model {
fn detach(&mut self, i: usize) {
let grandparent = self.parents[i];
for p in self.parents.iter_mut() {
if *p == Some(i as u16) {
*p = grandparent;
}
}
self.live[i] = false;
self.parents[i] = None;
}
fn below(&self, root: usize) -> [bool; N] {
let mut d = descendants(&self.parents, root);
d[root] = false;
d
}
}
struct Rng(u64);
impl Rng {
fn below(&mut self, n: usize) -> usize {
self.0 ^= self.0 << 13;
self.0 ^= self.0 >> 7;
self.0 ^= self.0 << 17;
(self.0 % n as u64) as usize
}
}
fn check(t: &CapTree<N>, m: &Model) {
for i in 0..N {
assert_eq!(t.is_live(i), m.live[i], "liveness of {i}");
assert_eq!(t.parent(i), m.parents[i].map(usize::from), "parent of {i}");
if !m.live[i] {
continue;
}
let mut out = [0u16; N];
let n = t.descendants(i, &mut out);
let mut got = [false; N];
for d in &out[..n] {
assert!(!got[*d as usize], "{d} listed twice under {i}");
got[*d as usize] = true;
}
assert_eq!(got, m.below(i), "subtree of {i}");
}
}
#[test]
fn matches_the_parent_array_model_under_random_operations() {
for seed in 1..200u64 {
let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15));
let mut t = CapTree::<N>::new();
let mut m = Model {
live: [false; N],
parents: [None; N],
};
for _ in 0..300 {
let i = rng.below(N);
match rng.below(4) {
0 | 1 => {
let parent = match rng.below(3) {
0 => None,
_ => Some(rng.below(N)),
};
let ok = !m.live[i] && parent.is_none_or(|p| p != i && m.live[p]);
assert_eq!(t.attach(i, parent), ok, "attach {i} under {parent:?}");
if ok {
m.live[i] = true;
m.parents[i] = parent.map(|p| p as u16);
}
}
2 => {
t.detach(i);
if m.live[i] {
m.detach(i);
}
}
_ => {
t.remove_descendants(i);
if m.live[i] {
let below = m.below(i);
for (j, b) in below.iter().enumerate() {
if *b {
m.live[j] = false;
m.parents[j] = None;
}
}
}
}
}
check(&t, &m);
}
}
}
#[test]
fn detaching_a_middle_node_hands_its_children_up() {
let mut t = CapTree::<4>::new();
assert!(t.attach(0, None));
assert!(t.attach(1, Some(0)));
assert!(t.attach(2, Some(1)));
assert!(t.attach(3, Some(1)));
t.detach(1);
assert_eq!(t.parent(2), Some(0));
assert_eq!(t.parent(3), Some(0));
let mut out = [0u16; 4];
assert_eq!(t.descendants(0, &mut out), 2);
}

7
crates/domain/src/endpoint/mod.rs
View File

@@ -154,13 +154,6 @@ impl EndpointTable {
self.generations.advance(id.raw() as usize);
Some(ep)
}
pub fn remove_thread_from_all(&mut self, thread: ThreadId) {
for ep in self.endpoints.iter_mut().flatten() {
ep.remove_sender(thread);
ep.remove_receiver(thread);
}
}
}
impl Default for EndpointTable {

4
crates/domain/src/endpoint/wait_queue.rs
View File

@@ -85,9 +85,7 @@ impl<const N: usize> Default for WaitQueue<N> {
mod proofs {
use super::*;
/// Against a simple reference model (a plain array kept in arrival
/// order), any sequence of pushes, pops and removals behaves the same:
/// FIFO order, no duplicates, nothing lost.
/// Matches a plain arrival-ordered array model: FIFO, no duplicates, nothing lost.
#[kani::proof]
#[kani::unwind(7)]
fn behaves_like_a_fifo_without_duplicates() {

4
crates/domain/src/generations.rs
View File

@@ -6,9 +6,7 @@ use types::generation::Generation;
#[path = "generations_tests.rs"]
mod tests;
/// A slot's generation changes every time the object in it is destroyed
/// or revoked, so capabilities minted earlier stop matching. A slot whose
/// counter runs out is retired for good rather than reused.
/// Bumped on every destroy or revoke so older capabilities stop matching; a slot whose counter runs out is retired.
pub struct SlotGenerations<const N: usize> {
gens: [Option<Generation>; N],
}

13
crates/domain/src/irq/mod.rs
View File

@@ -1,7 +1,7 @@
use types::{
badge::Badge,
generation::Generation,
id::{IrqId, NotificationId},
id::{IrqId, NotificationId, ProcessId},
};
#[cfg(test)]
@@ -17,6 +17,8 @@ pub struct IrqRoute {
pub notification: NotificationId,
pub generation: Generation,
pub badge: Badge,
/// The process that set the route, and so owns the line.
pub owner: ProcessId,
}
pub struct IrqRoutingTable {
@@ -53,6 +55,15 @@ impl IrqRoutingTable {
}
}
/// Drops every route `owner` set, when it dies.
pub fn clear_owned_by(&mut self, owner: ProcessId) {
for slot in self.routes.iter_mut() {
if slot.is_some_and(|r| r.owner == owner) {
*slot = None;
}
}
}
/// The route for `irq`; `None` for unknown or unrouted lines.
pub fn route(&self, irq: IrqId) -> Option<IrqRoute> {
*self.routes.get(irq.raw() as usize)?

1
crates/domain/src/irq/tests.rs
View File

@@ -8,6 +8,7 @@ fn route(n: u16) -> IrqRoute {
notification: NotificationId::from_raw(n),
generation: Generation::initial(),
badge: Badge::new(1 << n),
owner: types::id::ProcessId::from_raw(0),
}
}

1
crates/domain/src/lib.rs
View File

@@ -17,6 +17,7 @@
#[cfg(kani)]
pub mod proofs;
pub mod budget;
pub mod capability;
pub mod endpoint;
pub mod error;

42
crates/domain/src/memory/frame/mod.rs
View File

@@ -50,6 +50,8 @@ pub struct FrameTable<const N: usize = MAX_FRAMES> {
total: usize,
frames: [Option<FrameUse>; N],
free: usize,
/// Where the next allocation starts looking: just past the last one.
cursor: usize,
}
impl<const N: usize> FrameTable<N> {
@@ -60,6 +62,7 @@ impl<const N: usize> FrameTable<N> {
total,
frames: [None; N],
free: total,
cursor: 0,
}
}
@@ -101,14 +104,19 @@ impl<const N: usize> FrameTable<N> {
self.frames.get(self.index_of(pa)?)?.as_ref()
}
/// Next-fit: searches from just past the previous allocation, wrapping once.
pub fn alloc(&mut self, usage: FrameUse) -> Option<PhysicalAddress> {
let (idx, slot) = self
.live_mut()
.iter_mut()
.enumerate()
.find(|(_, f)| f.is_none())?;
*slot = Some(usage);
if self.free == 0 {
return None;
}
let total = self.total;
let start = self.cursor.min(total);
let idx = (start..total)
.chain(0..start)
.find(|&i| self.frames.get(i).is_some_and(Option::is_none))?;
*self.frames.get_mut(idx)? = Some(usage);
self.free -= 1;
self.cursor = idx + 1;
Some(self.address_of(idx))
}
@@ -167,7 +175,27 @@ impl<const N: usize> FrameTable<N> {
}
}
/// The frame mapped at `va` in `owner`, if `owner` holds it as a user page.
/// The perms of the frame at `pa` if it is `owner`'s user page at `va`; O(1), the caller supplies the frame.
pub fn user_at(
&self,
pa: PhysicalAddress,
owner: ProcessId,
va: VirtualAddress,
) -> Option<PagePerms> {
let page = u32::try_from(va.raw() / PAGE_SIZE).ok()?;
if !va.raw().is_multiple_of(PAGE_SIZE) {
return None;
}
match self.get(pa)? {
FrameUse::User {
owner: o,
page: p,
perms,
} if *o == owner && *p == page => Some(*perms),
_ => None,
}
}
pub fn find_user(
&self,
owner: ProcessId,

2
crates/domain/src/memory/frame/proofs.rs
View File

@@ -47,6 +47,8 @@ fn any_table() -> FrameTable<N> {
total: N,
frames,
free,
// Next-fit may resume anywhere, so the proofs start it anywhere.
cursor: kani::any::<usize>() % (N + 1),
}
}

27
crates/domain/src/memory/quota.rs
View File

@@ -1,5 +1,6 @@
/// Per-process page budget. Every frame charged to a process (user pages
/// and its page tables) counts against it.
/// A count against a limit: one of a process's budgets (see `budget`).
/// For pages, every frame charged to a process (user pages and its page
/// tables) counts against it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Quota {
used: usize,
@@ -7,8 +8,6 @@ pub struct Quota {
}
impl Quota {
pub const DEFAULT_LIMIT: usize = 1024;
pub const fn new(limit: usize) -> Self {
Self { used: 0, limit }
}
@@ -41,17 +40,19 @@ impl Quota {
}
}
pub fn set_limit(&mut self, limit: usize) -> bool {
if limit < self.used {
return false;
/// Gives up `amount` of the unused limit, for a child to hold.
pub fn carve(&mut self, amount: usize) -> bool {
match self.limit.checked_sub(amount) {
Some(rest) if rest >= self.used => {
self.limit = rest;
true
}
_ => false,
}
self.limit = limit;
true
}
}
impl Default for Quota {
fn default() -> Self {
Self::new(Self::DEFAULT_LIMIT)
/// Takes back a limit carved out earlier.
pub fn refund(&mut self, amount: usize) {
self.limit = self.limit.saturating_add(amount);
}
}

11
crates/domain/src/memory/quota_tests.rs
View File

@@ -28,10 +28,13 @@ fn uncharge_cannot_underflow() {
}
#[test]
fn limit_can_be_raised_but_not_below_usage() {
fn only_the_unused_limit_can_be_carved() {
let mut q = Quota::new(4);
q.charge(3);
assert!(!q.set_limit(2));
assert!(q.set_limit(10));
assert!(q.charge(7));
assert!(!q.carve(2));
assert_eq!(q.limit(), 4);
assert!(q.carve(1));
assert!(!q.charge(1));
q.refund(1);
assert!(q.charge(1));
}

11
crates/domain/src/notification/mod.rs
View File

@@ -116,6 +116,11 @@ impl Notification {
self.unbind(thread);
}
/// Takes `thread` out of the waiters, leaving any binding.
pub fn remove_waiter(&mut self, thread: ThreadId) {
self.waiters.remove_thread(thread);
}
/// Drains every waiter, e.g. when the notification is destroyed.
pub fn pop_waiter(&mut self) -> Option<ThreadId> {
self.waiters.pop()
@@ -177,12 +182,6 @@ impl NotificationTable {
.filter(move |n| n.owner == owner)
.map(|n| n.id)
}
pub fn remove_thread_from_all(&mut self, thread: ThreadId) {
for n in self.slots.iter_mut().flatten() {
n.remove_thread(thread);
}
}
}
impl Default for NotificationTable {

33
crates/domain/src/proofs/budget.rs Normal file
View File

@@ -0,0 +1,33 @@
use crate::budget::{Budget, Limits};
fn any_limits() -> Limits {
let l = Limits {
pages: kani::any(),
endpoints: kani::any(),
notifications: kani::any(),
regions: kani::any(),
};
kani::assume(l.pages < 1 << 20 && l.endpoints < 1 << 20);
kani::assume(l.notifications < 1 << 20 && l.regions < 1 << 20);
l
}
/// A carve never lets parent and child together exceed what the parent
/// had, never strands the parent's usage, and a refund undoes it.
#[kani::proof]
fn carving_conserves_the_budget() {
let (have, child) = (any_limits(), any_limits());
let used: usize = kani::any();
let mut b = Budget::new(have);
b.pages.charge(used);
let before = b;
if b.carve(child) {
assert_eq!(b.limits().pages + child.pages, have.pages);
assert_eq!(b.limits().regions + child.regions, have.regions);
assert!(b.pages.used() <= b.pages.limit());
b.refund(child);
assert_eq!(b, before);
} else {
assert_eq!(b, before);
}
}

1
crates/domain/src/proofs/mod.rs
View File

@@ -1,3 +1,4 @@
mod budget;
mod capability;
mod delegation;
mod thread;

2
crates/domain/src/resource.rs
View File

@@ -10,4 +10,6 @@ pub enum Resource {
ProcessCreate,
/// Machine-wide controls such as halting; given only to init at boot.
SystemControl,
/// The debug console (UART); held by the log server.
Console,
}

4
crates/domain/src/scheduler/priority_queue.rs
View File

@@ -17,9 +17,7 @@ struct ListEnds {
tail: Option<ThreadId>,
}
/// Ready queue: one FIFO per priority level, threaded through an
/// intrusive doubly linked list indexed by thread id. A thread is in at
/// most one list, so capacity never runs out and every op is O(1).
/// Ready queue: one FIFO per priority in an intrusive list indexed by thread id; O(1) ops.
pub struct PriorityQueue {
lists: [ListEnds; NUM_PRIORITIES],
links: [Links; MAX_THREADS],

7
crates/domain/src/syscall.rs
View File

@@ -62,9 +62,11 @@ pub enum Syscall {
AcknowledgeInterrupt {
irq: CapSlot,
},
/// The new process's `limits` are carved from the caller's budget.
CreateProcess {
create_cap: CapSlot,
image: DataRef,
limits: crate::budget::Limits,
},
Kill {
process: CapSlot,
@@ -93,9 +95,11 @@ pub enum Syscall {
DropCap {
slot: CapSlot,
},
/// `arg` arrives in the new thread's `a0`.
SpawnThread {
entry: EntryPoint,
stack: StackPointer,
arg: usize,
},
ExitThread,
RequestPages {
@@ -114,7 +118,9 @@ pub enum Syscall {
control: CapSlot,
},
CreateEndpoint,
/// Writes one byte to the console; needs a Console capability.
DebugPutchar {
console: CapSlot,
byte: u8,
},
MapPage {
@@ -127,5 +133,6 @@ pub enum Syscall {
process: CapSlot,
entry: EntryPoint,
stack: StackPointer,
arg: usize,
},
}

51
crates/domain/src/thread/context.rs Normal file
View File

@@ -0,0 +1,51 @@
//! Saved register state of a thread.
pub struct SavedContext {
registers: [usize; 32],
pc: usize,
status: usize,
}
impl SavedContext {
pub fn empty() -> Self {
Self {
registers: [0; 32],
pc: 0,
status: 0,
}
}
pub fn with_entry_and_stack(
entry: types::address::EntryPoint,
stack: types::address::StackPointer,
) -> Self {
let mut ctx = Self::empty();
ctx.pc = entry.raw();
ctx.registers[2] = stack.raw();
ctx
}
pub fn program_counter(&self) -> usize {
self.pc
}
pub fn set_pc(&mut self, pc: usize) {
self.pc = pc;
}
pub fn status(&self) -> usize {
self.status
}
pub fn set_status(&mut self, status: usize) {
self.status = status;
}
pub fn registers(&self) -> &[usize; 32] {
&self.registers
}
pub fn registers_mut(&mut self) -> &mut [usize; 32] {
&mut self.registers
}
}

185
crates/domain/src/thread/mod.rs
View File

@@ -1,11 +1,16 @@
use types::{
bounded::Priority,
generation::Generation,
id::{CoreId, EndpointId, NotificationId, ProcessId, ThreadId},
time::TimeSlice,
};
use crate::{error::KernelError, generations::SlotGenerations};
use crate::error::KernelError;
mod context;
mod table;
pub use context::SavedContext;
pub use table::{MAX_THREADS, ThreadTable};
#[cfg(test)]
#[path = "tests.rs"]
@@ -29,56 +34,6 @@ pub enum ThreadState {
Dead,
}
pub struct SavedContext {
registers: [usize; 32],
pc: usize,
status: usize,
}
impl SavedContext {
pub fn empty() -> Self {
Self {
registers: [0; 32],
pc: 0,
status: 0,
}
}
pub fn with_entry_and_stack(
entry: types::address::EntryPoint,
stack: types::address::StackPointer,
) -> Self {
let mut ctx = Self::empty();
ctx.pc = entry.raw();
ctx.registers[2] = stack.raw();
ctx
}
pub fn program_counter(&self) -> usize {
self.pc
}
pub fn set_pc(&mut self, pc: usize) {
self.pc = pc;
}
pub fn status(&self) -> usize {
self.status
}
pub fn set_status(&mut self, status: usize) {
self.status = status;
}
pub fn registers(&self) -> &[usize; 32] {
&self.registers
}
pub fn registers_mut(&mut self) -> &mut [usize; 32] {
&mut self.registers
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BlockReason {
/// Queued as a sender on the endpoint.
@@ -228,129 +183,3 @@ impl Thread {
}
}
}
pub const MAX_THREADS: usize = if cfg!(kani) { 3 } else { 256 };
pub struct ThreadTable {
threads: [Option<Thread>; MAX_THREADS],
generations: SlotGenerations<MAX_THREADS>,
}
impl ThreadTable {
pub fn new() -> Self {
Self {
threads: [const { None }; MAX_THREADS],
generations: SlotGenerations::new(),
}
}
/// Current generation of a live thread, so references to it (reply
/// slots) can detect that the id was reused.
pub fn generation(&self, id: ThreadId) -> Option<Generation> {
self.get(id).ok()?;
self.generations.current(id.raw() as usize)
}
fn free_slot(&self) -> Option<usize> {
self.threads
.iter()
.enumerate()
.position(|(i, t)| t.is_none() && self.generations.is_usable(i))
}
pub fn spawn(
&mut self,
process: ProcessId,
priority: Priority,
time_slice: TimeSlice,
core_affinity: Option<CoreId>,
) -> Result<ThreadId, KernelError> {
let idx = self.free_slot().ok_or(KernelError::TooManyThreads)?;
let id = ThreadId::from_raw(idx as u16);
*self
.threads
.get_mut(idx)
.ok_or(KernelError::TooManyThreads)? = Some(Thread::new(
id,
process,
priority,
time_slice,
core_affinity,
));
Ok(id)
}
pub fn spawn_with_context(
&mut self,
process: ProcessId,
priority: Priority,
time_slice: TimeSlice,
core_affinity: Option<CoreId>,
context: SavedContext,
) -> Result<ThreadId, KernelError> {
let idx = self.free_slot().ok_or(KernelError::TooManyThreads)?;
let id = ThreadId::from_raw(idx as u16);
*self
.threads
.get_mut(idx)
.ok_or(KernelError::TooManyThreads)? = Some(Thread::with_context(
id,
process,
priority,
time_slice,
core_affinity,
context,
));
Ok(id)
}
/// Every live thread with its id.
pub fn iter(&self) -> impl Iterator<Item = (ThreadId, &Thread)> {
self.threads
.iter()
.enumerate()
.filter_map(|(i, t)| Some((ThreadId::from_raw(i as u16), t.as_ref()?)))
}
pub fn get(&self, id: ThreadId) -> Result<&Thread, KernelError> {
let idx = id.raw() as usize;
self.threads
.get(idx)
.and_then(|s| s.as_ref())
.ok_or(KernelError::InvalidThread(id))
}
pub fn get_mut(&mut self, id: ThreadId) -> Result<&mut Thread, KernelError> {
let idx = id.raw() as usize;
self.threads
.get_mut(idx)
.and_then(|s| s.as_mut())
.ok_or(KernelError::InvalidThread(id))
}
pub fn destroy(&mut self, id: ThreadId) -> Result<Thread, KernelError> {
let idx = id.raw() as usize;
let thread = self
.threads
.get_mut(idx)
.and_then(|s| s.take())
.ok_or(KernelError::InvalidThread(id))?;
self.generations.advance(idx);
Ok(thread)
}
pub fn threads_for_process(&self, process: ProcessId) -> impl Iterator<Item = &Thread> {
self.threads
.iter()
.filter_map(|s| s.as_ref())
.filter(move |t| t.process() == process)
}
}
impl Default for ThreadTable {
fn default() -> Self {
Self::new()
}
}

137
crates/domain/src/thread/table.rs Normal file
View File

@@ -0,0 +1,137 @@
//! Fixed-capacity table of threads.
use types::{
bounded::Priority,
generation::Generation,
id::{CoreId, ProcessId, ThreadId},
time::TimeSlice,
};
use super::{SavedContext, Thread};
use crate::{error::KernelError, generations::SlotGenerations};
pub const MAX_THREADS: usize = if cfg!(kani) { 3 } else { 256 };
pub struct ThreadTable {
threads: [Option<Thread>; MAX_THREADS],
generations: SlotGenerations<MAX_THREADS>,
}
impl ThreadTable {
pub fn new() -> Self {
Self {
threads: [const { None }; MAX_THREADS],
generations: SlotGenerations::new(),
}
}
/// Current generation of a live thread, so references to it (reply
/// slots) can detect that the id was reused.
pub fn generation(&self, id: ThreadId) -> Option<Generation> {
self.get(id).ok()?;
self.generations.current(id.raw() as usize)
}
fn free_slot(&self) -> Option<usize> {
self.threads
.iter()
.enumerate()
.position(|(i, t)| t.is_none() && self.generations.is_usable(i))
}
pub fn spawn(
&mut self,
process: ProcessId,
priority: Priority,
time_slice: TimeSlice,
core_affinity: Option<CoreId>,
) -> Result<ThreadId, KernelError> {
let idx = self.free_slot().ok_or(KernelError::TooManyThreads)?;
let id = ThreadId::from_raw(idx as u16);
*self
.threads
.get_mut(idx)
.ok_or(KernelError::TooManyThreads)? = Some(Thread::new(
id,
process,
priority,
time_slice,
core_affinity,
));
Ok(id)
}
pub fn spawn_with_context(
&mut self,
process: ProcessId,
priority: Priority,
time_slice: TimeSlice,
core_affinity: Option<CoreId>,
context: SavedContext,
) -> Result<ThreadId, KernelError> {
let idx = self.free_slot().ok_or(KernelError::TooManyThreads)?;
let id = ThreadId::from_raw(idx as u16);
*self
.threads
.get_mut(idx)
.ok_or(KernelError::TooManyThreads)? = Some(Thread::with_context(
id,
process,
priority,
time_slice,
core_affinity,
context,
));
Ok(id)
}
/// Every live thread with its id.
pub fn iter(&self) -> impl Iterator<Item = (ThreadId, &Thread)> {
self.threads
.iter()
.enumerate()
.filter_map(|(i, t)| Some((ThreadId::from_raw(i as u16), t.as_ref()?)))
}
pub fn get(&self, id: ThreadId) -> Result<&Thread, KernelError> {
let idx = id.raw() as usize;
self.threads
.get(idx)
.and_then(|s| s.as_ref())
.ok_or(KernelError::InvalidThread(id))
}
pub fn get_mut(&mut self, id: ThreadId) -> Result<&mut Thread, KernelError> {
let idx = id.raw() as usize;
self.threads
.get_mut(idx)
.and_then(|s| s.as_mut())
.ok_or(KernelError::InvalidThread(id))
}
pub fn destroy(&mut self, id: ThreadId) -> Result<Thread, KernelError> {
let idx = id.raw() as usize;
let thread = self
.threads
.get_mut(idx)
.and_then(|s| s.take())
.ok_or(KernelError::InvalidThread(id))?;
self.generations.advance(idx);
Ok(thread)
}
pub fn threads_for_process(&self, process: ProcessId) -> impl Iterator<Item = &Thread> {
self.threads
.iter()
.filter_map(|s| s.as_ref())
.filter(move |t| t.process() == process)
}
}
impl Default for ThreadTable {
fn default() -> Self {
Self::new()
}
}

1
crates/drivers/block/virtio/Cargo.toml
View File

@@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2024"
[dependencies]
libkos = { workspace = true }
port-block-device = { workspace = true }
hw-virtio = { workspace = true }
types = { workspace = true }

40
crates/drivers/block/virtio/src/lib.rs
View File

@@ -9,7 +9,6 @@ use types::address::{PhysicalAddress, VirtualAddress};
const MMIO_BASE: usize = 0x20000;
const VQUEUE_VA: usize = 0x21000;
const REQ_HDR_VA: usize = 0x23000;
const PHYSINFO_VA: usize = 0x24000;
const SECTOR_SIZE: usize = 512;
struct UserMmio;
@@ -30,28 +29,34 @@ pub struct VirtioBlockDevice {
}
impl VirtioBlockDevice {
pub fn new() -> Self {
let queue = VirtioQueue::new(QueueAddrs::from_base(VQUEUE_VA));
let (hdr_pa, status_pa, data_pa) = unsafe {
let p = PHYSINFO_VA as *const u64;
(
core::ptr::read_volatile(p) as usize,
core::ptr::read_volatile(p.add(1)) as usize,
core::ptr::read_volatile(p.add(2)) as usize,
)
};
/// Takes over the virtio block device behind `mmio`. `None` if it is
/// not one or any step fails.
pub fn new(mmio: types::bounded::CapSlot) -> Option<Self> {
libkos::device::map_mmio(mmio, MMIO_BASE).ok()?;
let regs = UserMmio;
if regs.read(hw_virtio::mmio::REG_MAGIC) != hw_virtio::mmio::MAGIC
|| regs.read(hw_virtio::mmio::REG_DEVICE_ID) != hw_virtio::mmio::DEVICE_ID_BLOCK
{
return None;
}
let vq_pa = libkos::device::dma(2, VQUEUE_VA).ok()?;
let hdr_pa = libkos::device::dma(1, REQ_HDR_VA).ok()?;
hw_virtio::mmio::init_legacy_device(&regs, vq_pa, hw_virtio::queue::QUEUE_SIZE as u32);
// Header, status byte and one sector share the request page.
let dma = DmaAddrs {
header_pa: PhysicalAddress::from_raw(hdr_pa),
header_va: VirtualAddress::from_raw(REQ_HDR_VA),
status_pa: PhysicalAddress::from_raw(status_pa),
status_pa: PhysicalAddress::from_raw(hdr_pa + 16),
status_va: VirtualAddress::from_raw(REQ_HDR_VA + 16),
data_pa: PhysicalAddress::from_raw(data_pa),
data_pa: PhysicalAddress::from_raw(hdr_pa + 32),
data_va: VirtualAddress::from_raw(REQ_HDR_VA + 32),
};
Self { mmio: UserMmio, queue, dma }
Some(Self {
mmio: UserMmio,
queue: VirtioQueue::new(QueueAddrs::from_base(VQUEUE_VA)),
dma,
})
}
}
@@ -70,4 +75,7 @@ impl BlockDevice for VirtioBlockDevice {
}
fn write_sector(&mut self, _sector: SectorId, _buf: &[u8]) -> bool { false }
/// Writing is not implemented yet.
fn writable(&self) -> bool { false }
}

1
crates/drivers/framebuffer/virtio/Cargo.toml
View File

@@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2024"
[dependencies]
libkos = { workspace = true }
port-framebuffer = { workspace = true }
hw-virtio = { workspace = true }
types = { workspace = true }

74
crates/drivers/framebuffer/virtio/src/lib.rs
View File

@@ -10,9 +10,11 @@ use types::display::{Color, Dimension, ScreenCoord};
const GPU_MMIO_VA: usize = 0x20000;
const VQUEUE_VA: usize = 0x21000;
const CMDBUF_VA: usize = 0x23000;
const PHYSINFO_VA: usize = 0x24000;
const FB_VA: usize = 0x100000;
const FB_BPP: usize = 4;
const RESOURCE_ID: u32 = 1;
/// When the device will not say.
const DEFAULT_SIZE: (u16, u16) = (640, 480);
struct GpuMmio;
@@ -25,25 +27,6 @@ impl MmioAccess for GpuMmio {
}
}
struct PhysInfo {
cmd_pa: PhysicalAddress,
resp_pa: PhysicalAddress,
width: u16,
height: u16,
}
fn read_phys_info() -> PhysInfo {
unsafe {
let p = PHYSINFO_VA as *const u64;
PhysInfo {
cmd_pa: PhysicalAddress::from_raw(core::ptr::read_volatile(p) as usize),
resp_pa: PhysicalAddress::from_raw(core::ptr::read_volatile(p.add(1)) as usize),
width: core::ptr::read_volatile(p.add(3)) as u16,
height: core::ptr::read_volatile(p.add(4)) as u16,
}
}
}
pub struct VirtioFramebuffer {
mmio: GpuMmio,
queue: VirtioQueue,
@@ -54,21 +37,50 @@ pub struct VirtioFramebuffer {
}
impl VirtioFramebuffer {
pub fn try_new() -> Option<Self> {
let phys = read_phys_info();
if phys.width == 0 || phys.height == 0 || phys.cmd_pa.raw() == 0 {
/// Takes over the virtio GPU behind `mmio` and makes a framebuffer its
/// scanout. `None` if it is not a GPU or any step fails.
pub fn try_new(mmio: types::bounded::CapSlot) -> Option<Self> {
libkos::device::map_mmio(mmio, GPU_MMIO_VA).ok()?;
let regs = GpuMmio;
if regs.read(hw_virtio::mmio::REG_MAGIC) != hw_virtio::mmio::MAGIC
|| regs.read(hw_virtio::mmio::REG_DEVICE_ID) != hw_virtio::mmio::DEVICE_ID_GPU
{
return None;
}
let vq_pa = libkos::device::dma(2, VQUEUE_VA).ok()?;
let cmd_pa = libkos::device::dma(1, CMDBUF_VA).ok()?;
hw_virtio::mmio::init_legacy_device(&regs, vq_pa, hw_virtio::queue::QUEUE_SIZE as u32);
let queue = VirtioQueue::new(QueueAddrs::from_base(VQUEUE_VA));
// Commands go out from the start of the page, responses come back
// in its second half.
let dma = GpuDmaAddrs {
cmd_pa: PhysicalAddress::from_raw(cmd_pa),
cmd_va: VirtualAddress::from_raw(CMDBUF_VA),
resp_pa: PhysicalAddress::from_raw(cmd_pa + 512),
};
let (w, h) = gpu::get_display_info(&regs, &queue, &dma)
.and_then(|(w, h)| Some((u16::try_from(w).ok()?, u16::try_from(h).ok()?)))
.filter(|(w, h)| *w > 0 && *h > 0)
.unwrap_or(DEFAULT_SIZE);
let (width, height) = (Dimension::new(w), Dimension::new(h));
let fb_bytes = w as usize * h as usize * FB_BPP;
let fb_pages = u16::try_from(fb_bytes.div_ceil(4096)).ok()?;
let fb_pa = PhysicalAddress::from_raw(libkos::device::dma(fb_pages, FB_VA).ok()?);
let ready = gpu::create_2d(&regs, &queue, &dma, RESOURCE_ID, width, height)
&& gpu::attach_backing(&regs, &queue, &dma, RESOURCE_ID, fb_pa, fb_bytes as u32)
&& gpu::set_scanout(&regs, &queue, &dma, 0, RESOURCE_ID, width, height);
if !ready {
return None;
}
let width = Dimension::new(phys.width);
let height = Dimension::new(phys.height);
let stride = phys.width as usize * FB_BPP;
Some(Self {
mmio: GpuMmio,
queue: VirtioQueue::new(QueueAddrs::from_base(VQUEUE_VA)),
dma: GpuDmaAddrs { cmd_pa: phys.cmd_pa, cmd_va: VirtualAddress::from_raw(CMDBUF_VA), resp_pa: phys.resp_pa },
queue,
dma,
width,
height,
stride,
stride: w as usize * FB_BPP,
})
}
}
@@ -101,7 +113,7 @@ impl Framebuffer for VirtioFramebuffer {
fn flush(&mut self) {
let w = self.width.raw() as u32;
let h = self.height.raw() as u32;
gpu::transfer_to_host(&self.mmio, &self.queue, &self.dma, 1, 0, 0, w, h);
gpu::resource_flush(&self.mmio, &self.queue, &self.dma, 1, 0, 0, w, h);
gpu::transfer_to_host(&self.mmio, &self.queue, &self.dma, RESOURCE_ID, 0, 0, w, h);
gpu::resource_flush(&self.mmio, &self.queue, &self.dma, RESOURCE_ID, 0, 0, w, h);
}
}

2
crates/drivers/input/uart/Cargo.toml
View File

@@ -4,4 +4,6 @@ version = "0.1.0"
edition = "2024"
[dependencies]
libkos = { workspace = true }
types = { workspace = true }
port-input-device = { workspace = true }

13
crates/drivers/input/uart/src/lib.rs
View File

@@ -3,13 +3,22 @@
use port_input_device::{InputDevice, InputEvent, Key};
const UART_VA: usize = 0x20000;
const UART_IER_OFFSET: usize = 1;
const UART_LSR_OFFSET: usize = 5;
/// IER bit: interrupt when a received byte is waiting (cleared by reading
/// it, so `acknowledge` has nothing to do).
const IER_RX_READY: u8 = 1;
pub struct UartInputDevice;
impl UartInputDevice {
pub fn new() -> Self {
Self
/// Maps the 16550 UART whose region `mmio` names and enables its
/// receive interrupt.
pub fn new(mmio: types::bounded::CapSlot) -> Option<Self> {
libkos::device::map_mmio(mmio, UART_VA).ok()?;
// SAFETY: UART_VA now maps the UART's registers.
unsafe { core::ptr::write_volatile((UART_VA + UART_IER_OFFSET) as *mut u8, IER_RX_READY) };
Some(Self)
}
fn read_byte(&self) -> Option<u8> {

1
crates/drivers/input/virtio/Cargo.toml
View File

@@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2024"
[dependencies]
libkos = { workspace = true }
port-input-device = { workspace = true }
hw-virtio = { workspace = true }
types = { workspace = true }

28
crates/drivers/input/virtio/src/lib.rs
View File

@@ -6,7 +6,6 @@ use port_input_device::{InputDevice, InputEvent, Key};
const INPUT_MMIO_VA: usize = 0x25000;
const VQUEUE_VA: usize = 0x26000;
const EVBUF_VA: usize = 0x28000;
const PHYSINFO_VA: usize = 0x29000;
const QUEUE_SIZE: usize = 32;
const EVENT_SIZE: usize = 8;
@@ -28,20 +27,31 @@ impl hw_virtio::mmio::MmioAccess for MmioInput {
pub struct VirtioInputDevice {
last_used: u16,
next_avail: u16,
evbuf_pa: usize,
}
impl VirtioInputDevice {
pub fn try_new() -> Option<Self> {
let phys = unsafe { core::ptr::read_volatile(PHYSINFO_VA as *const u64) };
if phys == 0 { return None; }
/// Takes over the virtio input device behind `mmio`. `None` if it is
/// not one or any step fails.
pub fn try_new(mmio: types::bounded::CapSlot) -> Option<Self> {
libkos::device::map_mmio(mmio, INPUT_MMIO_VA).ok()?;
let regs = MmioInput;
if regs.read(hw_virtio::mmio::REG_MAGIC) != hw_virtio::mmio::MAGIC
|| regs.read(hw_virtio::mmio::REG_DEVICE_ID) != hw_virtio::mmio::DEVICE_ID_INPUT
{
return None;
}
let vq_pa = libkos::device::dma(2, VQUEUE_VA).ok()?;
let evbuf_pa = libkos::device::dma(1, EVBUF_VA).ok()?;
hw_virtio::mmio::init_legacy_device(&regs, vq_pa, QUEUE_SIZE as u32);
let mut dev = Self { last_used: 0, next_avail: 0 };
let mut dev = Self { last_used: 0, next_avail: 0, evbuf_pa };
dev.fill_queue();
Some(dev)
}
fn fill_queue(&mut self) {
let evbuf_phys = unsafe { core::ptr::read_volatile(PHYSINFO_VA as *const u64) } as usize;
let evbuf_phys = self.evbuf_pa;
let desc_base = VQUEUE_VA;
let avail_base = VQUEUE_VA + QUEUE_SIZE * 16;
@@ -157,4 +167,10 @@ impl InputDevice for VirtioInputDevice {
fn has_event(&self) -> bool {
self.used_idx() != self.last_used
}
fn acknowledge(&mut self) {
let regs = MmioInput;
let status = regs.read(hw_virtio::mmio::REG_INTERRUPT_STATUS);
regs.write(hw_virtio::mmio::REG_INTERRUPT_ACK, status);
}
}

3
crates/hw-virtio/src/gpu.rs
View File

@@ -75,7 +75,7 @@ pub fn resource_flush(mmio: &impl MmioAccess, queue: &VirtioQueue, dma: &GpuDmaA
fn write_cmd_header(cmd_va: VirtualAddress, cmd_type: u32) {
unsafe {
let p = cmd_va.raw() as *mut u8;
// Use volatile writes to zero the buffer — ensures device sees the zeros
// Volatile so the device sees the zeros.
for i in 0..128 {
core::ptr::write_volatile((p as *mut u32).add(i), 0);
}
@@ -91,7 +91,6 @@ pub fn get_display_info(mmio: &impl MmioAccess, queue: &VirtioQueue, dma: &GpuDm
];
if !queue.submit(mmio, &descs) { return None; }
unsafe {
let resp = dma.resp_pa.raw() as usize;
let resp_va = dma.cmd_va.raw() + 512;
let resp_type = core::ptr::read_volatile(resp_va as *const u32);
if resp_type != 0x1101 { return None; }

5
crates/hw-virtio/src/mmio.rs
View File

@@ -19,6 +19,8 @@ pub const REG_QUEUE_NUM_MAX: usize = 0x034;
pub const REG_QUEUE_NUM: usize = 0x038;
pub const REG_QUEUE_PFN: usize = 0x040;
pub const REG_QUEUE_NOTIFY: usize = 0x050;
pub const REG_INTERRUPT_STATUS: usize = 0x060;
pub const REG_INTERRUPT_ACK: usize = 0x064;
pub const STATUS_ACK: u32 = 1;
pub const STATUS_DRIVER: u32 = 2;
@@ -87,8 +89,7 @@ pub const MMIO_FIRST: usize = 0x10001000;
pub const MMIO_STRIDE: usize = 0x1000;
pub const MMIO_SLOTS: usize = 8;
/// Probes the MMIO slots starting at `first`, the address the caller sees
/// slot 0 at (`MMIO_FIRST` itself, or wherever it is mapped).
/// Probes the MMIO slots, with slot 0 at `first` as the caller maps it.
pub fn scan_for_device(first: usize, device_id: u32) -> Option<DirectMmio> {
for slot in 0..MMIO_SLOTS {
if let Some(mmio) = probe_slot(first + slot * MMIO_STRIDE, device_id) {

1
crates/init/Cargo.toml
View File

@@ -10,6 +10,7 @@ bench = false
[dependencies]
elf = { workspace = true }
hw-virtio = { workspace = true }
types = { workspace = true }
libkos = { workspace = true }

103
crates/init/src/devices.rs Normal file
View File

@@ -0,0 +1,103 @@
//! Finds the board's devices (`types::bootinfo`): disk, GPU, and keyboard
//! (or the UART when there is none).
use hw_virtio::mmio::{
DEVICE_ID_BLOCK, DEVICE_ID_GPU, DEVICE_ID_INPUT, MAGIC, REG_DEVICE_ID, REG_MAGIC,
};
use libkos::device::{self, Device};
use libkos::syscall;
use types::badge::Badge;
use types::bootinfo::{BOOT_INFO_VA, BootInfo, KIND_UART_16550, KIND_VIRTIO, NO_SLOT};
use types::bounded::CapSlot;
use types::permission::{Permission, PermissionSet};
/// Where init briefly maps a device to ask what it is (top of the window).
const PROBE_VA: usize = 0x01F0_0000;
pub struct Devices {
pub disk: Device,
pub gpu: Device,
pub input: Device,
}
fn slot(raw: u64) -> Option<CapSlot> {
if raw == NO_SLOT {
return None;
}
CapSlot::new(u8::try_from(raw).ok()?)
}
/// The virtio device id behind `region`, 0 for an empty slot. Read through
/// a copy of the capability, mapped only while looking and then dropped,
/// which unmaps it.
fn virtio_id(region: CapSlot) -> u32 {
let perms = PermissionSet::empty().with(Permission::Read).with(Permission::Write);
let Ok(copy) = syscall::mint(region, Badge::NONE, perms) else {
return 0;
};
let id = match device::map_mmio(copy, PROBE_VA) {
// SAFETY: PROBE_VA maps the device's registers until the drop below.
Ok(()) => unsafe {
let reg = |off: usize| core::ptr::read_volatile((PROBE_VA + off) as *const u32);
if reg(REG_MAGIC) == MAGIC { reg(REG_DEVICE_ID) } else { 0 }
},
Err(_) => 0,
};
syscall::drop_cap(copy);
id
}
fn boot_info() -> &'static BootInfo {
// SAFETY: boot maps the BootInfo page read-only at BOOT_INFO_VA before
// init runs; readers check its magic.
unsafe { &*(BOOT_INFO_VA as *const BootInfo) }
}
/// The capability that halts the machine, if boot said where it is.
pub fn system_control() -> Option<CapSlot> {
let info = boot_info();
if info.magic != types::bootinfo::MAGIC {
return None;
}
slot(info.system_control_slot)
}
/// The console capability, until init hands it to the log server.
pub fn console() -> Option<CapSlot> {
let info = boot_info();
if info.magic != types::bootinfo::MAGIC {
return None;
}
slot(info.console_slot)
}
/// Sorts the board's devices out.
pub fn discover() -> Devices {
let info = boot_info();
let mut found = Devices { disk: Device::NONE, gpu: Device::NONE, input: Device::NONE };
let mut uart = Device::NONE;
for d in info.devices() {
let Some(region) = slot(d.region_slot) else {
continue;
};
let dev = Device { mmio: Some(region), irq: slot(d.irq_slot), kind: d.kind as usize };
let place = match d.kind {
KIND_UART_16550 => &mut uart,
KIND_VIRTIO => match virtio_id(region) {
DEVICE_ID_BLOCK => &mut found.disk,
DEVICE_ID_GPU => &mut found.gpu,
DEVICE_ID_INPUT => &mut found.input,
_ => continue,
},
_ => continue,
};
// First device of each kind wins.
if place.mmio.is_none() {
*place = dev;
}
}
if found.input.mmio.is_none() {
found.input = uart;
}
found
}

173
crates/init/src/lib.rs
View File

@@ -2,51 +2,61 @@
extern crate alloc;
mod devices;
mod selftest;
use types::address::{EntryPoint, StackPointer, VirtualAddress};
use types::bounded::{CapSlot, PageCount};
use types::budget::Limits;
use types::permission::{Permission, PermissionSet};
use libkos::fs::Storage;
use libkos::syscall;
const PAGE_SIZE: usize = 4096;
const USER_CODE_BASE: usize = 0x10000;
const USER_STACK_TOP: StackPointer = StackPointer::from_raw(USER_CODE_BASE);
pub(crate) const USER_STACK_TOP: StackPointer = StackPointer::from_raw(USER_CODE_BASE);
const USER_STACK_PAGES: usize = 4;
const CMD_READ_FILE: usize = 1;
/// Where a program file is read to before it is loaded (16 pages: the
/// largest program init will load).
const SCRATCH_VA: usize = 0x30000;
const SCRATCH_PAGES: u16 = 16;
/// Where the buffer storage shares with init is mapped.
const STORAGE_BUF_VA: usize = 0x0180_0000;
const CREATE_CAP: CapSlot = CapSlot::const_new(0);
const STORAGE_EP: CapSlot = CapSlot::const_new(2);
const DISPLAY_EP: CapSlot = CapSlot::const_new(3);
const INPUT_EP: CapSlot = CapSlot::const_new(5);
fn request_file(shared_buf: *mut u8, filename: &[u8]) -> usize {
unsafe {
for (i, &b) in filename.iter().enumerate() {
core::ptr::write_volatile(shared_buf.add(i), b);
}
}
syscall::send(STORAGE_EP, CMD_READ_FILE, filename.len(), 0, 0);
let (file_size, _, _, _) = syscall::receive(STORAGE_EP);
file_size
/// Reads all of `filename` into `scratch`; its size, or `None` if it is
/// missing, unreadable or larger than `scratch`.
fn read_file(storage: &mut Storage, scratch: &mut [u8], filename: &[u8]) -> Option<usize> {
let handle = storage.open(filename).ok()?;
let size = storage.file_size(handle).ok().filter(|s| *s <= scratch.len());
let read = size.and_then(|s| storage.read(handle, &mut scratch[..s]).ok());
let _ = storage.close(handle);
read.filter(|r| Some(*r) == size)
}
struct LoadedProcess {
process_cap: CapSlot,
entry_point: EntryPoint,
pub(crate) struct LoadedProcess {
pub(crate) process_cap: CapSlot,
pub(crate) entry_point: EntryPoint,
}
fn load_binary(shared_buf: *mut u8, filename: &[u8]) -> Option<LoadedProcess> {
let file_size = request_file(shared_buf, filename);
if file_size == 0 {
return None;
}
let elf_data = unsafe { core::slice::from_raw_parts(shared_buf as *const u8, file_size) };
pub(crate) fn load_binary(
storage: &mut Storage,
scratch: &mut [u8],
filename: &[u8],
limits: Limits,
) -> Option<LoadedProcess> {
let file_size = read_file(storage, scratch, filename)?;
let elf_data = &scratch[..file_size];
let elf_info = match elf::parse(elf_data) {
Ok(info) => info,
Err(_) => return None,
};
let process_cap = syscall::create_process(CREATE_CAP)?;
let process_cap = syscall::create_process(CREATE_CAP, limits)?;
for segment in elf_info.segments() {
let page_start = segment.vaddr & !0xFFF;
@@ -63,16 +73,15 @@ fn load_binary(shared_buf: *mut u8, filename: &[u8]) -> Option<LoadedProcess> {
}
let copy_len = segment.file_size.min(segment.mem_size);
if copy_len > 0 {
let src = segment.file_offset;
if src + copy_len <= file_size {
unsafe {
core::ptr::copy_nonoverlapping(
shared_buf.add(src),
local_ptr.add(offset_in_page),
copy_len,
);
}
let src = segment
.file_offset
.checked_add(copy_len)
.and_then(|end| elf_data.get(segment.file_offset..end));
if let Some(src) = src.filter(|_| copy_len > 0) {
// SAFETY: the pages just requested hold seg_pages * PAGE_SIZE
// bytes, which covers offset_in_page + copy_len.
unsafe {
core::ptr::copy_nonoverlapping(src.as_ptr(), local_ptr.add(offset_in_page), copy_len);
}
}
@@ -99,19 +108,50 @@ fn load_binary(shared_buf: *mut u8, filename: &[u8]) -> Option<LoadedProcess> {
struct ServerInfo {
filename: &'static [u8],
name: &'static str,
limits: Limits,
}
/// Enough for a small program: its image, stack and a little heap.
pub(crate) const SMALL: Limits = Limits {
pages: 64,
endpoints: 2,
notifications: 2,
regions: 2,
};
const SERVERS: &[ServerInfo] = &[
ServerInfo { filename: b"log.elf", name: "log" },
ServerInfo { filename: b"permis.elf", name: "permis" },
ServerInfo { filename: b"session.elf", name: "session" },
ServerInfo { filename: b"log.elf", name: "log", limits: SMALL },
ServerInfo { filename: b"permis.elf", name: "permis", limits: SMALL },
ServerInfo { filename: b"session.elf", name: "session", limits: SMALL },
];
pub fn run(shared_buf_va: usize) -> ! {
let shared_buf = shared_buf_va as *mut u8;
const SNAKE: Limits = Limits { pages: 128, ..SMALL };
pub fn run() -> ! {
libkos::start::set_console(devices::console());
libkos::println!("init: started (pid={:#x})", syscall::get_process_id().raw());
// Drivers wait for their device before serving, so this comes first:
// storage needs its disk before init can load anything.
let found = devices::discover();
for (name, ep, dev) in [
("disk", STORAGE_EP, found.disk),
("gpu", DISPLAY_EP, found.gpu),
("input", INPUT_EP, found.input),
] {
let given = libkos::device::give_device(ep, dev).is_ok() && dev.mmio.is_some();
libkos::println!("init: {}: {}", name, if given { "handed to its driver" } else { "none" });
}
let Ok(mut storage) = Storage::attach(STORAGE_EP, STORAGE_BUF_VA) else {
libkos::println!("init: cannot reach storage, nothing to load");
idle();
};
let Some(scratch) = scratch() else {
libkos::println!("init: no memory to load programs into");
idle();
};
let ipc_perms = PermissionSet::empty()
.with(Permission::Read)
.with(Permission::Execute);
@@ -119,7 +159,7 @@ pub fn run(shared_buf_va: usize) -> ! {
for server in SERVERS {
libkos::print!("init: loading {}... ", server.name);
let loaded = match load_binary(shared_buf, server.filename) {
let loaded = match load_binary(&mut storage, scratch, server.filename, server.limits) {
Some(l) => l,
None => {
libkos::println!("SKIP");
@@ -129,17 +169,27 @@ pub fn run(shared_buf_va: usize) -> ! {
if let Some(ep_cap) = syscall::create_endpoint() {
syscall::delegate(ep_cap, loaded.process_cap, ipc_perms);
syscall::spawn_thread_in(loaded.process_cap, loaded.entry_point, USER_STACK_TOP);
start(&loaded, true);
}
libkos::println!("ok (entry={:#x})", loaded.entry_point.raw());
}
// The test image carries the SMP self-test; it runs instead of the
// desktop and halts the machine.
if storage.open(b"smptest.elf").is_ok_and(|h| storage.close(h).is_ok()) {
libkos::println!("init: running the SMP self-test");
selftest::run(&mut storage, scratch);
}
libkos::print!("init: loading snake... ");
if let Some(snake) = load_binary(shared_buf, b"snake.elf") {
syscall::delegate(INPUT_EP, snake.process_cap, ipc_perms);
syscall::delegate(DISPLAY_EP, snake.process_cap, ipc_perms);
syscall::spawn_thread_in(snake.process_cap, snake.entry_point, USER_STACK_TOP);
if let Some(snake) = load_binary(&mut storage, scratch, b"snake.elf", SNAKE) {
// A client only calls: Execute, never Read, so it cannot pose as
// the server and take other clients' requests.
let client = PermissionSet::empty().with(Permission::Execute);
syscall::delegate(INPUT_EP, snake.process_cap, client);
syscall::delegate(DISPLAY_EP, snake.process_cap, client);
start(&snake, false);
libkos::println!("ok");
} else {
libkos::println!("SKIP");
@@ -147,7 +197,42 @@ pub fn run(shared_buf_va: usize) -> ! {
libkos::println!("init: all loaded, supervising");
idle();
}
/// Starts `program`, lending it init's console if `console`.
pub(crate) fn start(program: &LoadedProcess, console: bool) {
let write = PermissionSet::empty().with(Permission::Write);
let slot = libkos::start::console()
.filter(|_| console)
.and_then(|c| syscall::delegate(c, program.process_cap, write));
let arg = types::start_arg::encode(slot, None);
syscall::spawn_thread_in(program.process_cap, program.entry_point, USER_STACK_TOP, arg);
}
/// A region of init's own to read program files into.
fn scratch() -> Option<&'static mut [u8]> {
let region = syscall::create_region(PageCount::new(SCRATCH_PAGES)?).ok()?;
syscall::map_region(
region,
VirtualAddress::from_raw(SCRATCH_VA),
types::mapping::PagePerms::ReadWrite,
)
.ok()?;
// SAFETY: the region is mapped read-write at SCRATCH_VA for good.
Some(unsafe { core::slice::from_raw_parts_mut(SCRATCH_VA as *mut u8, SCRATCH_PAGES as usize * PAGE_SIZE) })
}
/// Nothing to supervise yet: block on a notification nobody signals
/// rather than spin a core on Yield.
pub(crate) fn idle() -> ! {
let idle = syscall::create_notification();
loop {
syscall::yield_now();
match idle {
Ok(n) => {
let _ = syscall::wait(n);
}
Err(_) => syscall::yield_now(),
}
}
}

4
crates/init/src/main.rs
View File

@@ -4,6 +4,6 @@
extern crate libkos;
#[unsafe(no_mangle)]
extern "C" fn main(shared_buf: usize) {
init::run(shared_buf);
extern "C" fn main() {
init::run();
}

108
crates/init/src/selftest.rs Normal file
View File

@@ -0,0 +1,108 @@
//! SMP self-test, run when the disk carries `smptest.elf`: after revoking a
//! spinner's page or killing it, no spinner counter may move on any hart.
use core::sync::atomic::{AtomicUsize, Ordering};
use libkos::fs::Storage;
use libkos::syscall;
use types::address::VirtualAddress;
use types::bounded::{CapSlot, PageCount};
use types::mapping::PagePerms;
use types::permission::{Permission, PermissionSet};
const OBSERVE_VA: usize = 0x0140_0000;
const THREADS: usize = 4;
/// QEMU virt's timebase is 10 MHz.
const TICKS_PER_MS: u64 = 10_000;
fn word(i: usize) -> &'static AtomicUsize {
// SAFETY: OBSERVE_VA maps the counter page for the whole test.
unsafe { &*((OBSERVE_VA + i * 8) as *const AtomicUsize) }
}
fn counters() -> [usize; THREADS] {
core::array::from_fn(|i| word(1 + i).load(Ordering::SeqCst))
}
/// Spins (yielding) until `done` or `ms` milliseconds pass.
fn wait_until(ms: u64, mut done: impl FnMut() -> bool) -> bool {
let deadline = syscall::get_time().raw() + ms * TICKS_PER_MS;
while syscall::get_time().raw() < deadline {
if done() {
return true;
}
syscall::yield_now();
}
done()
}
/// Starts a spinner on `page`; its process capability.
fn start_spinner(storage: &mut Storage, scratch: &mut [u8], page: CapSlot) -> Result<CapSlot, &'static str> {
for i in 0..=THREADS {
word(i).store(0, Ordering::SeqCst);
}
let spinner = crate::load_binary(storage, scratch, b"smptest.elf", crate::SMALL).ok_or("cannot load smptest.elf")?;
let rw = PermissionSet::empty()
.with(Permission::Read)
.with(Permission::Write)
.with(Permission::Delegate);
// The page first: the spinner expects it in slot 0.
if syscall::delegate(page, spinner.process_cap, rw).is_none() {
return Err("cannot give the spinner its page");
}
crate::start(&spinner, true);
if !wait_until(3000, || counters().iter().all(|c| *c > 0)) {
return Err("spinner threads did not all start");
}
Ok(spinner.process_cap)
}
/// After `stop`, no counter may move.
fn check_stopped(what: &'static str) -> Result<(), &'static str> {
let before = counters();
wait_until(200, || false);
if counters() != before {
return Err(what);
}
Ok(())
}
fn run_phases(storage: &mut Storage, scratch: &mut [u8]) -> Result<(), &'static str> {
let page = syscall::create_region(PageCount::new(1).ok_or("page count")?).map_err(|_| "no counter page")?;
syscall::map_region(page, VirtualAddress::from_raw(OBSERVE_VA), PagePerms::ReadWrite)
.map_err(|_| "cannot map the counter page")?;
// Revoke must unmap the page on every hart; the spinner then faults and dies.
let spinner = start_spinner(storage, scratch, page)?;
if !syscall::revoke(page) {
return Err("revoke failed");
}
check_stopped("a counter moved after its page was revoked (stale TLB)")?;
if syscall::kill(spinner) {
return Err("the spinner survived losing its page");
}
libkos::println!("smptest: revoke stops every hart: ok");
// Kill must stop threads running on other harts before it returns.
let spinner = start_spinner(storage, scratch, page)?;
if !syscall::kill(spinner) {
return Err("kill failed");
}
check_stopped("a counter moved after its process was killed")?;
libkos::println!("smptest: kill stops every hart: ok");
Ok(())
}
/// Runs the test, reports, and halts the machine.
pub fn run(storage: &mut Storage, scratch: &mut [u8]) -> ! {
match run_phases(storage, scratch) {
Ok(()) => libkos::println!("SMPTEST PASS"),
Err(why) => libkos::println!("SMPTEST FAIL: {}", why),
}
match crate::devices::system_control() {
Some(control) => syscall::halt(control),
None => {
libkos::println!("smptest: no SystemControl capability to halt with");
crate::idle()
}
}
}

17
crates/kernel/src/abi.rs
View File

@@ -34,9 +34,7 @@ fn message(a: &[usize; 6]) -> Option<Message> {
})
}
/// Decodes syscall `nr` with arguments `a0..a5`. `None` for an unknown
/// number or an argument with no meaning (e.g. a slot out of range); the
/// caller then gets `UnknownSyscall` and nothing else happens.
/// Decodes syscall `nr` with args `a0..a5`; `None` for an unknown number or meaningless argument.
pub fn decode(nr: usize, a: &[usize; 6]) -> Option<Syscall> {
Some(match nr {
SYS_EXIT => Syscall::Exit,
@@ -96,6 +94,12 @@ pub fn decode(nr: usize, a: &[usize; 6]) -> Option<Syscall> {
SYS_CREATE_PROCESS => Syscall::CreateProcess {
create_cap: slot(a[0])?,
image: DataRef::empty(),
limits: domain::budget::Limits {
pages: a[1],
endpoints: a[2],
notifications: a[3],
regions: a[4],
},
},
SYS_KILL => Syscall::Kill {
process: slot(a[0])?,
@@ -105,6 +109,7 @@ pub fn decode(nr: usize, a: &[usize; 6]) -> Option<Syscall> {
SYS_SPAWN_THREAD => Syscall::SpawnThread {
entry: EntryPoint::from_raw(a[0]),
stack: StackPointer::from_raw(a[1]),
arg: a[2],
},
SYS_REQUEST_PAGES => Syscall::RequestPages {
count: pages(a[0])?,
@@ -117,6 +122,7 @@ pub fn decode(nr: usize, a: &[usize; 6]) -> Option<Syscall> {
process: slot(a[0])?,
entry: EntryPoint::from_raw(a[1]),
stack: StackPointer::from_raw(a[2]),
arg: a[3],
},
SYS_CREATE_ENDPOINT => Syscall::CreateEndpoint,
SYS_CREATE_REGION => Syscall::CreateRegion {
@@ -130,7 +136,10 @@ pub fn decode(nr: usize, a: &[usize; 6]) -> Option<Syscall> {
SYS_REGION_ADDRESS => Syscall::RegionAddress {
region: slot(a[0])?,
},
SYS_DEBUG_PUTCHAR => Syscall::DebugPutchar { byte: a[0] as u8 },
SYS_DEBUG_PUTCHAR => Syscall::DebugPutchar {
console: slot(a[0])?,
byte: a[1] as u8,
},
SYS_MAP_PAGE => Syscall::MapPage {
process: slot(a[0])?,
src_va: VirtualAddress::from_raw(a[1]),

53
crates/kernel/src/board.rs
View File

@@ -1,13 +1,10 @@
//! The authority init starts with. The board describes its devices; the
//! kernel turns each into capabilities (an MMIO region and an IRQ line)
//! and adds one SystemControl capability. Init then decides who gets what;
//! nothing else is privileged by its process id.
//! The authority init starts with: per-device MMIO and IRQ capabilities plus one SystemControl.
use domain::{error::KernelError, resource::Resource};
use platform::Platform;
use types::{
address::PhysicalAddress,
bounded::PageCount,
bounded::{CapSlot, PageCount},
id::{IrqId, ProcessId, RegionId},
permission::{Permission, PermissionSet},
};
@@ -23,10 +20,22 @@ pub struct BoardDevice {
pub irq: Option<IrqId>,
}
/// Where `grant_boot_authority` put init's machine-wide capabilities.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BootCaps {
pub system_control: CapSlot,
pub console: CapSlot,
}
/// Where `grant_boot_authority` put one device's capabilities in init.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BoardCaps {
pub region: CapSlot,
pub irq: Option<CapSlot>,
}
impl<P: Platform> Kernel<P> {
/// Creates one device region per board device, owned by `init`. Fails
/// on the first device that is misaligned, overlaps the frame pool or
/// another device, or names an IRQ the kernel cannot route.
/// Creates one device region per board device, owned by `init`; fails on the first invalid one.
pub fn register_board(
&mut self,
init: ProcessId,
@@ -43,30 +52,36 @@ impl<P: Platform> Kernel<P> {
Ok(())
}
/// Gives `init` a capability to every registered device's region and
/// IRQ line, then one SystemControl capability. All carry Delegate so
/// init can hand them on. Done separately from `register_board` so
/// boot code can wire drivers to the regions first.
/// Gives `init` (with Delegate) caps to every device region and IRQ, then SystemControl.
/// Writes each device's slots into `placed` in order; returns the SystemControl
/// and Console slots.
pub fn grant_boot_authority(
&mut self,
init: ProcessId,
devices: &[BoardDevice],
) -> Result<(), KernelError> {
placed: &mut [Option<BoardCaps>],
) -> Result<BootCaps, KernelError> {
let delegate = PermissionSet::empty().with(Permission::Delegate);
let control = delegate.with(Permission::Execute);
let mmio = delegate.with(Permission::Read).with(Permission::Write);
for device in devices {
for (i, device) in devices.iter().enumerate() {
let region = self
.device_region_at(device.base)
.filter(|id| self.regions.get(*id).is_some_and(|r| r.owner() == init))
.ok_or(KernelError::NoSuchObject)?;
self.grant_cap(init, Resource::Memory(region), mmio)?;
if let Some(irq) = device.irq {
self.grant_cap(init, Resource::Interrupt(irq), control)?;
let region = self.grant_cap(init, Resource::Memory(region), mmio)?;
let irq = match device.irq {
Some(irq) => Some(self.grant_cap(init, Resource::Interrupt(irq), control)?),
None => None,
};
if let Some(slot) = placed.get_mut(i) {
*slot = Some(BoardCaps { region, irq });
}
}
self.grant_cap(init, Resource::SystemControl, control)?;
Ok(())
Ok(BootCaps {
system_control: self.grant_cap(init, Resource::SystemControl, control)?,
console: self.grant_cap(init, Resource::Console, delegate.with(Permission::Write))?,
})
}
/// The device region that starts at `base`.

100
crates/kernel/src/budget.rs Normal file
View File

@@ -0,0 +1,100 @@
//! Per-process budgets (see `domain::budget`): carved from the creator,
//! charged as a process takes frames and kernel objects, returned on death.
use domain::{
budget::{Budget, Limits},
capability::CapabilityTable,
error::KernelError,
memory::quota::Quota,
};
use platform::Platform;
use types::id::ProcessId;
use crate::Kernel;
/// Picks the part of a budget an object counts against.
pub(crate) type Part = fn(&mut Budget) -> &mut Quota;
pub(crate) const ENDPOINTS: Part = |b| &mut b.endpoints;
pub(crate) const NOTIFICATIONS: Part = |b| &mut b.notifications;
pub(crate) const REGIONS: Part = |b| &mut b.regions;
impl<P: Platform> Kernel<P> {
pub(crate) fn budget_mut(&mut self, pid: ProcessId) -> Result<&mut Budget, KernelError> {
self.budgets
.get_mut(pid.raw() as usize)
.ok_or(KernelError::InvalidProcess(pid))
}
pub fn limits_of(&self, pid: ProcessId) -> Option<Limits> {
self.processes.get(pid).ok()?;
self.budgets.get(pid.raw() as usize).map(Budget::limits)
}
/// A new process holding `limits`, carved from `parent`'s unused budget;
/// only boot makes one without a parent.
pub(crate) fn new_process(
&mut self,
parent: Option<ProcessId>,
limits: Limits,
) -> Result<ProcessId, KernelError> {
if let Some(p) = parent {
self.processes.get(p)?;
if !self.budget_mut(p)?.carve(limits) {
return Err(KernelError::OutOfMemory);
}
}
let pid = match self.processes.create(CapabilityTable::new(), None) {
Ok(pid) => pid,
Err(e) => {
if let Some(p) = parent {
self.budget_mut(p)?.refund(limits);
}
return Err(e);
}
};
*self.budget_mut(pid)? = Budget::new(limits);
if let Some(slot) = self.parents.get_mut(pid.raw() as usize) {
*slot = parent;
}
if let Err(e) = self.create_address_space(pid) {
self.release_budget(pid);
let _ = self.processes.destroy(pid);
return Err(e);
}
Ok(pid)
}
/// Gives a dead process's limits back to its parent; its children are
/// carved from what it held, so they pass to that parent too.
pub(crate) fn release_budget(&mut self, pid: ProcessId) {
let i = pid.raw() as usize;
let parent = self.parents.get_mut(i).and_then(Option::take);
let limits = self
.budgets
.get_mut(i)
.map(|b| core::mem::replace(b, Budget::EMPTY).limits());
if let (Some(p), Some(limits)) = (parent, limits)
&& let Ok(b) = self.budget_mut(p)
{
b.refund(limits);
}
for slot in self.parents.iter_mut().filter(|s| **s == Some(pid)) {
*slot = parent;
}
}
pub(crate) fn charge_object(&mut self, pid: ProcessId, part: Part) -> Result<(), KernelError> {
if part(self.budget_mut(pid)?).charge(1) {
Ok(())
} else {
Err(KernelError::NoFreeSlot)
}
}
pub(crate) fn uncharge_object(&mut self, pid: ProcessId, part: Part) {
if let Ok(b) = self.budget_mut(pid) {
part(b).uncharge(1);
}
}
}

123
crates/kernel/src/captree.rs
View File

@@ -1,10 +1,9 @@
//! The capability derivation tree. Every derived capability (Mint,
//! Delegate, IPC transfer) records its parent; `Revoke` deletes a subtree
//! and deleting a capability re-parents its children, so the tree can only
//! shrink by explicit deletion, never by a middleman stepping out.
//! Derivation tree: every derived capability records its parent; `Revoke` deletes a subtree,
//! deleting re-parents children. All capability insertion/removal goes through here so
//! `Capability::parent` and `Kernel::tree` stay in step.
use domain::{
capability::{CapRef, Capability, MAX_CAPS_PER_PROCESS, derivation::descendants},
capability::{CapRef, Capability, MAX_CAPS_PER_PROCESS},
error::KernelError,
process::MAX_PROCESSES,
};
@@ -17,7 +16,7 @@ use types::{
use crate::Kernel;
/// Every capability slot in the system, flattened.
const ALL_CAPS: usize = MAX_PROCESSES * MAX_CAPS_PER_PROCESS;
pub(crate) const ALL_CAPS: usize = MAX_PROCESSES * MAX_CAPS_PER_PROCESS;
fn flat(pid: ProcessId, slot: CapSlot) -> usize {
pid.raw() as usize * MAX_CAPS_PER_PROCESS + slot.value() as usize
@@ -46,47 +45,55 @@ impl<P: Platform> Kernel<P> {
self.cap_ref(r.process, r.slot) == Some(r)
}
/// Stores `cap` in `to`'s table as a child of `parent`.
/// Stores `cap` in `to`'s table as a child of `parent` (a root if
/// `None`).
pub(crate) fn insert_derived(
&mut self,
to: ProcessId,
cap: Capability,
parent: Option<CapRef>,
) -> Result<CapSlot, KernelError> {
self.processes
let parent = parent.filter(|r| self.resolves(*r));
let slot = self
.processes
.get_mut(to)?
.capabilities_mut()
.insert(cap.with_parent(parent))
}
/// Calls `f` with every live capability's flat index and reference.
fn for_each_cap(&self, mut f: impl FnMut(usize, CapRef, &Capability)) {
for p in 0..MAX_PROCESSES {
let pid = ProcessId::from_raw(p as u16);
let Ok(process) = self.processes.get(pid) else {
continue;
};
for (slot, cap) in process.capabilities().iter() {
if let Some(r) = self.cap_ref(pid, slot) {
f(flat(pid, slot), r, cap);
}
}
.insert(cap.with_parent(parent))?;
let linked = self
.tree
.attach(flat(to, slot), parent.map(|r| flat(r.process, r.slot)));
if !linked {
// A slot the tree still thinks is in use: refuse rather than
// let the two disagree.
let _ = self
.processes
.get_mut(to)
.map(|p| p.capabilities_mut().remove(slot));
return Err(KernelError::NoFreeSlot);
}
Ok(slot)
}
/// Removes every region mapping made through `via`.
fn unmap_mappings_via(&mut self, via: CapRef) {
let mut doomed: [Option<RegionId>; domain::memory::MAX_REGIONS] =
[None; domain::memory::MAX_REGIONS];
for (slot, (id, _)) in doomed.iter_mut().zip(
self.regions
.iter()
.filter(|(_, r)| r.mappings().any(|m| m.via == Some(via))),
) {
*slot = Some(id);
self.unmap_mappings_where(|r| r == via);
}
/// Removes every region mapping made through a capability `doomed`
/// picks, in one pass over the regions.
fn unmap_mappings_where(&mut self, doomed: impl Fn(CapRef) -> bool) {
const MAX: usize = domain::memory::MAX_REGIONS * domain::memory::MAX_REGION_MAPPINGS;
let mut found: [Option<(RegionId, ProcessId)>; MAX] = [None; MAX];
let hits = self.regions.iter().flat_map(|(id, r)| {
r.mappings()
.filter(|m| m.via.is_some_and(&doomed))
.map(move |m| (id, m.process))
});
for (slot, hit) in found.iter_mut().zip(hits) {
*slot = Some(hit);
}
for id in doomed.into_iter().flatten() {
self.unmap_region_from(id, via.process);
for (id, pid) in found.into_iter().flatten() {
self.unmap_region_from(id, pid);
}
}
@@ -99,22 +106,17 @@ impl<P: Platform> Kernel<P> {
let grandparent = self.processes.get(pid)?.capabilities().get(slot)?.parent();
self.unmap_mappings_via(me);
let mut children = [false; ALL_CAPS];
self.for_each_cap(|i, _, cap| {
if cap.parent() == Some(me)
&& let Some(c) = children.get_mut(i)
{
*c = true;
}
});
for (i, _) in children.iter().enumerate().filter(|(_, c)| **c) {
if let Some((cpid, cslot)) = unflat(i)
let mut children = [0u16; ALL_CAPS];
let n = self.tree.children(flat(pid, slot), &mut children);
for c in children.iter().take(n) {
if let Some((cpid, cslot)) = unflat(usize::from(*c))
&& let Ok(p) = self.processes.get_mut(cpid)
&& let Ok(child) = p.capabilities_mut().get_mut(cslot)
{
child.set_parent(grandparent);
}
}
self.tree.detach(flat(pid, slot));
self.processes.get_mut(pid)?.capabilities_mut().remove(slot)
}
@@ -128,31 +130,30 @@ impl<P: Platform> Kernel<P> {
if self.cap_ref(pid, slot).is_none() {
return Err(KernelError::InvalidSlot(slot));
}
// Snapshot the tree as parent indices. A parent that no longer
// resolves cannot exist (deletion re-parents), but is treated as
// a root rather than trusted.
let mut parents: [Option<u16>; ALL_CAPS] = [None; ALL_CAPS];
self.for_each_cap(|i, _, cap| {
if let Some(p) = parents.get_mut(i) {
*p = cap
.parent()
.filter(|r| self.resolves(*r))
.map(|r| flat(r.process, r.slot) as u16);
}
});
let root = flat(pid, slot);
let marked = descendants(&parents, root);
for (i, _) in marked.iter().enumerate().filter(|(i, m)| **m && *i != root) {
if let Some((dpid, dslot)) = unflat(i)
&& let Some(r) = self.cap_ref(dpid, dslot)
{
self.unmap_mappings_via(r);
let mut doomed = [0u16; ALL_CAPS];
let n = self.tree.descendants(root, &mut doomed);
let mut marked = [false; ALL_CAPS];
for d in doomed.iter().take(n) {
if let Some(m) = marked.get_mut(usize::from(*d)) {
*m = true;
}
}
self.unmap_mappings_where(|r| {
marked
.get(flat(r.process, r.slot))
.copied()
.unwrap_or(false)
});
for d in doomed.iter().take(n) {
if let Some((dpid, dslot)) = unflat(usize::from(*d)) {
let _ = self
.processes
.get_mut(dpid)
.map(|p| p.capabilities_mut().remove(dslot));
}
}
self.tree.remove_descendants(root);
Ok(())
}

170
crates/kernel/src/lib.rs
View File

@@ -16,6 +16,7 @@
pub mod abi;
mod board;
mod budget;
mod captree;
pub mod interrupt;
mod memory;
@@ -23,8 +24,9 @@ mod memory;
pub mod proofs;
mod smp;
pub mod syscall;
mod teardown;
pub use board::BoardDevice;
pub use board::{BoardCaps, BoardDevice, BootCaps};
pub use syscall::{SyscallOutcome, SyscallResult};
#[cfg(test)]
@@ -32,12 +34,13 @@ pub use syscall::{SyscallOutcome, SyscallResult};
mod tests;
use domain::{
capability::{Capability, CapabilityTable},
budget::{Budget, Limits},
capability::Capability,
endpoint::EndpointTable,
error::KernelError,
irq::IrqRoutingTable,
memory::{RegionTable, frame::FrameTable, quota::Quota},
notification::{MAX_NOTIFICATIONS, NotificationTable},
memory::{RegionTable, frame::FrameTable},
notification::NotificationTable,
process::{MAX_PROCESSES, ProcessTable},
resource::Resource,
scheduler::{Scheduler, multi_core::MultiCoreScheduler},
@@ -60,7 +63,9 @@ pub struct Kernel<P: Platform> {
endpoints: EndpointTable,
scheduler: MultiCoreScheduler,
frames: FrameTable,
quotas: [Quota; MAX_PROCESSES],
budgets: [Budget; MAX_PROCESSES],
/// Whose budget each process was carved from (see budget.rs).
parents: [Option<ProcessId>; MAX_PROCESSES],
regions: RegionTable,
irqs: IrqRoutingTable,
notifications: NotificationTable,
@@ -68,6 +73,10 @@ pub struct Kernel<P: Platform> {
pending_results: [Option<SyscallResult>; MAX_THREADS],
/// The core each thread runs on (see smp.rs).
homes: [types::id::CoreId; MAX_THREADS],
/// The capability derivation tree as links (see captree.rs).
tree: domain::capability::tree::CapTree<{ captree::ALL_CAPS }>,
/// Where each process's next heap search starts (see memory.rs).
heap_cursors: [usize; MAX_PROCESSES],
}
impl<P: Platform> Kernel<P> {
@@ -80,22 +89,23 @@ impl<P: Platform> Kernel<P> {
endpoints: EndpointTable::new(),
scheduler: MultiCoreScheduler::new(num_cores),
frames: FrameTable::new(pool_base, pool_pages),
quotas: [Quota::default(); MAX_PROCESSES],
budgets: [Budget::EMPTY; MAX_PROCESSES],
parents: [None; MAX_PROCESSES],
regions: RegionTable::new(),
irqs: IrqRoutingTable::new(),
notifications: NotificationTable::new(),
ipc: [syscall::IpcState::default(); MAX_THREADS],
pending_results: [const { None }; MAX_THREADS],
homes: [types::id::CoreId::from_raw(0); MAX_THREADS],
tree: domain::capability::tree::CapTree::new(),
heap_cursors: [0; MAX_PROCESSES],
}
}
/// Builds the kernel directly in `slot`, one table at a time, so the
/// whole structure never has to fit on the (small) boot stack.
/// Builds the kernel in place, one table at a time, to stay off the boot stack.
///
/// # Safety
/// `slot` must be valid for writes and properly aligned. Any previous
/// value is overwritten without being dropped.
/// `slot` must be valid for writes and aligned; the old value is overwritten without drop.
pub unsafe fn init_in_place(
slot: *mut Self,
platform: P,
@@ -113,13 +123,16 @@ impl<P: Platform> Kernel<P> {
addr_of_mut!((*slot).endpoints).write(EndpointTable::new());
addr_of_mut!((*slot).scheduler).write(MultiCoreScheduler::new(num_cores));
addr_of_mut!((*slot).frames).write(FrameTable::new(pool_base, pool_pages));
addr_of_mut!((*slot).quotas).write([Quota::default(); MAX_PROCESSES]);
addr_of_mut!((*slot).budgets).write([Budget::EMPTY; MAX_PROCESSES]);
addr_of_mut!((*slot).parents).write([None; MAX_PROCESSES]);
addr_of_mut!((*slot).regions).write(RegionTable::new());
addr_of_mut!((*slot).irqs).write(IrqRoutingTable::new());
addr_of_mut!((*slot).notifications).write(NotificationTable::new());
addr_of_mut!((*slot).ipc).write([syscall::IpcState::default(); MAX_THREADS]);
addr_of_mut!((*slot).pending_results).write([const { None }; MAX_THREADS]);
addr_of_mut!((*slot).homes).write([types::id::CoreId::from_raw(0); MAX_THREADS]);
addr_of_mut!((*slot).tree).write(domain::capability::tree::CapTree::new());
addr_of_mut!((*slot).heap_cursors).write([0; MAX_PROCESSES]);
}
}
@@ -128,11 +141,14 @@ impl<P: Platform> Kernel<P> {
priority: Priority,
time_slice: TimeSlice,
) -> Result<(ProcessId, ThreadId), KernelError> {
let pid = self.processes.create(CapabilityTable::new(), None)?;
if let Err(e) = self.create_address_space(pid) {
let _ = self.processes.destroy(pid);
return Err(e);
}
// Init holds every frame and every kernel object to start with.
let everything = Limits {
pages: self.frames.free_count(),
endpoints: domain::endpoint::MAX_ENDPOINTS,
notifications: domain::notification::MAX_NOTIFICATIONS,
regions: domain::memory::MAX_REGIONS,
};
let pid = self.new_process(None, everything)?;
let tid = self.threads.spawn(pid, priority, time_slice, None)?;
self.processes.get_mut(pid)?.add_thread(tid)?;
@@ -141,10 +157,7 @@ impl<P: Platform> Kernel<P> {
PermissionSet::empty().with(Permission::Execute),
Generation::initial(),
);
self.processes
.get_mut(pid)?
.capabilities_mut()
.insert(cap)?;
self.insert_derived(pid, cap, None)?;
let core = self.platform.core_id();
self.set_home(tid, core);
@@ -207,13 +220,13 @@ impl<P: Platform> Kernel<P> {
self.scheduler.next(core)
}
pub fn create_process_internal(&mut self) -> Option<ProcessId> {
let pid = self.processes.create(CapabilityTable::new(), None).ok()?;
if self.create_address_space(pid).is_err() {
let _ = self.processes.destroy(pid);
return None;
}
Some(pid)
/// A process for boot code to load, its `limits` carved from `parent`.
pub fn create_process_internal(
&mut self,
parent: ProcessId,
limits: Limits,
) -> Option<ProcessId> {
self.new_process(Some(parent), limits).ok()
}
pub fn spawn_thread_internal(&mut self, pid: ProcessId) -> Option<ThreadId> {
@@ -233,7 +246,11 @@ impl<P: Platform> Kernel<P> {
owner: ProcessId,
) -> Result<types::id::EndpointId, KernelError> {
self.processes.get(owner)?;
self.endpoints.create(owner).ok_or(KernelError::NoFreeSlot)
self.charge_object(owner, budget::ENDPOINTS)?;
self.endpoints
.create(owner)
.ok_or(KernelError::NoFreeSlot)
.inspect_err(|_| self.uncharge_object(owner, budget::ENDPOINTS))
}
pub fn grant_capability(
@@ -258,94 +275,6 @@ impl<P: Platform> Kernel<P> {
}
}
pub(crate) fn destroy_process(&mut self, pid: ProcessId) {
use domain::process::MAX_THREADS_PER_PROCESS;
let thread_ids: [Option<ThreadId>; MAX_THREADS_PER_PROCESS] = match self.processes.get(pid)
{
Ok(p) => {
let mut ids = [None; MAX_THREADS_PER_PROCESS];
for (slot, tid) in ids.iter_mut().zip(p.thread_ids()) {
*slot = Some(tid);
}
ids
}
Err(_) => return,
};
self.cleanup_threads(&thread_ids);
self.delete_all_caps(pid);
self.destroy_endpoints_owned_by(pid);
self.destroy_notifications_owned_by(pid);
self.teardown_memory(pid);
let _ = self.processes.destroy(pid);
}
pub(crate) fn cleanup_threads(&mut self, thread_ids: &[Option<ThreadId>]) {
use domain::scheduler::Scheduler;
for tid in thread_ids.iter().flatten() {
self.endpoints.remove_thread_from_all(*tid);
self.notifications.remove_thread_from_all(*tid);
let ipc = self.ipc_mut(*tid).map(core::mem::take).unwrap_or_default();
if let Some(caller) = ipc.reply_to {
self.release_caller(caller);
}
self.evict(*tid);
self.scheduler.remove(*tid);
self.set_pending_result(*tid, None);
let _ = self.threads.destroy(*tid);
}
}
/// Destroys every endpoint `owner` created, waking anyone blocked on
/// them with an error instead of leaving them blocked forever.
pub(crate) fn destroy_endpoints_owned_by(&mut self, owner: ProcessId) {
let mut owned = [None; domain::endpoint::MAX_ENDPOINTS];
for (slot, id) in owned.iter_mut().zip(self.endpoints.ids_owned_by(owner)) {
*slot = Some(id);
}
for id in owned.into_iter().flatten() {
let Some(mut ep) = self.endpoints.destroy(id) else {
continue;
};
while let Some(tid) = ep.dequeue_sender().or_else(|| ep.dequeue_receiver()) {
self.wake_with_error(tid, KernelError::InvalidEndpoint(id));
}
}
}
/// Destroys every notification `owner` created, waking its waiters
/// with an error and unbinding its bound thread.
pub(crate) fn destroy_notifications_owned_by(&mut self, owner: ProcessId) {
let mut owned = [None; MAX_NOTIFICATIONS];
for (slot, id) in owned.iter_mut().zip(self.notifications.ids_owned_by(owner)) {
*slot = Some(id);
}
for id in owned.into_iter().flatten() {
let Some(mut n) = self.notifications.destroy(id) else {
continue;
};
if let Some(bound) = n.bound()
&& let Some(s) = self.ipc_mut(bound)
{
s.bound = None;
}
while let Some(tid) = n.pop_waiter() {
self.wake_with_error(tid, KernelError::NoSuchObject);
}
}
}
/// Makes a blocked thread runnable with `error` as its syscall result,
/// dropping whatever it was about to send.
fn wake_with_error(&mut self, tid: ThreadId, error: KernelError) {
if let Some(s) = self.ipc_mut(tid) {
s.outgoing = None;
}
self.wake(tid, SyscallResult::Error(error));
}
fn check_cap(
&self,
owner: ProcessId,
@@ -371,9 +300,10 @@ impl<P: Platform> Kernel<P> {
Resource::Memory(id) => self.regions.generation(*id),
Resource::Notification(id) => self.notifications.generation(*id),
Resource::Process(pid) => self.processes.generation(*pid),
Resource::Interrupt(_) | Resource::ProcessCreate | Resource::SystemControl => {
Some(Generation::initial())
}
Resource::Interrupt(_)
| Resource::ProcessCreate
| Resource::SystemControl
| Resource::Console => Some(Generation::initial()),
}
}
@@ -387,6 +317,6 @@ impl<P: Platform> Kernel<P> {
.resource_generation(&resource)
.ok_or(KernelError::NoSuchObject)?;
let cap = Capability::new(resource, permissions, generation);
self.processes.get_mut(to)?.capabilities_mut().insert(cap)
self.insert_derived(to, cap, None)
}
}

486
crates/kernel/src/memory.rs
View File

@@ -1,486 +0,0 @@
//! Address-space bookkeeping. The kernel decides every user mapping from
//! its own records (`FrameTable`, `RegionTable`, quotas) and only then tells
//! the platform to apply it; it never asks the hardware what is mapped.
use domain::{
error::KernelError,
memory::{
Region, RegionBacking, RegionMapping,
frame::{FrameTable, FrameUse, PAGE_SIZE},
quota::Quota,
},
};
use platform::{FrameSource, MapError, Platform};
use types::{
address::{PhysicalAddress, VirtualAddress},
bounded::PageCount,
id::{ProcessId, RegionId},
mapping::PagePerms,
};
use crate::Kernel;
/// First VA the kernel considers when picking a spot for `RequestPages`.
/// Below it sit the program image and boot-time device windows.
pub(crate) const USER_HEAP_START: usize = 0x40000;
/// Page-table frames for one address space, charged to its owner.
struct ChargedFrames<'a> {
frames: &'a mut FrameTable,
quota: &'a mut Quota,
owner: ProcessId,
}
impl FrameSource for ChargedFrames<'_> {
fn alloc_table_frame(&mut self) -> Option<PhysicalAddress> {
if !self.quota.charge(1) {
return None;
}
match self.frames.alloc(FrameUse::PageTable { owner: self.owner }) {
Some(pa) => Some(pa),
None => {
self.quota.uncharge(1);
None
}
}
}
}
fn map_error(e: MapError) -> KernelError {
match e {
MapError::OutOfFrames => KernelError::OutOfMemory,
MapError::InvalidAddress => KernelError::InvalidAddress,
MapError::NoAddressSpace => KernelError::OutOfMemory,
}
}
impl<P: Platform> Kernel<P> {
/// Whether `[va, va + pages)` is page aligned and inside the user window.
pub(crate) fn in_user_window(va: VirtualAddress, pages: usize) -> bool {
let start = va.raw();
let Some(len) = pages.checked_mul(PAGE_SIZE) else {
return false;
};
let Some(end) = start.checked_add(len) else {
return false;
};
pages > 0
&& start.is_multiple_of(PAGE_SIZE)
&& start >= P::USER_VA_START
&& end <= P::USER_VA_END
}
fn is_mapped(&self, pid: ProcessId, va: VirtualAddress) -> bool {
self.frames.find_user(pid, va).is_some() || self.regions.covering(pid, va).is_some()
}
fn range_is_free(&self, pid: ProcessId, va: VirtualAddress, pages: usize) -> bool {
(0..pages).all(|i| !self.is_mapped(pid, page_at(va, i)))
}
fn quota_mut(&mut self, pid: ProcessId) -> Result<&mut Quota, KernelError> {
self.quotas
.get_mut(pid.raw() as usize)
.ok_or(KernelError::InvalidProcess(pid))
}
pub fn page_usage(&self, pid: ProcessId) -> usize {
self.quotas.get(pid.raw() as usize).map_or(0, Quota::used)
}
pub fn set_page_limit(&mut self, pid: ProcessId, limit: usize) -> Result<(), KernelError> {
self.processes.get(pid)?;
if self.quota_mut(pid)?.set_limit(limit) {
Ok(())
} else {
Err(KernelError::OutOfMemory)
}
}
pub fn free_page_count(&self) -> usize {
self.frames.free_count()
}
/// Maps `pa` at `va` in `pid`, building page tables on `pid`'s quota.
fn platform_map(
&mut self,
pid: ProcessId,
va: VirtualAddress,
pa: PhysicalAddress,
perms: PagePerms,
) -> Result<(), KernelError> {
let quota = self
.quotas
.get_mut(pid.raw() as usize)
.ok_or(KernelError::InvalidProcess(pid))?;
let mut source = ChargedFrames {
frames: &mut self.frames,
quota,
owner: pid,
};
self.platform
.map_page(pid, va, pa, perms, &mut source)
.map_err(map_error)
}
pub(crate) fn create_address_space(&mut self, pid: ProcessId) -> Result<(), KernelError> {
let quota = self
.quotas
.get_mut(pid.raw() as usize)
.ok_or(KernelError::InvalidProcess(pid))?;
*quota = Quota::default();
let mut source = ChargedFrames {
frames: &mut self.frames,
quota,
owner: pid,
};
let created = self
.platform
.create_address_space(pid, &mut source)
.map_err(map_error);
if created.is_err() {
self.frames.free_owned_by(pid);
if let Some(q) = self.quotas.get_mut(pid.raw() as usize) {
*q = Quota::default();
}
}
created
}
/// Allocates, zeroes and maps one fresh page for `pid` at `va`.
fn map_fresh_page(
&mut self,
pid: ProcessId,
va: VirtualAddress,
perms: PagePerms,
) -> Result<PhysicalAddress, KernelError> {
let usage = FrameUse::user(pid, va, perms).ok_or(KernelError::InvalidAddress)?;
if !self.quota_mut(pid)?.charge(1) {
return Err(KernelError::OutOfMemory);
}
let Some(pa) = self.frames.alloc(usage) else {
self.quota_mut(pid)?.uncharge(1);
return Err(KernelError::OutOfMemory);
};
self.platform.zero_frame(pa);
if let Err(e) = self.platform_map(pid, va, pa, perms) {
self.frames.free(pa);
self.quota_mut(pid)?.uncharge(1);
return Err(e);
}
Ok(pa)
}
/// Removes one user page of `pid` and returns its frame to the pool.
fn unmap_and_free(&mut self, pid: ProcessId, va: VirtualAddress) {
if let Some((pa, _)) = self.frames.find_user(pid, va) {
self.platform.unmap_page(pid, va);
self.frames.free(pa);
if let Some(q) = self.quotas.get_mut(pid.raw() as usize) {
q.uncharge(1);
}
}
}
/// Frame backing the user page at `va` in `pid`. Lets boot code fill
/// pages it has just mapped for a process.
pub fn user_page_frame(&self, pid: ProcessId, va: VirtualAddress) -> Option<PhysicalAddress> {
self.frames.find_user(pid, va).map(|(pa, _)| pa)
}
/// Maps `pages` fresh zeroed pages at `va`, all or nothing.
pub fn map_fresh_range(
&mut self,
pid: ProcessId,
va: VirtualAddress,
pages: usize,
perms: PagePerms,
) -> Result<(), KernelError> {
if !Self::in_user_window(va, pages) {
return Err(KernelError::InvalidAddress);
}
if !self.range_is_free(pid, va, pages) {
return Err(KernelError::AddressInUse);
}
for i in 0..pages {
if let Err(e) = self.map_fresh_page(pid, page_at(va, i), perms) {
for j in 0..i {
self.unmap_and_free(pid, page_at(va, j));
}
return Err(e);
}
}
Ok(())
}
/// First-fit search for `pages` free pages in the heap part of the window.
pub(crate) fn find_free_heap_range(
&self,
pid: ProcessId,
pages: usize,
) -> Option<VirtualAddress> {
let mut candidate = USER_HEAP_START.max(P::USER_VA_START);
while Self::in_user_window(VirtualAddress::from_raw(candidate), pages) {
let va = VirtualAddress::from_raw(candidate);
match (0..pages).find(|&i| self.is_mapped(pid, page_at(va, i))) {
None => return Some(va),
Some(busy) => candidate += (busy + 1) * PAGE_SIZE,
}
}
None
}
/// Validates that every page of the range is a user page `pid` owns,
/// then releases them all.
pub(crate) fn release_range(
&mut self,
pid: ProcessId,
va: VirtualAddress,
pages: usize,
) -> Result<(), KernelError> {
if !Self::in_user_window(va, pages) {
return Err(KernelError::InvalidAddress);
}
if !(0..pages).all(|i| self.frames.find_user(pid, page_at(va, i)).is_some()) {
return Err(KernelError::InvalidAddress);
}
for i in 0..pages {
self.unmap_and_free(pid, page_at(va, i));
}
Ok(())
}
/// Moves the page at `src` in `from` to `dst` in `to`. The frame and its
/// quota charge change hands; `from` loses access.
pub(crate) fn move_page(
&mut self,
from: ProcessId,
src: VirtualAddress,
to: ProcessId,
dst: VirtualAddress,
perms: PagePerms,
) -> Result<(), KernelError> {
if !Self::in_user_window(src, 1) || !Self::in_user_window(dst, 1) {
return Err(KernelError::InvalidAddress);
}
let (pa, _) = self
.frames
.find_user(from, src)
.ok_or(KernelError::InvalidAddress)?;
self.processes.get(to)?;
if !self.range_is_free(to, dst, 1) {
return Err(KernelError::AddressInUse);
}
let usage = FrameUse::user(to, dst, perms).ok_or(KernelError::InvalidAddress)?;
if !self.quota_mut(to)?.charge(1) {
return Err(KernelError::OutOfMemory);
}
if let Err(e) = self.platform_map(to, dst, pa, perms) {
self.quota_mut(to)?.uncharge(1);
return Err(e);
}
self.platform.unmap_page(from, src);
self.frames.retag(pa, usage);
self.quota_mut(from)?.uncharge(1);
Ok(())
}
/// Allocates a physically contiguous, zeroed region charged to `owner`.
pub fn create_pool_region(
&mut self,
owner: ProcessId,
pages: PageCount,
) -> Result<RegionId, KernelError> {
self.processes.get(owner)?;
let id = self.regions.next_free_id().ok_or(KernelError::NoFreeSlot)?;
let count = pages.value() as usize;
if !self.quota_mut(owner)?.charge(count) {
return Err(KernelError::OutOfMemory);
}
let Some(base) = self
.frames
.alloc_contiguous(count, |_| FrameUse::Region { region: id })
else {
self.quota_mut(owner)?.uncharge(count);
return Err(KernelError::OutOfMemory);
};
for i in 0..count {
self.platform.zero_frame(base.offset(i));
}
self.regions
.insert(Region::new(owner, RegionBacking::Pool, base, pages))
.ok_or(KernelError::NoFreeSlot)
}
/// Describes device memory at `base` so it can be mapped like a region.
pub fn create_device_region(
&mut self,
owner: ProcessId,
base: PhysicalAddress,
pages: PageCount,
) -> Result<RegionId, KernelError> {
self.processes.get(owner)?;
if !base.raw().is_multiple_of(PAGE_SIZE) {
return Err(KernelError::InvalidAddress);
}
// Device memory must never alias frames the kernel hands out.
let len = pages.value() as usize * PAGE_SIZE;
let start = base.raw();
let end = start.checked_add(len).ok_or(KernelError::InvalidAddress)?;
let pool_start = self.frames.base().raw();
let pool_end = pool_start + self.frames.capacity() * PAGE_SIZE;
if start < pool_end && end > pool_start {
return Err(KernelError::InvalidAddress);
}
// Nor another device's window: each device has one region, so
// revoking or delegating it means the same thing everywhere.
let overlaps = self.regions.iter().any(|(_, r)| {
let r_start = r.base().raw();
let r_end = r_start.saturating_add(r.pages().value() as usize * PAGE_SIZE);
r.backing() == RegionBacking::Device && start < r_end && end > r_start
});
if overlaps {
return Err(KernelError::AddressInUse);
}
self.regions
.insert(Region::new(owner, RegionBacking::Device, base, pages))
.ok_or(KernelError::NoFreeSlot)
}
pub fn region_base(&self, id: RegionId) -> Option<PhysicalAddress> {
self.regions.get(id).map(Region::base)
}
pub fn region_pages(&self, id: RegionId) -> Option<PageCount> {
self.regions.get(id).map(Region::pages)
}
/// Maps all of region `id` at `va` in `pid`, all or nothing.
/// Maps region `id` for boot code; the mapping is not tied to any
/// capability and lasts until the region or the process dies.
pub fn map_region(
&mut self,
id: RegionId,
pid: ProcessId,
va: VirtualAddress,
perms: PagePerms,
) -> Result<(), KernelError> {
self.map_region_via(id, pid, va, perms, None)
}
/// Maps all of region `id` at `va` in `pid`, all or nothing. A mapping
/// made through a capability (`via`) is removed when that capability
/// is deleted.
pub(crate) fn map_region_via(
&mut self,
id: RegionId,
pid: ProcessId,
va: VirtualAddress,
perms: PagePerms,
via: Option<domain::capability::CapRef>,
) -> Result<(), KernelError> {
self.processes.get(pid)?;
let region = self.regions.get(id).ok_or(KernelError::InvalidRegion)?;
let (base, pages) = (region.base(), region.pages().value() as usize);
if region.mappings().any(|m| m.process == pid) {
return Err(KernelError::AddressInUse);
}
// W^X holds across processes too: nobody may write what someone
// else can execute.
let clashes = |other: PagePerms| {
matches!(
(perms, other),
(PagePerms::ReadWrite, PagePerms::ReadExecute)
| (PagePerms::ReadExecute, PagePerms::ReadWrite)
)
};
if region.mappings().any(|m| clashes(m.perms)) {
return Err(KernelError::WritableAndExecutable);
}
if !Self::in_user_window(va, pages) {
return Err(KernelError::InvalidAddress);
}
if !self.range_is_free(pid, va, pages) {
return Err(KernelError::AddressInUse);
}
for i in 0..pages {
if let Err(e) = self.platform_map(pid, page_at(va, i), base.offset(i), perms) {
for j in 0..i {
self.platform.unmap_page(pid, page_at(va, j));
}
return Err(e);
}
}
let recorded = self.regions.get_mut(id).is_some_and(|r| {
r.add_mapping(RegionMapping {
process: pid,
va,
perms,
via,
})
});
if !recorded {
for i in 0..pages {
self.platform.unmap_page(pid, page_at(va, i));
}
return Err(KernelError::NoFreeSlot);
}
Ok(())
}
/// Removes `pid`'s mapping of region `id`, if it has one.
pub(crate) fn unmap_region_from(&mut self, id: RegionId, pid: ProcessId) {
let Some(region) = self.regions.get_mut(id) else {
return;
};
let pages = region.pages().value() as usize;
if let Some(m) = region.remove_mapping(pid) {
for i in 0..pages {
self.platform.unmap_page(pid, page_at(m.va, i));
}
}
}
/// Frees regions owned by `pid` (unmapping them from every other
/// process) and forgets `pid`'s mappings of other regions.
fn teardown_regions(&mut self, pid: ProcessId) {
let mut owned = [None; domain::memory::MAX_REGIONS];
for (slot, id) in owned.iter_mut().zip(self.regions.ids_owned_by(pid)) {
*slot = Some(id);
}
for id in owned.into_iter().flatten() {
let Some(region) = self.regions.remove(id) else {
continue;
};
let pages = region.pages().value() as usize;
for m in region.mappings().filter(|m| m.process != pid) {
for i in 0..pages {
self.platform.unmap_page(m.process, page_at(m.va, i));
}
}
if region.backing() == RegionBacking::Pool {
for i in 0..pages {
self.frames.free(region.base().offset(i));
}
}
}
for (_, region) in self.regions.iter_mut() {
region.remove_mapping(pid);
}
}
/// Releases everything `pid` holds in memory: its address space, page
/// tables, user pages and owned regions.
pub(crate) fn teardown_memory(&mut self, pid: ProcessId) {
self.teardown_regions(pid);
self.platform.destroy_address_space(pid);
self.frames.free_owned_by(pid);
if let Some(q) = self.quotas.get_mut(pid.raw() as usize) {
*q = Quota::default();
}
}
}
fn page_at(base: VirtualAddress, i: usize) -> VirtualAddress {
VirtualAddress::from_raw(base.raw() + i * PAGE_SIZE)
}

200
crates/kernel/src/memory/mod.rs Normal file
View File

@@ -0,0 +1,200 @@
//! Address-space bookkeeping. The kernel decides every user mapping from
//! its own records (`FrameTable`, `RegionTable`, quotas) and only then tells
//! the platform to apply it; it never asks the hardware what is mapped.
mod pages;
mod region;
use domain::{
error::KernelError,
memory::{
frame::{FrameTable, FrameUse, PAGE_SIZE},
quota::Quota,
},
};
use platform::{FrameSource, MapError, Platform};
use types::{
address::{PhysicalAddress, VirtualAddress},
id::ProcessId,
mapping::PagePerms,
};
use crate::Kernel;
/// First VA the kernel considers when picking a spot for `RequestPages`.
/// Below it sit the program image and boot-time device windows.
pub(crate) const USER_HEAP_START: usize = 0x40000;
/// Page-table frames for one address space, charged to its owner.
struct ChargedFrames<'a> {
frames: &'a mut FrameTable,
quota: &'a mut Quota,
owner: ProcessId,
}
impl FrameSource for ChargedFrames<'_> {
fn alloc_table_frame(&mut self) -> Option<PhysicalAddress> {
if !self.quota.charge(1) {
return None;
}
match self.frames.alloc(FrameUse::PageTable { owner: self.owner }) {
Some(pa) => Some(pa),
None => {
self.quota.uncharge(1);
None
}
}
}
}
fn map_error(e: MapError) -> KernelError {
match e {
MapError::OutOfFrames => KernelError::OutOfMemory,
MapError::InvalidAddress => KernelError::InvalidAddress,
MapError::NoAddressSpace => KernelError::OutOfMemory,
}
}
impl<P: Platform> Kernel<P> {
/// Whether `[va, va + pages)` is page aligned and inside the user window.
pub(crate) fn in_user_window(va: VirtualAddress, pages: usize) -> bool {
let start = va.raw();
let Some(len) = pages.checked_mul(PAGE_SIZE) else {
return false;
};
let Some(end) = start.checked_add(len) else {
return false;
};
pages > 0
&& start.is_multiple_of(PAGE_SIZE)
&& start >= P::USER_VA_START
&& end <= P::USER_VA_END
}
/// `pid`'s user page at `va`: the platform's page table says where it
/// points, the frame table confirms the frame really is that page.
fn user_frame(
&self,
pid: ProcessId,
va: VirtualAddress,
) -> Option<(PhysicalAddress, PagePerms)> {
let pa = self.platform.translate(pid, va)?;
self.frames.user_at(pa, pid, va).map(|perms| (pa, perms))
}
fn is_mapped(&self, pid: ProcessId, va: VirtualAddress) -> bool {
self.user_frame(pid, va).is_some() || self.regions.covering(pid, va).is_some()
}
fn range_is_free(&self, pid: ProcessId, va: VirtualAddress, pages: usize) -> bool {
(0..pages).all(|i| !self.is_mapped(pid, page_at(va, i)))
}
fn quota_mut(&mut self, pid: ProcessId) -> Result<&mut Quota, KernelError> {
self.budget_mut(pid).map(|b| &mut b.pages)
}
pub fn page_usage(&self, pid: ProcessId) -> usize {
self.budgets
.get(pid.raw() as usize)
.map_or(0, |b| b.pages.used())
}
pub fn free_page_count(&self) -> usize {
self.frames.free_count()
}
/// Maps `pa` at `va` in `pid`, building page tables on `pid`'s quota.
fn platform_map(
&mut self,
pid: ProcessId,
va: VirtualAddress,
pa: PhysicalAddress,
perms: PagePerms,
) -> Result<(), KernelError> {
let quota = &mut self
.budgets
.get_mut(pid.raw() as usize)
.ok_or(KernelError::InvalidProcess(pid))?
.pages;
let mut source = ChargedFrames {
frames: &mut self.frames,
quota,
owner: pid,
};
self.platform
.map_page(pid, va, pa, perms, &mut source)
.map_err(map_error)
}
pub(crate) fn create_address_space(&mut self, pid: ProcessId) -> Result<(), KernelError> {
let quota = &mut self
.budgets
.get_mut(pid.raw() as usize)
.ok_or(KernelError::InvalidProcess(pid))?
.pages;
let mut source = ChargedFrames {
frames: &mut self.frames,
quota,
owner: pid,
};
let created = self
.platform
.create_address_space(pid, &mut source)
.map_err(map_error);
if created.is_err() {
self.frames.free_owned_by(pid);
}
created
}
/// Allocates, zeroes and maps one fresh page for `pid` at `va`.
fn map_fresh_page(
&mut self,
pid: ProcessId,
va: VirtualAddress,
perms: PagePerms,
) -> Result<PhysicalAddress, KernelError> {
let usage = FrameUse::user(pid, va, perms).ok_or(KernelError::InvalidAddress)?;
if !self.quota_mut(pid)?.charge(1) {
return Err(KernelError::OutOfMemory);
}
let Some(pa) = self.frames.alloc(usage) else {
self.quota_mut(pid)?.uncharge(1);
return Err(KernelError::OutOfMemory);
};
self.platform.zero_frame(pa);
if let Err(e) = self.platform_map(pid, va, pa, perms) {
self.frames.free(pa);
self.quota_mut(pid)?.uncharge(1);
return Err(e);
}
Ok(pa)
}
/// Removes one user page of `pid` and returns its frame to the pool.
fn unmap_and_free(&mut self, pid: ProcessId, va: VirtualAddress) {
if let Some((pa, _)) = self.user_frame(pid, va) {
self.platform.unmap_page(pid, va);
self.frames.free(pa);
if let Some(b) = self.budgets.get_mut(pid.raw() as usize) {
b.pages.uncharge(1);
}
}
}
/// Releases everything `pid` holds in memory: its address space, page
/// tables, user pages and owned regions.
pub(crate) fn teardown_memory(&mut self, pid: ProcessId) {
self.teardown_regions(pid);
self.platform.destroy_address_space(pid);
self.frames.free_owned_by(pid);
if let Some(c) = self.heap_cursors.get_mut(pid.raw() as usize) {
*c = 0;
}
}
}
fn page_at(base: VirtualAddress, i: usize) -> VirtualAddress {
VirtualAddress::from_raw(base.raw() + i * PAGE_SIZE)
}

138
crates/kernel/src/memory/pages.rs Normal file
View File

@@ -0,0 +1,138 @@
//! Page moves, release and heap placement.
use domain::{
error::KernelError,
memory::frame::{FrameUse, PAGE_SIZE},
};
use platform::Platform;
use types::{
address::{PhysicalAddress, VirtualAddress},
id::ProcessId,
mapping::PagePerms,
};
use super::{USER_HEAP_START, page_at};
use crate::Kernel;
impl<P: Platform> Kernel<P> {
/// Frame backing the user page at `va` in `pid`. Lets boot code fill
/// pages it has just mapped for a process.
pub fn user_page_frame(&self, pid: ProcessId, va: VirtualAddress) -> Option<PhysicalAddress> {
self.user_frame(pid, va).map(|(pa, _)| pa)
}
/// Maps `pages` fresh zeroed pages at `va`, all or nothing.
pub fn map_fresh_range(
&mut self,
pid: ProcessId,
va: VirtualAddress,
pages: usize,
perms: PagePerms,
) -> Result<(), KernelError> {
if !Self::in_user_window(va, pages) {
return Err(KernelError::InvalidAddress);
}
if !self.range_is_free(pid, va, pages) {
return Err(KernelError::AddressInUse);
}
for i in 0..pages {
if let Err(e) = self.map_fresh_page(pid, page_at(va, i), perms) {
for j in 0..i {
self.unmap_and_free(pid, page_at(va, j));
}
return Err(e);
}
}
Ok(())
}
/// Next-fit search for `pages` free heap pages from `pid`'s last request, then from the bottom.
pub(crate) fn find_free_heap_range(
&self,
pid: ProcessId,
pages: usize,
) -> Option<VirtualAddress> {
let bottom = USER_HEAP_START.max(P::USER_VA_START);
let cursor = self
.heap_cursors
.get(pid.raw() as usize)
.copied()
.unwrap_or(0)
.max(bottom);
self.first_fit_from(pid, cursor, pages)
.or_else(|| self.first_fit_from(pid, bottom, pages))
}
/// Records where `pid`'s next heap search starts.
pub(crate) fn advance_heap_cursor(&mut self, pid: ProcessId, past: VirtualAddress) {
if let Some(c) = self.heap_cursors.get_mut(pid.raw() as usize) {
*c = past.raw();
}
}
fn first_fit_from(&self, pid: ProcessId, from: usize, pages: usize) -> Option<VirtualAddress> {
let mut candidate = from;
while Self::in_user_window(VirtualAddress::from_raw(candidate), pages) {
let va = VirtualAddress::from_raw(candidate);
match (0..pages).find(|&i| self.is_mapped(pid, page_at(va, i))) {
None => return Some(va),
Some(busy) => candidate += (busy + 1) * PAGE_SIZE,
}
}
None
}
/// Validates that every page of the range is a user page `pid` owns,
/// then releases them all.
pub(crate) fn release_range(
&mut self,
pid: ProcessId,
va: VirtualAddress,
pages: usize,
) -> Result<(), KernelError> {
if !Self::in_user_window(va, pages) {
return Err(KernelError::InvalidAddress);
}
if !(0..pages).all(|i| self.user_frame(pid, page_at(va, i)).is_some()) {
return Err(KernelError::InvalidAddress);
}
for i in 0..pages {
self.unmap_and_free(pid, page_at(va, i));
}
Ok(())
}
/// Moves the page at `src` in `from` to `dst` in `to`. The frame and its
/// quota charge change hands; `from` loses access.
pub(crate) fn move_page(
&mut self,
from: ProcessId,
src: VirtualAddress,
to: ProcessId,
dst: VirtualAddress,
perms: PagePerms,
) -> Result<(), KernelError> {
if !Self::in_user_window(src, 1) || !Self::in_user_window(dst, 1) {
return Err(KernelError::InvalidAddress);
}
let (pa, _) = self
.user_frame(from, src)
.ok_or(KernelError::InvalidAddress)?;
self.processes.get(to)?;
if !self.range_is_free(to, dst, 1) {
return Err(KernelError::AddressInUse);
}
let usage = FrameUse::user(to, dst, perms).ok_or(KernelError::InvalidAddress)?;
if !self.quota_mut(to)?.charge(1) {
return Err(KernelError::OutOfMemory);
}
if let Err(e) = self.platform_map(to, dst, pa, perms) {
self.quota_mut(to)?.uncharge(1);
return Err(e);
}
self.platform.unmap_page(from, src);
self.frames.retag(pa, usage);
self.quota_mut(from)?.uncharge(1);
Ok(())
}
}

206
crates/kernel/src/memory/region.rs Normal file
View File

@@ -0,0 +1,206 @@
//! Shared and device regions and their mappings.
use domain::{
error::KernelError,
memory::{
Region, RegionBacking, RegionMapping,
frame::{FrameUse, PAGE_SIZE},
},
};
use platform::Platform;
use types::{
address::{PhysicalAddress, VirtualAddress},
bounded::PageCount,
id::{ProcessId, RegionId},
mapping::PagePerms,
};
use super::page_at;
use crate::Kernel;
impl<P: Platform> Kernel<P> {
/// Allocates a physically contiguous, zeroed region charged to `owner`.
pub fn create_pool_region(
&mut self,
owner: ProcessId,
pages: PageCount,
) -> Result<RegionId, KernelError> {
self.processes.get(owner)?;
let id = self.regions.next_free_id().ok_or(KernelError::NoFreeSlot)?;
let count = pages.value() as usize;
self.charge_object(owner, crate::budget::REGIONS)?;
if !self.quota_mut(owner)?.charge(count) {
self.uncharge_object(owner, crate::budget::REGIONS);
return Err(KernelError::OutOfMemory);
}
let Some(base) = self
.frames
.alloc_contiguous(count, |_| FrameUse::Region { region: id })
else {
self.quota_mut(owner)?.uncharge(count);
self.uncharge_object(owner, crate::budget::REGIONS);
return Err(KernelError::OutOfMemory);
};
for i in 0..count {
self.platform.zero_frame(base.offset(i));
}
self.regions
.insert(Region::new(owner, RegionBacking::Pool, base, pages))
.ok_or(KernelError::NoFreeSlot)
}
/// Describes device memory at `base` so it can be mapped like a region.
pub fn create_device_region(
&mut self,
owner: ProcessId,
base: PhysicalAddress,
pages: PageCount,
) -> Result<RegionId, KernelError> {
self.processes.get(owner)?;
if !base.raw().is_multiple_of(PAGE_SIZE) {
return Err(KernelError::InvalidAddress);
}
// Device memory must never alias frames the kernel hands out.
let len = pages.value() as usize * PAGE_SIZE;
let start = base.raw();
let end = start.checked_add(len).ok_or(KernelError::InvalidAddress)?;
let pool_start = self.frames.base().raw();
let pool_end = pool_start + self.frames.capacity() * PAGE_SIZE;
if start < pool_end && end > pool_start {
return Err(KernelError::InvalidAddress);
}
// Nor another device's window: each device has one region, so
// revoking or delegating it means the same thing everywhere.
let overlaps = self.regions.iter().any(|(_, r)| {
let r_start = r.base().raw();
let r_end = r_start.saturating_add(r.pages().value() as usize * PAGE_SIZE);
r.backing() == RegionBacking::Device && start < r_end && end > r_start
});
if overlaps {
return Err(KernelError::AddressInUse);
}
self.charge_object(owner, crate::budget::REGIONS)?;
self.regions
.insert(Region::new(owner, RegionBacking::Device, base, pages))
.ok_or(KernelError::NoFreeSlot)
.inspect_err(|_| self.uncharge_object(owner, crate::budget::REGIONS))
}
pub fn region_base(&self, id: RegionId) -> Option<PhysicalAddress> {
self.regions.get(id).map(Region::base)
}
pub fn region_pages(&self, id: RegionId) -> Option<PageCount> {
self.regions.get(id).map(Region::pages)
}
/// Maps region `id` for boot code, all or nothing; not tied to any capability.
pub fn map_region(
&mut self,
id: RegionId,
pid: ProcessId,
va: VirtualAddress,
perms: PagePerms,
) -> Result<(), KernelError> {
self.map_region_via(id, pid, va, perms, None)
}
/// Maps region `id`, all or nothing; a mapping made `via` a capability dies with it.
pub(crate) fn map_region_via(
&mut self,
id: RegionId,
pid: ProcessId,
va: VirtualAddress,
perms: PagePerms,
via: Option<domain::capability::CapRef>,
) -> Result<(), KernelError> {
self.processes.get(pid)?;
let region = self.regions.get(id).ok_or(KernelError::InvalidRegion)?;
let (base, pages) = (region.base(), region.pages().value() as usize);
if region.mappings().any(|m| m.process == pid) {
return Err(KernelError::AddressInUse);
}
// W^X holds across processes too: nobody may write what someone
// else can execute.
let clashes = |other: PagePerms| {
matches!(
(perms, other),
(PagePerms::ReadWrite, PagePerms::ReadExecute)
| (PagePerms::ReadExecute, PagePerms::ReadWrite)
)
};
if region.mappings().any(|m| clashes(m.perms)) {
return Err(KernelError::WritableAndExecutable);
}
if !Self::in_user_window(va, pages) {
return Err(KernelError::InvalidAddress);
}
if !self.range_is_free(pid, va, pages) {
return Err(KernelError::AddressInUse);
}
for i in 0..pages {
if let Err(e) = self.platform_map(pid, page_at(va, i), base.offset(i), perms) {
for j in 0..i {
self.platform.unmap_page(pid, page_at(va, j));
}
return Err(e);
}
}
let recorded = self.regions.get_mut(id).is_some_and(|r| {
r.add_mapping(RegionMapping {
process: pid,
va,
perms,
via,
})
});
if !recorded {
for i in 0..pages {
self.platform.unmap_page(pid, page_at(va, i));
}
return Err(KernelError::NoFreeSlot);
}
Ok(())
}
/// Removes `pid`'s mapping of region `id`, if it has one.
pub(crate) fn unmap_region_from(&mut self, id: RegionId, pid: ProcessId) {
let Some(region) = self.regions.get_mut(id) else {
return;
};
let pages = region.pages().value() as usize;
if let Some(m) = region.remove_mapping(pid) {
for i in 0..pages {
self.platform.unmap_page(pid, page_at(m.va, i));
}
}
}
/// Frees regions owned by `pid` (unmapping them from every other
/// process) and forgets `pid`'s mappings of other regions.
pub(super) fn teardown_regions(&mut self, pid: ProcessId) {
let mut owned = [None; domain::memory::MAX_REGIONS];
for (slot, id) in owned.iter_mut().zip(self.regions.ids_owned_by(pid)) {
*slot = Some(id);
}
for id in owned.into_iter().flatten() {
let Some(region) = self.regions.remove(id) else {
continue;
};
let pages = region.pages().value() as usize;
for m in region.mappings().filter(|m| m.process != pid) {
for i in 0..pages {
self.platform.unmap_page(m.process, page_at(m.va, i));
}
}
if region.backing() == RegionBacking::Pool {
for i in 0..pages {
self.frames.free(region.base().offset(i));
}
}
}
for (_, region) in self.regions.iter_mut() {
region.remove_mapping(pid);
}
}
}

23
crates/kernel/src/proofs/authority.rs
View File

@@ -1,15 +1,6 @@
//! Mint creates no authority. The real kernel, with tables shrunk under
//! `cfg(kani)`, runs `mint` with arbitrary arguments in a small state
//! whose capabilities carry arbitrary rights. Afterwards every capability
//! to an object that already existed carries no more rights than some
//! capability to that object carried before.
//!
//! Only this harness is kept. The same check over delegate, delete,
//! revoke, create and kill either exceeded 12 GB in CBMC or tripped
//! modelling artifacts ("unreachable code" on safe `Option` matches that
//! an exhaustive host replay never reaches). Those operations are covered
//! by the syscall storm's global invariants (tests/storm.rs) and the
//! capability-tree model test (tests/captree_model.rs) instead.
//! Mint creates no authority: after `mint` with arbitrary arguments, no capability to a
//! pre-existing object carries rights beyond some earlier one. Other operations exceeded
//! 12 GB in CBMC; tests/storm.rs and tests/captree_model.rs cover them.
use super::platform::KaniPlatform;
use crate::Kernel;
@@ -79,7 +70,13 @@ fn any_slot() -> CapSlot {
fn small_world() -> Kernel<KaniPlatform> {
let mut k = Kernel::new(KaniPlatform, PhysicalAddress::from_raw(0x8400_0000), 6);
let (init, _) = k.boot(Priority::DEFAULT, TimeSlice::DEFAULT).unwrap();
let b = k.create_process_internal().unwrap();
let small = domain::budget::Limits {
pages: 2,
endpoints: 1,
notifications: 1,
regions: 1,
};
let b = k.create_process_internal(init, small).unwrap();
k.spawn_thread_internal(b).unwrap();
let ep = k.create_endpoint(init).unwrap();
let n = k.create_notification(b).unwrap();

3
crates/kernel/src/proofs/platform.rs
View File

@@ -35,6 +35,9 @@ impl Platform for KaniPlatform {
Ok(())
}
fn unmap_page(&mut self, _process: ProcessId, _va: VirtualAddress) {}
fn translate(&self, _process: ProcessId, _va: VirtualAddress) -> Option<PhysicalAddress> {
None
}
fn zero_frame(&mut self, _pa: PhysicalAddress) {}
fn enable_user_mode(&mut self, _entry: EntryPoint, _stack: StackPointer) -> ! {
kani::assume(false);

43
crates/kernel/src/smp.rs
View File

@@ -1,9 +1,5 @@
//! Cores. Every thread has a home core, chosen when it is created, and
//! only ever runs there: it waits in that core's run queue and is never
//! migrated. Making a thread ready from another core sends that core an
//! IPI when it has to look at its queue again (it is idle, or the thread
//! outranks what it is running), and so does taking away the thread a
//! core is running.
//! Cores: each thread has a home core and never migrates; waking or removing a thread on
//! another core sends that core an IPI when it must re-check its queue.
use domain::{
scheduler::{Scheduler, SchedulerDecision},
@@ -35,17 +31,17 @@ impl<P: Platform> Kernel<P> {
/// Homes a new thread on the core with the fewest live threads.
pub(crate) fn place(&mut self, tid: ThreadId) {
let cores = self.scheduler.num_cores().max(1);
let load = |core: CoreId| {
self.threads
.iter()
.filter(|(t, _)| *t != tid && self.home(*t) == core)
.count()
};
let cores = usize::from(self.scheduler.num_cores().max(1));
let mut load = [0usize; u8::MAX as usize + 1];
for (t, _) in self.threads.iter().filter(|(t, _)| *t != tid) {
if let Some(n) = load.get_mut(usize::from(self.home(t).raw())) {
*n += 1;
}
}
let best = (0..cores)
.map(CoreId::from_raw)
.min_by_key(|c| load(*c))
.unwrap_or(CoreId::from_raw(0));
.min_by_key(|c| load.get(*c).copied().unwrap_or(usize::MAX))
.and_then(|c| u8::try_from(c).ok())
.map_or(CoreId::from_raw(0), CoreId::from_raw);
self.set_home(tid, best);
}
@@ -79,10 +75,17 @@ impl<P: Platform> Kernel<P> {
}
}
/// What `core` should do after an IPI: keep running its thread, or
/// switch to a higher-priority one now waiting. When it has no thread
/// (it was idle, or its thread was destroyed) the caller picks the
/// next one itself.
/// The highest priority waiting in `core`'s run queue.
pub fn highest_waiting(&self, core: CoreId) -> Option<Priority> {
self.scheduler.highest_waiting(core)
}
pub fn thread_priority(&self, tid: ThreadId) -> Option<Priority> {
self.threads.get(tid).ok().map(|t| t.priority())
}
/// What `core` does after an IPI: keep its thread or switch to a higher-priority one;
/// with no thread the caller picks.
pub fn reschedule(&mut self, core: CoreId) -> SchedulerDecision {
let Some(current) = self.scheduler.current(core) else {
return SchedulerDecision::Continue;

9
crates/kernel/src/syscall/capability.rs
View File

@@ -43,12 +43,11 @@ impl<P: Platform> Kernel<P> {
};
let parent = self.cap_ref(caller_pid, slot);
let inserted = self.insert_derived(target_pid, copy, parent);
if let Err(e) = inserted {
return SyscallOutcome::Complete(SyscallResult::Error(e));
// The slot it landed in, so the delegator can tell the target.
match self.insert_derived(target_pid, copy, parent) {
Ok(slot) => SyscallOutcome::Complete(SyscallResult::Slot(slot)),
Err(e) => SyscallOutcome::Complete(SyscallResult::Error(e)),
}
SyscallOutcome::Complete(SyscallResult::Ok)
}
pub(crate) fn handle_revoke(

296
crates/kernel/src/syscall/ipc.rs → crates/kernel/src/syscall/ipc/endpoint.rs
View File

@@ -1,121 +1,22 @@
//! Endpoint IPC, notifications and interrupt delivery. docs/ipc.md is the
//! specification; the comments below only point at the non-obvious rules.
//! Endpoint send/receive, call/reply, capability transfer and minting.
use domain::{
capability::Capability,
error::KernelError,
message::Message,
notification::SignalEffect,
resource::Resource,
scheduler::Scheduler,
thread::{BlockReason, ThreadState},
capability::Capability, error::KernelError, message::Message, resource::Resource,
thread::BlockReason,
};
use platform::Platform;
use types::{
badge::Badge,
bounded::CapSlot,
generation::Generation,
id::{EndpointId, IrqId, NotificationId, ProcessId, ThreadId},
id::{EndpointId, ProcessId, ThreadId},
permission::{Permission, PermissionSet},
};
use super::{Delivery, DeliveryKind, SyscallOutcome, SyscallResult};
use super::super::{Delivery, DeliveryKind, SyscallOutcome, SyscallResult};
use super::Outgoing;
use crate::Kernel;
/// A thread named by id and generation, so a stale reference can be told
/// apart from a new thread that reused the id.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ThreadRef {
pub tid: ThreadId,
pub generation: Generation,
}
/// A message a blocked sender is waiting to hand over.
#[derive(Debug, Clone, Copy)]
pub(crate) struct Outgoing {
message: Message,
badge: Badge,
/// The sender's endpoint cap had `Delegate`, so a capability may travel.
grant: bool,
}
/// Per-thread IPC bookkeeping.
#[derive(Debug, Clone, Copy, Default)]
pub struct IpcState {
pub(crate) outgoing: Option<Outgoing>,
/// The caller this thread owes a reply to.
pub(crate) reply_to: Option<ThreadRef>,
pub(crate) bound: Option<NotificationId>,
}
fn rw_delegate() -> PermissionSet {
PermissionSet::empty()
.with(Permission::Read)
.with(Permission::Write)
.with(Permission::Delegate)
}
impl<P: Platform> Kernel<P> {
pub(crate) fn ipc(&self, tid: ThreadId) -> Option<&IpcState> {
self.ipc.get(tid.raw() as usize)
}
pub(crate) fn ipc_mut(&mut self, tid: ThreadId) -> Option<&mut IpcState> {
self.ipc.get_mut(tid.raw() as usize)
}
pub(crate) fn set_pending_result(&mut self, tid: ThreadId, result: Option<SyscallResult>) {
if let Some(r) = self.pending_results.get_mut(tid.raw() as usize) {
*r = result;
}
}
fn thread_ref(&self, tid: ThreadId) -> Option<ThreadRef> {
Some(ThreadRef {
tid,
generation: self.threads.generation(tid)?,
})
}
fn is_live(&self, r: ThreadRef) -> bool {
self.threads.generation(r.tid) == Some(r.generation)
}
fn is_blocked_as(&self, tid: ThreadId, f: impl Fn(&BlockReason) -> bool) -> bool {
matches!(self.threads.get(tid).map(|t| *t.state()), Ok(ThreadState::Blocked(r)) if f(&r))
}
/// Blocks the running caller.
pub(crate) fn block_caller(&mut self, tid: ThreadId, reason: BlockReason) -> SyscallOutcome {
if let Ok(t) = self.threads.get_mut(tid) {
let _ = t.block(reason);
}
self.scheduler.remove(tid);
SyscallOutcome::BlockCaller(reason)
}
/// Makes a blocked thread runnable with `result` as its syscall result.
pub(crate) fn wake(&mut self, tid: ThreadId, result: SyscallResult) {
let Ok(thread) = self.threads.get_mut(tid) else {
return;
};
let priority = thread.priority();
if thread.unblock().is_ok() {
self.set_pending_result(tid, Some(result));
self.make_ready(tid, priority);
}
}
/// Releases a caller waiting on `reply` with `NoReply`, if it is still
/// that caller and still waiting.
pub(crate) fn release_caller(&mut self, reply: ThreadRef) {
if self.is_live(reply)
&& self.is_blocked_as(reply.tid, |r| *r == BlockReason::AwaitingReply)
{
self.wake(reply.tid, SyscallResult::Error(KernelError::NoReply));
}
}
fn endpoint_cap(
&self,
pid: ProcessId,
@@ -133,22 +34,8 @@ impl<P: Platform> Kernel<P> {
}
}
fn notification_cap(
&self,
pid: ProcessId,
slot: CapSlot,
needed: Permission,
) -> Result<(NotificationId, Badge, Generation), KernelError> {
let cap = self.check_cap(pid, slot, needed)?;
match cap.resource() {
Resource::Notification(id) => Ok((*id, cap.badge(), *cap.generation())),
_ => Err(KernelError::WrongObject),
}
}
/// Copies the capability in `slot` of `from` into `to`'s table, if the
/// transfer is allowed and still valid. Returns where it landed.
fn transfer_cap(
pub(super) fn transfer_cap(
&mut self,
from: ProcessId,
slot: Option<CapSlot>,
@@ -159,17 +46,22 @@ impl<P: Platform> Kernel<P> {
if !grant {
return None;
}
let copy: Capability = self
.check_cap(from, slot, Permission::Delegate)
.ok()?
.clone();
// The receiver gets the capability but not the right to pass it
// on; a sender that wants that uses Delegate with explicit rights.
let source = self.check_cap(from, slot, Permission::Delegate).ok()?;
let copy = Capability::new(
*source.resource(),
source.permissions().without(Permission::Delegate),
*source.generation(),
)
.with_badge(source.badge());
let parent = self.cap_ref(from, slot);
self.insert_derived(to, copy, parent).ok()
}
/// Hands `out` from `sender` to `receiver`. For a call, the receiver's
/// reply slot now names the sender.
fn deliver(
pub(super) fn deliver(
&mut self,
sender: ThreadId,
out: Outgoing,
@@ -409,161 +301,9 @@ impl<P: Platform> Kernel<P> {
Ok(slot) => SyscallOutcome::Complete(SyscallResult::Slot(slot)),
Err(e) => {
self.endpoints.destroy(id);
self.uncharge_object(pid, crate::budget::ENDPOINTS);
SyscallOutcome::Complete(SyscallResult::Error(e))
}
}
}
// ---- notifications ----
pub fn create_notification(&mut self, owner: ProcessId) -> Result<NotificationId, KernelError> {
self.processes.get(owner)?;
self.notifications
.create(owner)
.ok_or(KernelError::NoFreeSlot)
}
pub(crate) fn create_notification_cap(
&mut self,
pid: ProcessId,
) -> Result<CapSlot, KernelError> {
let id = self.create_notification(pid)?;
self.grant_cap(pid, Resource::Notification(id), rw_delegate())
.inspect_err(|_| {
self.notifications.destroy(id);
})
}
/// Signals `id` with `badge`; shared by `Signal` and interrupts.
fn signal_notification(&mut self, id: NotificationId, badge: Badge) {
let bound_receiving = self
.notifications
.get(id)
.and_then(|n| n.bound())
.is_some_and(|t| self.is_blocked_as(t, |r| matches!(r, BlockReason::Receiving(_))));
let Some(ntfn) = self.notifications.get_mut(id) else {
return;
};
match ntfn.signal(badge, bound_receiving) {
SignalEffect::WakeWaiter(t, bits) => {
self.wake(t, SyscallResult::Received(Delivery::signal(bits)));
}
SignalEffect::WakeBound(t, bits) => {
self.endpoints.remove_thread_from_all(t);
self.wake(t, SyscallResult::Received(Delivery::signal(bits)));
}
SignalEffect::Stored => {}
}
}
pub(crate) fn signal(&mut self, pid: ProcessId, slot: CapSlot) -> Result<(), KernelError> {
let (id, badge, _) = self.notification_cap(pid, slot, Permission::Write)?;
self.signal_notification(id, badge);
Ok(())
}
pub(crate) fn wait(&mut self, tid: ThreadId, pid: ProcessId, slot: CapSlot) -> SyscallOutcome {
let id = match self.notification_cap(pid, slot, Permission::Read) {
Ok((id, _, _)) => id,
Err(e) => return SyscallOutcome::Complete(SyscallResult::Error(e)),
};
let Some(ntfn) = self.notifications.get_mut(id) else {
return SyscallOutcome::Complete(SyscallResult::Error(KernelError::NoSuchObject));
};
if ntfn.has_bits() {
let bits = ntfn.take_bits();
return SyscallOutcome::Complete(SyscallResult::Received(Delivery::signal(bits)));
}
if !ntfn.add_waiter(tid) {
return SyscallOutcome::Complete(SyscallResult::Error(KernelError::EndpointFull));
}
self.block_caller(tid, BlockReason::WaitingNotification(id))
}
pub(crate) fn poll(&mut self, pid: ProcessId, slot: CapSlot) -> Result<usize, KernelError> {
let (id, _, _) = self.notification_cap(pid, slot, Permission::Read)?;
self.notifications
.get_mut(id)
.map(|n| n.take_bits())
.ok_or(KernelError::NoSuchObject)
}
pub(crate) fn bind_notification(
&mut self,
tid: ThreadId,
pid: ProcessId,
slot: CapSlot,
) -> Result<(), KernelError> {
let (id, _, _) = self.notification_cap(pid, slot, Permission::Read)?;
if self.ipc(tid).is_none_or(|s| s.bound.is_some()) {
return Err(KernelError::InvalidArgument);
}
let ntfn = self
.notifications
.get_mut(id)
.ok_or(KernelError::NoSuchObject)?;
if !ntfn.bind(tid) {
return Err(KernelError::InvalidArgument);
}
if let Some(s) = self.ipc_mut(tid) {
s.bound = Some(id);
}
Ok(())
}
// ---- interrupts ----
fn interrupt_cap(&self, pid: ProcessId, slot: CapSlot) -> Result<IrqId, KernelError> {
match self.check_cap(pid, slot, Permission::Execute)?.resource() {
Resource::Interrupt(irq) => Ok(*irq),
_ => Err(KernelError::WrongObject),
}
}
pub(crate) fn set_irq_notification(
&mut self,
pid: ProcessId,
irq_slot: CapSlot,
ntfn_slot: CapSlot,
) -> Result<(), KernelError> {
let irq = self.interrupt_cap(pid, irq_slot)?;
let (notification, badge, generation) =
self.notification_cap(pid, ntfn_slot, Permission::Write)?;
let route = domain::irq::IrqRoute {
notification,
generation,
badge,
};
if self.irqs.set(irq, route) {
Ok(())
} else {
Err(KernelError::InvalidIrq(irq))
}
}
pub(crate) fn ack_interrupt(
&mut self,
pid: ProcessId,
slot: CapSlot,
) -> Result<(), KernelError> {
let irq = self.interrupt_cap(pid, slot)?;
self.platform.unmask_irq(irq);
Ok(())
}
/// An interrupt fired: mask the line and signal its notification. The
/// line stays masked until the handler acknowledges it.
pub fn handle_interrupt(&mut self, irq: IrqId) {
self.platform.mask_irq(irq);
let Some(route) = self.irqs.route(irq) else {
return;
};
// A route set through a capability that has since been revoked, or
// to a notification whose slot was reused, is dead.
if self.notifications.generation(route.notification) != Some(route.generation) {
self.irqs.clear(irq);
return;
}
self.signal_notification(route.notification, route.badge);
}
}

83
crates/kernel/src/syscall/ipc/irq.rs Normal file
View File

@@ -0,0 +1,83 @@
//! Interrupt routing to notifications.
use domain::{error::KernelError, resource::Resource};
use platform::Platform;
use types::{
bounded::CapSlot,
id::{IrqId, ProcessId},
permission::Permission,
};
use crate::Kernel;
impl<P: Platform> Kernel<P> {
fn interrupt_cap(&self, pid: ProcessId, slot: CapSlot) -> Result<IrqId, KernelError> {
match self.check_cap(pid, slot, Permission::Execute)?.resource() {
Resource::Interrupt(irq) => Ok(*irq),
_ => Err(KernelError::WrongObject),
}
}
pub(crate) fn set_irq_notification(
&mut self,
pid: ProcessId,
irq_slot: CapSlot,
ntfn_slot: CapSlot,
) -> Result<(), KernelError> {
let irq = self.interrupt_cap(pid, irq_slot)?;
let (notification, badge, generation) =
self.notification_cap(pid, ntfn_slot, Permission::Write)?;
if self.irq_owner(irq).is_some_and(|owner| owner != pid) {
return Err(KernelError::IrqAlreadyOwned(irq));
}
let route = domain::irq::IrqRoute {
notification,
generation,
badge,
owner: pid,
};
if self.irqs.set(irq, route) {
Ok(())
} else {
Err(KernelError::InvalidIrq(irq))
}
}
pub(crate) fn ack_interrupt(
&mut self,
pid: ProcessId,
slot: CapSlot,
) -> Result<(), KernelError> {
let irq = self.interrupt_cap(pid, slot)?;
if self.irq_owner(irq).is_some_and(|owner| owner != pid) {
return Err(KernelError::IrqAlreadyOwned(irq));
}
self.platform.unmask_irq(irq);
Ok(())
}
/// The process that routed `irq`, while its route still leads to a
/// live notification. One owner per line: other holders of the IRQ
/// capability cannot take it over.
fn irq_owner(&self, irq: IrqId) -> Option<ProcessId> {
let route = self.irqs.route(irq)?;
(self.notifications.generation(route.notification) == Some(route.generation))
.then_some(route.owner)
}
/// An interrupt fired: mask the line and signal its notification. The
/// line stays masked until the handler acknowledges it.
pub fn handle_interrupt(&mut self, irq: IrqId) {
self.platform.mask_irq(irq);
let Some(route) = self.irqs.route(irq) else {
return;
};
// A route set through a capability that has since been revoked, or
// to a notification whose slot was reused, is dead.
if self.notifications.generation(route.notification) != Some(route.generation) {
self.irqs.clear(irq);
return;
}
self.signal_notification(route.notification, route.badge);
}
}

109
crates/kernel/src/syscall/ipc/mod.rs Normal file
View File

@@ -0,0 +1,109 @@
//! Endpoint IPC, notifications and interrupt delivery (see docs/ipc.md).
mod endpoint;
mod irq;
mod notification;
use domain::{
error::KernelError,
message::Message,
scheduler::Scheduler,
thread::{BlockReason, ThreadState},
};
use platform::Platform;
use types::{
badge::Badge,
generation::Generation,
id::{NotificationId, ThreadId},
};
use super::{SyscallOutcome, SyscallResult};
use crate::Kernel;
/// A thread named by id and generation, so a stale reference can be told
/// apart from a new thread that reused the id.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ThreadRef {
pub tid: ThreadId,
pub generation: Generation,
}
/// A message a blocked sender is waiting to hand over.
#[derive(Debug, Clone, Copy)]
pub(crate) struct Outgoing {
message: Message,
badge: Badge,
/// The sender's endpoint cap had `Delegate`, so a capability may travel.
grant: bool,
}
/// Per-thread IPC bookkeeping.
#[derive(Debug, Clone, Copy, Default)]
pub struct IpcState {
pub(crate) outgoing: Option<Outgoing>,
/// The caller this thread owes a reply to.
pub(crate) reply_to: Option<ThreadRef>,
pub(crate) bound: Option<NotificationId>,
}
impl<P: Platform> Kernel<P> {
pub(crate) fn ipc(&self, tid: ThreadId) -> Option<&IpcState> {
self.ipc.get(tid.raw() as usize)
}
pub(crate) fn ipc_mut(&mut self, tid: ThreadId) -> Option<&mut IpcState> {
self.ipc.get_mut(tid.raw() as usize)
}
pub(crate) fn set_pending_result(&mut self, tid: ThreadId, result: Option<SyscallResult>) {
if let Some(r) = self.pending_results.get_mut(tid.raw() as usize) {
*r = result;
}
}
fn thread_ref(&self, tid: ThreadId) -> Option<ThreadRef> {
Some(ThreadRef {
tid,
generation: self.threads.generation(tid)?,
})
}
fn is_live(&self, r: ThreadRef) -> bool {
self.threads.generation(r.tid) == Some(r.generation)
}
fn is_blocked_as(&self, tid: ThreadId, f: impl Fn(&BlockReason) -> bool) -> bool {
matches!(self.threads.get(tid).map(|t| *t.state()), Ok(ThreadState::Blocked(r)) if f(&r))
}
/// Blocks the running caller.
pub(crate) fn block_caller(&mut self, tid: ThreadId, reason: BlockReason) -> SyscallOutcome {
if let Ok(t) = self.threads.get_mut(tid) {
let _ = t.block(reason);
}
self.scheduler.remove(tid);
SyscallOutcome::BlockCaller(reason)
}
/// Makes a blocked thread runnable with `result` as its syscall result.
pub(crate) fn wake(&mut self, tid: ThreadId, result: SyscallResult) {
let Ok(thread) = self.threads.get_mut(tid) else {
return;
};
let priority = thread.priority();
if thread.unblock().is_ok() {
self.set_pending_result(tid, Some(result));
self.make_ready(tid, priority);
}
}
/// Releases a caller waiting on `reply` with `NoReply`, if it is still
/// that caller and still waiting.
pub(crate) fn release_caller(&mut self, reply: ThreadRef) {
if self.is_live(reply)
&& self.is_blocked_as(reply.tid, |r| *r == BlockReason::AwaitingReply)
{
self.wake(reply.tid, SyscallResult::Error(KernelError::NoReply));
}
}
}

136
crates/kernel/src/syscall/ipc/notification.rs Normal file
View File

@@ -0,0 +1,136 @@
//! Notification objects: signal, wait, poll, bind.
use domain::{
error::KernelError, notification::SignalEffect, resource::Resource, thread::BlockReason,
};
use platform::Platform;
use types::{
badge::Badge,
bounded::CapSlot,
generation::Generation,
id::{NotificationId, ProcessId, ThreadId},
permission::{Permission, PermissionSet},
};
use super::super::{Delivery, SyscallOutcome, SyscallResult};
use crate::Kernel;
fn rw_delegate() -> PermissionSet {
PermissionSet::empty()
.with(Permission::Read)
.with(Permission::Write)
.with(Permission::Delegate)
}
impl<P: Platform> Kernel<P> {
pub(super) fn notification_cap(
&self,
pid: ProcessId,
slot: CapSlot,
needed: Permission,
) -> Result<(NotificationId, Badge, Generation), KernelError> {
let cap = self.check_cap(pid, slot, needed)?;
match cap.resource() {
Resource::Notification(id) => Ok((*id, cap.badge(), *cap.generation())),
_ => Err(KernelError::WrongObject),
}
}
pub fn create_notification(&mut self, owner: ProcessId) -> Result<NotificationId, KernelError> {
self.processes.get(owner)?;
self.charge_object(owner, crate::budget::NOTIFICATIONS)?;
self.notifications
.create(owner)
.ok_or(KernelError::NoFreeSlot)
.inspect_err(|_| self.uncharge_object(owner, crate::budget::NOTIFICATIONS))
}
pub(crate) fn create_notification_cap(
&mut self,
pid: ProcessId,
) -> Result<CapSlot, KernelError> {
let id = self.create_notification(pid)?;
self.grant_cap(pid, Resource::Notification(id), rw_delegate())
.inspect_err(|_| {
self.notifications.destroy(id);
self.uncharge_object(pid, crate::budget::NOTIFICATIONS);
})
}
/// Signals `id` with `badge`; shared by `Signal` and interrupts.
pub(super) fn signal_notification(&mut self, id: NotificationId, badge: Badge) {
let bound_receiving = self
.notifications
.get(id)
.and_then(|n| n.bound())
.is_some_and(|t| self.is_blocked_as(t, |r| matches!(r, BlockReason::Receiving(_))));
let Some(ntfn) = self.notifications.get_mut(id) else {
return;
};
match ntfn.signal(badge, bound_receiving) {
SignalEffect::WakeWaiter(t, bits) => {
self.wake(t, SyscallResult::Received(Delivery::signal(bits)));
}
SignalEffect::WakeBound(t, bits) => {
self.leave_queue(t);
self.wake(t, SyscallResult::Received(Delivery::signal(bits)));
}
SignalEffect::Stored => {}
}
}
pub(crate) fn signal(&mut self, pid: ProcessId, slot: CapSlot) -> Result<(), KernelError> {
let (id, badge, _) = self.notification_cap(pid, slot, Permission::Write)?;
self.signal_notification(id, badge);
Ok(())
}
pub(crate) fn wait(&mut self, tid: ThreadId, pid: ProcessId, slot: CapSlot) -> SyscallOutcome {
let id = match self.notification_cap(pid, slot, Permission::Read) {
Ok((id, _, _)) => id,
Err(e) => return SyscallOutcome::Complete(SyscallResult::Error(e)),
};
let Some(ntfn) = self.notifications.get_mut(id) else {
return SyscallOutcome::Complete(SyscallResult::Error(KernelError::NoSuchObject));
};
if ntfn.has_bits() {
let bits = ntfn.take_bits();
return SyscallOutcome::Complete(SyscallResult::Received(Delivery::signal(bits)));
}
if !ntfn.add_waiter(tid) {
return SyscallOutcome::Complete(SyscallResult::Error(KernelError::EndpointFull));
}
self.block_caller(tid, BlockReason::WaitingNotification(id))
}
pub(crate) fn poll(&mut self, pid: ProcessId, slot: CapSlot) -> Result<usize, KernelError> {
let (id, _, _) = self.notification_cap(pid, slot, Permission::Read)?;
self.notifications
.get_mut(id)
.map(|n| n.take_bits())
.ok_or(KernelError::NoSuchObject)
}
pub(crate) fn bind_notification(
&mut self,
tid: ThreadId,
pid: ProcessId,
slot: CapSlot,
) -> Result<(), KernelError> {
let (id, _, _) = self.notification_cap(pid, slot, Permission::Read)?;
if self.ipc(tid).is_none_or(|s| s.bound.is_some()) {
return Err(KernelError::InvalidArgument);
}
let ntfn = self
.notifications
.get_mut(id)
.ok_or(KernelError::NoSuchObject)?;
if !ntfn.bind(tid) {
return Err(KernelError::InvalidArgument);
}
if let Some(s) = self.ipc_mut(tid) {
s.bound = Some(id);
}
Ok(())
}
}

23
crates/kernel/src/syscall/lifecycle.rs
View File

@@ -9,12 +9,14 @@ impl<P: Platform> Kernel<P> {
pid: types::id::ProcessId,
entry: types::address::EntryPoint,
stack: types::address::StackPointer,
arg: usize,
) -> SyscallOutcome {
use domain::thread::SavedContext;
let priority = types::bounded::Priority::DEFAULT;
let time_slice = types::time::TimeSlice::DEFAULT;
let context = SavedContext::with_entry_and_stack(entry, stack);
let mut context = SavedContext::with_entry_and_stack(entry, stack);
context.registers_mut()[10] = arg;
let tid = match self
.threads
@@ -39,6 +41,7 @@ impl<P: Platform> Kernel<P> {
caller_pid: types::id::ProcessId,
create_cap: types::bounded::CapSlot,
_image: types::address::DataRef,
limits: domain::budget::Limits,
) -> SyscallOutcome {
if let Err(e) = self.check_cap(
caller_pid,
@@ -48,19 +51,11 @@ impl<P: Platform> Kernel<P> {
return SyscallOutcome::Complete(SyscallResult::Error(e));
}
let new_pid = match self
.processes
.create(domain::capability::CapabilityTable::new(), None)
{
let new_pid = match self.new_process(Some(caller_pid), limits) {
Ok(p) => p,
Err(e) => return SyscallOutcome::Complete(SyscallResult::Error(e)),
};
if let Err(e) = self.create_address_space(new_pid) {
let _ = self.processes.destroy(new_pid);
return SyscallOutcome::Complete(SyscallResult::Error(e));
}
match self.grant_cap(
caller_pid,
domain::resource::Resource::Process(new_pid),
@@ -73,9 +68,7 @@ impl<P: Platform> Kernel<P> {
) {
Ok(slot) => SyscallOutcome::Complete(SyscallResult::Slot(slot)),
Err(e) => {
// Rollback: destroy orphaned process
self.teardown_memory(new_pid);
let _ = self.processes.destroy(new_pid);
self.destroy_process(new_pid);
SyscallOutcome::Complete(SyscallResult::Error(e))
}
}
@@ -87,6 +80,7 @@ impl<P: Platform> Kernel<P> {
process_cap: types::bounded::CapSlot,
entry: types::address::EntryPoint,
stack: types::address::StackPointer,
arg: usize,
) -> SyscallOutcome {
use domain::thread::SavedContext;
@@ -111,7 +105,8 @@ impl<P: Platform> Kernel<P> {
let priority = types::bounded::Priority::DEFAULT;
let time_slice = types::time::TimeSlice::DEFAULT;
let context = SavedContext::with_entry_and_stack(entry, stack);
let mut context = SavedContext::with_entry_and_stack(entry, stack);
context.registers_mut()[10] = arg;
let tid = match self
.threads

Some files were not shown because too many files have changed in this diff Show More