GKaszewski/k-os
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(kernel): kernel-owned memory model

Browse Source 
- FrameTable: per-frame owner/use, replaces PagePool + ProcessMemory
- Quota per process, page tables charged to owner
- Regions for shared mem + device MMIO, torn down w/ owner
- Platform: create/destroy AS, map/unmap, zero_frame; drop translate_va
- RequestPages/ReleasePages/MapPage(move) checked vs kernel records
- user window 0x1000..32MiB, private pts by construction
- boot loader + device setup go through kernel api
- kernel built in place, boot allocator bounded
- isolation tests, randomized invariant test, FrameTable kani proofs
- fixes: inflated release, unmapped release, dst va unchecked,
  heap overflow, unbounded pt alloc, unzeroed frames, double free
This commit is contained in:
Gabriel Kaszewski
2026-10-26 19:40:50 +01:00
parent e931485ca7
commit 8d2c40cb1e
43 changed files with 2815 additions and 1172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
crates/adapter-riscv/src/boot/hardware.rs
View File

@@ -1,6 +1,6 @@
use crate::hw::uart::Uart;
use crate::hw::virtio_blk::VirtioBlk;
use crate::{KERNEL, board, kernel_cell::KernelCell, mmu, platform, trap};
use crate::{KERNEL, board, mmu, platform, trap};
unsafe extern "C" {
static _bss_start: u8;
@@ -22,12 +22,21 @@ pub fn init(uart: &Uart) -> &'static mut mmu::PageTable {
crate::hw::plic::Plic::init();
uart.write_str("k-os: traps and timer initialized\n");
if board::USER_POOL_PAGES > domain::memory::frame::MAX_FRAMES {
crate::halt(uart, "user pool larger than the frame table");
}
let riscv_platform = platform::RiscVPlatform::new(kernel_pt as *const mmu::PageTable as usize);
let pool = domain::memory::page_pool::PagePool::new(
types::address::PhysicalAddress::from_raw(board::USER_POOL_BASE),
board::USER_POOL_PAGES,
);
KERNEL.init(kernel::Kernel::new(riscv_platform, pool));
// SAFETY: `init_in_place` writes every field of the kernel.
unsafe {
KERNEL.init_with(|slot| {
kernel::Kernel::init_in_place(
slot,
riscv_platform,
types::address::PhysicalAddress::from_raw(board::USER_POOL_BASE),
board::USER_POOL_PAGES,
)
});
}
kernel_pt
}

191
crates/adapter-riscv/src/boot/loader.rs
View File

@@ -1,9 +1,14 @@
use crate::hw::uart::Uart;
use crate::hw::virtio_blk::VirtioBlk;
use crate::{boot_fs, mmu};
use types::address::VirtualAddress;
use types::id::{ProcessId, ThreadId};
use types::mapping::PagePerms;
const PAGE_SIZE: usize = mmu::PAGE_SIZE;
const USER_VA_BASE: usize = 0x10000;
/// The stack sits just below the image and stops short of the null page.
const USER_STACK_PAGES: usize = USER_VA_BASE / PAGE_SIZE - 1;
const MAX_FILE_SIZE: usize = 64 * 1024;
pub struct LoadedProcess {
@@ -15,11 +20,23 @@ fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
unsafe { &mut *crate::kernel_ptr() }
}
pub fn load_init(
uart: &Uart,
blk: &mut VirtioBlk,
kernel_pt: &mmu::PageTable,
) -> (LoadedProcess, ThreadId) {
/// Scratch space for reading ELF files, allocated once from the boot heap.
fn elf_buffer() -> &'static mut [u8] {
static mut BUF: usize = 0;
// SAFETY: boot is single threaded; the buffer is kernel-only memory
// from the boot allocator and is never handed to a process.
unsafe {
if BUF == 0 {
BUF = mmu::allocator::alloc_zeroed_page();
for _ in 1..MAX_FILE_SIZE / PAGE_SIZE {
mmu::allocator::alloc_zeroed_page();
}
}
core::slice::from_raw_parts_mut(BUF as *mut u8, MAX_FILE_SIZE)
}
}
pub fn load_init(uart: &Uart, blk: &mut VirtioBlk) -> (LoadedProcess, ThreadId) {
let (pid, tid) = kern()
.boot(
types::bounded::Priority::DEFAULT,
@@ -27,10 +44,7 @@ pub fn load_init(
)
.unwrap_or_else(|_| crate::halt(uart, "boot failed"));
let entry = load_elf(blk, kernel_pt, pid, b"init.elf");
if entry == 0 {
crate::halt(uart, "init.elf not found");
}
let entry = load_elf(uart, blk, pid, b"init.elf");
setup_thread(tid, entry);
uart.write_str("k-os: init loaded\n");
@@ -40,17 +54,13 @@ pub fn load_init(
pub fn load_server(
uart: &Uart,
blk: &mut VirtioBlk,
kernel_pt: &mmu::PageTable,
filename: &[u8],
label: &str,
) -> LoadedProcess {
let pid = kern()
.create_process_internal()
.unwrap_or_else(|| crate::halt(uart, "process create failed"));
let entry = load_elf(blk, kernel_pt, pid, filename);
if entry == 0 {
crate::halt(uart, "elf not found");
}
let entry = load_elf(uart, blk, pid, filename);
let tid = kern()
.spawn_thread_internal(pid)
.unwrap_or_else(|| crate::halt(uart, "thread spawn failed"));
@@ -98,100 +108,89 @@ fn setup_thread(tid: ThreadId, entry: usize) {
}
}
fn load_elf(
blk: &mut VirtioBlk,
kernel_pt: &mmu::PageTable,
pid: ProcessId,
filename: &[u8],
) -> usize {
let buf_base = mmu::allocator::alloc_zeroed_page();
let file_pages = MAX_FILE_SIZE / 4096;
for _ in 1..file_pages {
mmu::allocator::alloc_zeroed_page();
/// Copies `data` into `pid`'s already mapped pages starting at `va`.
fn copy_to_user(pid: ProcessId, va: usize, data: &[u8]) -> bool {
let mut done = 0;
while done < data.len() {
let addr = va + done;
let page = addr & !(PAGE_SIZE - 1);
let offset = addr - page;
let Some(pa) = kern().user_page_frame(pid, VirtualAddress::from_raw(page)) else {
return false;
};
let len = (PAGE_SIZE - offset).min(data.len() - done);
// SAFETY: `pa` is a pool frame the kernel just mapped for `pid`;
// pool frames are identity mapped kernel RAM.
unsafe {
core::ptr::copy_nonoverlapping(
data.as_ptr().add(done),
(pa.raw() as *mut u8).add(offset),
len,
);
}
done += len;
}
let buf = unsafe { core::slice::from_raw_parts_mut(buf_base as *mut u8, MAX_FILE_SIZE) };
true
}
fn segment_perms(flags: elf::SegmentFlags) -> Option<PagePerms> {
match (flags.write, flags.execute) {
(true, true) => None,
(false, true) => Some(PagePerms::ReadExecute),
(true, false) => Some(PagePerms::ReadWrite),
(false, false) => Some(PagePerms::ReadOnly),
}
}
/// Loads `filename` into `pid` through the kernel's own mapping API, so
/// every page is tracked, charged and checked like any other. Halts on any
/// malformed or overlapping image: boot servers are trusted to be sane.
fn load_elf(uart: &Uart, blk: &mut VirtioBlk, pid: ProcessId, filename: &[u8]) -> usize {
let buf = elf_buffer();
let file_size = boot_fs::read_file_from_disk(blk, filename, buf).unwrap_or(0);
if file_size == 0 {
return 0;
crate::halt(uart, "ELF not found");
}
let elf_info = match elf::parse(&buf[..file_size]) {
Ok(info) => info,
Err(_) => return 0,
};
let user_pt = mmu::create_user_page_table(
kernel_pt,
types::address::PhysicalAddress::from_raw(0),
0,
types::address::VirtualAddress::from_raw(USER_VA_BASE),
);
let file = &buf[..file_size];
let elf_info = elf::parse(file).unwrap_or_else(|_| crate::halt(uart, "bad ELF"));
for segment in elf_info.segments() {
let page_start = segment.vaddr & !0xFFF;
let offset_in_page = segment.vaddr & 0xFFF;
let total_size = offset_in_page + segment.mem_size;
let seg_pages = (total_size + 4095) / 4096;
let perms =
segment_perms(segment.flags).unwrap_or_else(|| crate::halt(uart, "W+X segment"));
let page_start = segment.vaddr & !(PAGE_SIZE - 1);
let offset_in_page = segment.vaddr - page_start;
let pages = offset_in_page
.checked_add(segment.mem_size)
.map(|total| total.div_ceil(PAGE_SIZE))
.unwrap_or_else(|| crate::halt(uart, "segment too large"));
if pages == 0 {
continue;
}
for p in 0..seg_pages {
let page = mmu::allocator::alloc_zeroed_page();
let page_va = page_start + p * 4096;
let seg_file_start = if p == 0 { 0 } else { p * 4096 - offset_in_page };
let dest_off = if p == 0 { offset_in_page } else { 0 };
let copy_len = if seg_file_start < segment.file_size {
(segment.file_size - seg_file_start).min(4096 - dest_off)
} else {
0
};
kern()
.map_fresh_range(pid, VirtualAddress::from_raw(page_start), pages, perms)
.unwrap_or_else(|_| crate::halt(uart, "cannot map segment"));
if copy_len > 0 {
let src = segment.file_offset + seg_file_start;
if src + copy_len <= file_size {
unsafe {
core::ptr::copy_nonoverlapping(
buf.as_ptr().add(src),
(page as *mut u8).add(dest_off),
copy_len,
);
}
}
}
let flags = if segment.flags.execute {
mmu::Sv39Flags::user_rx()
} else if segment.flags.write {
mmu::Sv39Flags::user_rw()
} else {
mmu::Sv39Flags::user_rx()
};
mmu::map(
user_pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(page_va),
pa: types::address::PhysicalAddress::from_raw(page),
pages: 1,
flags,
},
);
let copy_len = segment.file_size.min(segment.mem_size);
let data = segment
.file_offset
.checked_add(copy_len)
.and_then(|end| file.get(segment.file_offset..end))
.unwrap_or_else(|| crate::halt(uart, "segment outside file"));
if !copy_to_user(pid, segment.vaddr, data) {
crate::halt(uart, "segment copy failed");
}
}
for i in 0..16usize {
let sp = mmu::allocator::alloc_zeroed_page();
mmu::map(
user_pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(USER_VA_BASE - (16 - i) * 4096),
pa: types::address::PhysicalAddress::from_raw(sp),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
},
);
}
let stack_bottom = USER_VA_BASE - USER_STACK_PAGES * PAGE_SIZE;
kern()
.platform_mut()
.set_page_table(pid, user_pt as *const mmu::PageTable as usize);
.map_fresh_range(
pid,
VirtualAddress::from_raw(stack_bottom),
USER_STACK_PAGES,
PagePerms::ReadWrite,
)
.unwrap_or_else(|_| crate::halt(uart, "cannot map stack"));
elf_info.entry_point
}

9
crates/adapter-riscv/src/boot_fs.rs
View File

@@ -1,11 +1,9 @@
use crate::hw::virtio_blk::VirtioBlk;
const SECTOR_SIZE: usize = 512;
const MAX_FILE_SIZE: usize = 64 * 1024; // 64K max for init binary
struct BootFat16 {
reserved_sectors: u16,
sectors_per_fat: u16,
root_dir_start: u32,
root_dir_sectors: u32,
data_start: u32,
@@ -31,12 +29,11 @@ impl BootFat16 {
let fat_start = reserved_sectors as u32;
let root_dir_start = fat_start + (num_fats as u32) * (sectors_per_fat as u32);
let root_dir_sectors = ((root_entry_count as u32) * 32 + 511) / 512;
let root_dir_sectors = ((root_entry_count as u32) * 32).div_ceil(512);
let data_start = root_dir_start + root_dir_sectors;
Some(Self {
reserved_sectors,
sectors_per_fat,
root_dir_start,
root_dir_sectors,
data_start,
@@ -84,7 +81,7 @@ pub fn read_file_from_disk(
let mut found = false;
for s in 0..fs.root_dir_sectors {
if !blk.read_sector((fs.root_dir_start + s) as u32, &mut sector_buf) {
if !blk.read_sector(fs.root_dir_start + s, &mut sector_buf) {
continue;
}
for i in 0..(SECTOR_SIZE / 32) {
@@ -121,7 +118,7 @@ pub fn read_file_from_disk(
let mut cluster = first_cluster;
let sectors_per_cluster = fs.sectors_per_cluster as u32;
while read < size && cluster >= 2 && cluster < 0xFFF8 {
while read < size && (2..0xFFF8).contains(&cluster) {
for s in 0..sectors_per_cluster {
if read >= size {
break;

77
crates/adapter-riscv/src/devices/gpu_setup.rs
View File

@@ -1,13 +1,9 @@
use super::{dma_buffer, info_page, kern, map_mmio};
use crate::hw::uart::Uart;
use crate::hw::virtio_gpu::VirtioGpu;
use crate::mmu;
use types::display::Dimension;
use types::id::ProcessId;
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
unsafe { &mut *crate::kernel_ptr() }
}
const GPU_VA: usize = 0x20000;
const VQUEUE_VA: usize = 0x21000;
const CMDBUF_VA: usize = 0x23000;
@@ -15,6 +11,8 @@ const PHYSINFO_VA: usize = 0x24000;
const FB_VA: usize = 0x100000;
const DEFAULT_WIDTH: Dimension = Dimension::new(640);
const DEFAULT_HEIGHT: Dimension = Dimension::new(480);
/// The framebuffer alone is ~1000 pages at 1280x800.
const DISPLAY_PAGE_LIMIT: usize = 2048;
pub fn configure(uart: &Uart, display_pid: ProcessId) {
let gpu = match VirtioGpu::probe() {
@@ -25,16 +23,13 @@ pub fn configure(uart: &Uart, display_pid: ProcessId) {
}
};
let pt_addr = kern().platform_mut().page_table_for(display_pid);
let pt = unsafe { &mut *(pt_addr as *mut mmu::PageTable) };
kern()
.set_page_limit(display_pid, DISPLAY_PAGE_LIMIT)
.unwrap_or_else(|_| crate::halt(uart, "display quota"));
map_page(pt, GPU_VA, gpu.base());
let vq_phys = alloc_contiguous(2);
map_range(pt, VQUEUE_VA, vq_phys, 2);
let cmd_phys = mmu::allocator::alloc_zeroed_page();
map_page(pt, CMDBUF_VA, cmd_phys);
map_mmio(uart, display_pid, GPU_VA, gpu.base(), 1);
let vq_phys = dma_buffer(uart, display_pid, VQUEUE_VA, 2);
let cmd_phys = dma_buffer(uart, display_pid, CMDBUF_VA, 1);
gpu.init_device(vq_phys);
@@ -45,7 +40,6 @@ pub fn configure(uart: &Uart, display_pid: ProcessId) {
cmd_va: types::address::VirtualAddress::from_raw(cmd_phys),
resp_pa: types::address::PhysicalAddress::from_raw(cmd_phys + 512),
};
use hw_virtio::mmio::MmioAccess;
let (width, height) = match hw_virtio::gpu::get_display_info(gpu.mmio(), &queue, &dma) {
Some((w, h)) => {
@@ -62,20 +56,21 @@ pub fn configure(uart: &Uart, display_pid: ProcessId) {
}
};
let fb_pages = (width.raw() as usize * height.raw() as usize * 4 + 4095) / 4096;
let fb_phys = alloc_contiguous(fb_pages);
map_range(pt, FB_VA, fb_phys, fb_pages);
let fb_pages = (width.raw() as usize * height.raw() as usize * 4).div_ceil(4096);
let fb_phys = dma_buffer(uart, display_pid, FB_VA, fb_pages);
let info_phys = mmu::allocator::alloc_zeroed_page();
map_page(pt, PHYSINFO_VA, info_phys);
unsafe {
let info = info_phys as *mut u64;
core::ptr::write_volatile(info, cmd_phys as u64);
core::ptr::write_volatile(info.add(1), (cmd_phys + 512) as u64);
core::ptr::write_volatile(info.add(2), fb_phys as u64);
core::ptr::write_volatile(info.add(3), width.raw() as u64);
core::ptr::write_volatile(info.add(4), height.raw() as u64);
}
info_page(
uart,
display_pid,
PHYSINFO_VA,
&[
cmd_phys as u64,
(cmd_phys + 512) as u64,
fb_phys as u64,
width.raw() as u64,
height.raw() as u64,
],
);
if gpu.setup_framebuffer(cmd_phys, vq_phys, fb_phys, width, height) {
uart.write_str("k-os: VirtIO GPU configured\n");
@@ -83,29 +78,3 @@ pub fn configure(uart: &Uart, display_pid: ProcessId) {
uart.write_str("k-os: VirtIO GPU setup failed (display will be headless)\n");
}
}
fn map_page(pt: &mut mmu::PageTable, va: usize, pa: usize) {
mmu::map(
pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(va),
pa: types::address::PhysicalAddress::from_raw(pa),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
},
);
}
fn map_range(pt: &mut mmu::PageTable, va_base: usize, pa_base: usize, count: usize) {
for i in 0..count {
map_page(pt, va_base + i * 4096, pa_base + i * 4096);
}
}
fn alloc_contiguous(pages: usize) -> usize {
let first = mmu::allocator::alloc_zeroed_page();
for _ in 1..pages {
mmu::allocator::alloc_zeroed_page();
}
first
}

61
crates/adapter-riscv/src/devices/input_setup.rs
View File

@@ -1,5 +1,6 @@
use super::{dma_buffer, info_page, map_mmio};
use crate::board;
use crate::hw::uart::Uart;
use crate::{board, mmu};
use types::id::ProcessId;
const UART_VA: usize = 0x20000;
@@ -8,34 +9,22 @@ const INPUT_VQUEUE_VA: usize = 0x26000;
const INPUT_EVBUF_VA: usize = 0x28000;
const INPUT_PHYSINFO_VA: usize = 0x29000;
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
unsafe { &mut *crate::kernel_ptr() }
}
pub fn configure(uart: &Uart, input_pid: ProcessId) {
let pt_addr = kern().platform_mut().page_table_for(input_pid);
let pt = unsafe { &mut *(pt_addr as *mut mmu::PageTable) };
// Always map UART for fallback
map_page(pt, UART_VA, board::UART_BASE);
map_mmio(uart, input_pid, UART_VA, board::UART_BASE, 1);
// Try to find VirtIO input device
if let Some(mmio) = hw_virtio::mmio::scan_for_device(hw_virtio::mmio::DEVICE_ID_INPUT) {
let base = mmio.base();
map_page(pt, INPUT_MMIO_VA, base);
let vq_phys = alloc_contiguous(2);
map_range(pt, INPUT_VQUEUE_VA, vq_phys, 2);
let evbuf_phys = mmu::allocator::alloc_zeroed_page();
map_page(pt, INPUT_EVBUF_VA, evbuf_phys);
let info_phys = mmu::allocator::alloc_zeroed_page();
map_page(pt, INPUT_PHYSINFO_VA, info_phys);
unsafe {
core::ptr::write_volatile(info_phys as *mut u64, evbuf_phys as u64);
core::ptr::write_volatile((info_phys as *mut u64).add(1), vq_phys as u64);
}
map_mmio(uart, input_pid, INPUT_MMIO_VA, base, 1);
let vq_phys = dma_buffer(uart, input_pid, INPUT_VQUEUE_VA, 2);
let evbuf_phys = dma_buffer(uart, input_pid, INPUT_EVBUF_VA, 1);
info_page(
uart,
input_pid,
INPUT_PHYSINFO_VA,
&[evbuf_phys as u64, vq_phys as u64],
);
hw_virtio::mmio::init_legacy_device(&mmio, vq_phys, 32);
uart.write_str("k-os: VirtIO keyboard configured\n");
@@ -43,29 +32,3 @@ pub fn configure(uart: &Uart, input_pid: ProcessId) {
uart.write_str("k-os: UART mapped for input\n");
}
}
fn map_page(pt: &mut mmu::PageTable, va: usize, pa: usize) {
mmu::map(
pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(va),
pa: types::address::PhysicalAddress::from_raw(pa),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
},
);
}
fn map_range(pt: &mut mmu::PageTable, va_base: usize, pa_base: usize, count: usize) {
for i in 0..count {
map_page(pt, va_base + i * 4096, pa_base + i * 4096);
}
}
fn alloc_contiguous(pages: usize) -> usize {
let first = mmu::allocator::alloc_zeroed_page();
for _ in 1..pages {
mmu::allocator::alloc_zeroed_page();
}
first
}

81
crates/adapter-riscv/src/devices/mod.rs
View File

@@ -1,3 +1,84 @@
pub mod gpu_setup;
pub mod input_setup;
pub mod storage_setup;
use crate::hw::uart::Uart;
use types::address::{PhysicalAddress, VirtualAddress};
use types::bounded::PageCount;
use types::id::ProcessId;
use types::mapping::PagePerms;
// Boot-time device wiring. Everything here goes through the kernel's
// mapping API so the kernel knows about every page a driver can touch.
// P5 replaces these hardcoded grants with capabilities handed to init.
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
unsafe { &mut *crate::kernel_ptr() }
}
fn pages(uart: &Uart, n: usize) -> PageCount {
u16::try_from(n)
.ok()
.and_then(PageCount::new)
.unwrap_or_else(|| crate::halt(uart, "bad page count"))
}
/// Maps device MMIO at `pa` into `pid` at `va`.
pub(crate) fn map_mmio(uart: &Uart, pid: ProcessId, va: usize, pa: usize, n: usize) {
let id = kern()
.create_device_region(pid, PhysicalAddress::from_raw(pa), pages(uart, n))
.unwrap_or_else(|_| crate::halt(uart, "device region failed"));
kern()
.map_region(id, pid, VirtualAddress::from_raw(va), PagePerms::ReadWrite)
.unwrap_or_else(|_| crate::halt(uart, "device map failed"));
}
/// Physically contiguous, zeroed DMA memory for `owner`, mapped read-write
/// at `va`. Returns its physical base for programming the device.
pub(crate) fn dma_buffer(uart: &Uart, owner: ProcessId, va: usize, n: usize) -> usize {
let id = kern()
.create_pool_region(owner, pages(uart, n))
.unwrap_or_else(|_| crate::halt(uart, "DMA region failed"));
kern()
.map_region(
id,
owner,
VirtualAddress::from_raw(va),
PagePerms::ReadWrite,
)
.unwrap_or_else(|_| crate::halt(uart, "DMA map failed"));
kern()
.region_base(id)
.unwrap_or_else(|| crate::halt(uart, "DMA region vanished"))
.raw()
}
/// Shares a fresh region between `owner` and `other` at the same `va`.
pub(crate) fn shared_buffer(uart: &Uart, owner: ProcessId, other: ProcessId, va: usize, n: usize) {
let id = kern()
.create_pool_region(owner, pages(uart, n))
.unwrap_or_else(|_| crate::halt(uart, "shared region failed"));
for pid in [owner, other] {
kern()
.map_region(id, pid, VirtualAddress::from_raw(va), PagePerms::ReadWrite)
.unwrap_or_else(|_| crate::halt(uart, "shared map failed"));
}
}
/// Writes `words` into a fresh page mapped at `va` in `pid`, so a driver
/// can learn the physical addresses of its DMA buffers.
pub(crate) fn info_page(uart: &Uart, pid: ProcessId, va: usize, words: &[u64]) {
let va = VirtualAddress::from_raw(va);
kern()
.map_fresh_range(pid, va, 1, PagePerms::ReadOnly)
.unwrap_or_else(|_| crate::halt(uart, "info page failed"));
let pa = kern()
.user_page_frame(pid, va)
.unwrap_or_else(|| crate::halt(uart, "info page vanished"));
let info = pa.raw() as *mut u64;
for (i, &w) in words.iter().enumerate() {
// SAFETY: `pa` is a pool frame just mapped for `pid`, identity
// mapped in kernel space, and `words` fits in one page.
unsafe { core::ptr::write_volatile(info.add(i), w) };
}
}

118
crates/adapter-riscv/src/devices/storage_setup.rs
View File

@@ -1,11 +1,7 @@
use super::{dma_buffer, info_page, map_mmio, shared_buffer};
use crate::hw::uart::Uart;
use crate::mmu;
use types::id::ProcessId;
fn kern() -> &'static mut kernel::Kernel<crate::platform::RiscVPlatform> {
unsafe { &mut *crate::kernel_ptr() }
}
const VIRTIO_VA: usize = 0x20000;
const VQUEUE_VA: usize = 0x21000;
const REQHDR_VA: usize = 0x23000;
@@ -14,111 +10,27 @@ const SHARED_BUF_VA: usize = 0x30000;
const SHARED_BUF_PAGES: usize = 16;
pub fn configure(uart: &Uart, init_pid: ProcessId, storage_pid: ProcessId, virtio_base: usize) {
let storage_pt = get_pt(storage_pid);
let init_pt = get_pt(init_pid);
map_virtio_mmio(storage_pt, virtio_base);
let vqueue_phys = alloc_and_map_virtqueue(storage_pt);
let reqhdr_phys = alloc_and_map_reqhdr(storage_pt);
write_phys_info(storage_pt, reqhdr_phys);
map_mmio(uart, storage_pid, VIRTIO_VA, virtio_base, 1);
let vqueue_phys = dma_buffer(uart, storage_pid, VQUEUE_VA, 2);
let reqhdr_phys = dma_buffer(uart, storage_pid, REQHDR_VA, 1);
info_page(
uart,
storage_pid,
PHYSINFO_VA,
&[
reqhdr_phys as u64,
(reqhdr_phys + 16) as u64,
(reqhdr_phys + 32) as u64,
],
);
reinit_device(virtio_base, vqueue_phys);
uart.write_str("k-os: VirtIO configured for storage\n");
map_shared_buffer(init_pt, storage_pt);
shared_buffer(uart, init_pid, storage_pid, SHARED_BUF_VA, SHARED_BUF_PAGES);
uart.write_str("k-os: shared buffer mapped\n");
}
fn get_pt(pid: ProcessId) -> &'static mut mmu::PageTable {
let addr = kern().platform_mut().page_table_for(pid);
unsafe { &mut *(addr as *mut mmu::PageTable) }
}
fn map_virtio_mmio(pt: &mut mmu::PageTable, virtio_base: usize) {
mmu::map(
pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(VIRTIO_VA),
pa: types::address::PhysicalAddress::from_raw(virtio_base),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
},
);
}
fn alloc_and_map_virtqueue(pt: &mut mmu::PageTable) -> usize {
let phys = mmu::allocator::alloc_zeroed_page();
mmu::allocator::alloc_zeroed_page();
for i in 0..2usize {
mmu::map(
pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(VQUEUE_VA + i * 4096),
pa: types::address::PhysicalAddress::from_raw(phys + i * 4096),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
},
);
}
phys
}
fn alloc_and_map_reqhdr(pt: &mut mmu::PageTable) -> usize {
let phys = mmu::allocator::alloc_zeroed_page();
mmu::map(
pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(REQHDR_VA),
pa: types::address::PhysicalAddress::from_raw(phys),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
},
);
phys
}
fn write_phys_info(pt: &mut mmu::PageTable, reqhdr_phys: usize) {
let phys = mmu::allocator::alloc_zeroed_page();
mmu::map(
pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(PHYSINFO_VA),
pa: types::address::PhysicalAddress::from_raw(phys),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
},
);
unsafe {
let info = phys as *mut u64;
core::ptr::write_volatile(info, reqhdr_phys as u64);
core::ptr::write_volatile(info.add(1), (reqhdr_phys + 16) as u64);
core::ptr::write_volatile(info.add(2), (reqhdr_phys + 32) as u64);
}
}
fn reinit_device(base: usize, vqueue_phys: usize) {
let mmio = hw_virtio::mmio::DirectMmio::new(base);
hw_virtio::mmio::init_legacy_device(&mmio, vqueue_phys, hw_virtio::queue::QUEUE_SIZE as u32);
}
fn map_shared_buffer(init_pt: &mut mmu::PageTable, storage_pt: &mut mmu::PageTable) {
for i in 0..SHARED_BUF_PAGES {
let page = mmu::allocator::alloc_zeroed_page();
let va = SHARED_BUF_VA + i * 4096;
let mapping = mmu::Mapping {
va: types::address::VirtualAddress::from_raw(va),
pa: types::address::PhysicalAddress::from_raw(page),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
};
mmu::map(init_pt, mapping);
mmu::map(
storage_pt,
mmu::Mapping {
va: types::address::VirtualAddress::from_raw(va),
pa: types::address::PhysicalAddress::from_raw(page),
pages: 1,
flags: mmu::Sv39Flags::user_rw(),
},
);
}
}

18
crates/adapter-riscv/src/hw/virtio_blk.rs
View File

@@ -29,20 +29,18 @@ impl VirtioBlk {
pub fn probe() -> Option<Self> {
let mmio = mmio::scan_for_device(mmio::DEVICE_ID_BLOCK)?;
let vq_phys = unsafe { core::ptr::addr_of!(VQUEUE) as usize };
let vq_phys = core::ptr::addr_of!(VQUEUE) as usize;
mmio::init_legacy_device(&mmio, vq_phys, QUEUE_SIZE as u32);
let queue = VirtioQueue::new(QueueAddrs::from_base(vq_phys));
let dma = unsafe {
DmaAddrs {
header_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(REQ_HEADER) as usize),
header_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(REQ_HEADER) as usize),
status_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(REQ_STATUS) as usize),
status_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(REQ_STATUS) as usize),
data_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(SECTOR_BUF) as usize),
data_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(SECTOR_BUF) as usize),
}
let dma = DmaAddrs {
header_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(REQ_HEADER) as usize),
header_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(REQ_HEADER) as usize),
status_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(REQ_STATUS) as usize),
status_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(REQ_STATUS) as usize),
data_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(SECTOR_BUF) as usize),
data_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(SECTOR_BUF) as usize),
};
Some(Self { mmio, queue, dma })

9
crates/adapter-riscv/src/kernel_cell.rs
View File

@@ -18,8 +18,13 @@ impl<T> KernelCell<T> {
}
}
pub fn init(&self, value: T) {
unsafe { (*self.inner.get()).write(value) };
/// Initializes the cell by letting `build` write the value straight
/// into its storage, so a large `T` never lives on the stack.
///
/// # Safety
/// `build` must fully initialize the `T` behind the pointer.
pub unsafe fn init_with(&self, build: impl FnOnce(*mut T)) {
build(unsafe { (*self.inner.get()).as_mut_ptr() });
self.initialized.store(true, Ordering::Release);
}

11
crates/adapter-riscv/src/main.rs
View File

@@ -41,16 +41,15 @@ extern "C" fn kmain() -> ! {
let uart = Uart::new();
uart.write_str("k-os: booted on RISC-V\n");
let kernel_pt = boot::hardware::init(&uart);
boot::hardware::init(&uart);
let mut blk =
boot::hardware::probe_block_device(&uart).unwrap_or_else(|| halt(&uart, "no block device"));
let (init, _) = boot::loader::load_init(&uart, &mut blk, kernel_pt);
let storage = boot::loader::load_server(&uart, &mut blk, kernel_pt, b"storage.elf", "storage");
let display = boot::loader::load_server(&uart, &mut blk, kernel_pt, b"display.elf", "display");
let input = boot::loader::load_server(&uart, &mut blk, kernel_pt, b"input.elf", "input");
let (init, _) = boot::loader::load_init(&uart, &mut blk);
let storage = boot::loader::load_server(&uart, &mut blk, b"storage.elf", "storage");
let display = boot::loader::load_server(&uart, &mut blk, b"display.elf", "display");
let input = boot::loader::load_server(&uart, &mut blk, b"input.elf", "input");
let virtio_base = blk.base();
drop(blk);
devices::storage_setup::configure(&uart, init.pid, storage.pid, virtio_base);
devices::gpu_setup::configure(&uart, display.pid);

15
crates/adapter-riscv/src/mmu/allocator.rs
View File

@@ -1,7 +1,9 @@
use super::page_table::{PAGE_SIZE, PageTable};
use crate::board;
// Boot-time bump allocator for page tables.
// Only used during init before the scheduler starts.
// Boot-time bump allocator for kernel-only memory: the kernel page table
// and boot scratch buffers. User memory and user page tables come from the
// kernel's frame table instead. It must never reach the user pool.
static mut NEXT_FREE: usize = 0;
pub fn init_allocator(start: usize) {
@@ -13,6 +15,9 @@ pub fn init_allocator(start: usize) {
pub fn alloc_zeroed_page() -> usize {
unsafe {
let addr = NEXT_FREE;
if addr + PAGE_SIZE > board::USER_POOL_BASE {
crate::halt(&crate::hw::uart::Uart::new(), "boot allocator exhausted");
}
NEXT_FREE += PAGE_SIZE;
core::ptr::write_bytes(addr as *mut u8, 0, PAGE_SIZE);
addr
@@ -20,9 +25,5 @@ pub fn alloc_zeroed_page() -> usize {
}
pub fn alloc_page_table() -> &'static mut PageTable {
unsafe {
let addr = alloc_zeroed_page();
let table = &mut *(addr as *mut PageTable);
table
}
unsafe { &mut *(alloc_zeroed_page() as *mut PageTable) }
}

4
crates/adapter-riscv/src/mmu/mod.rs
View File

@@ -3,6 +3,6 @@ pub mod page_table;
pub use allocator::init_allocator;
pub use page_table::{
Mapping, PAGE_SIZE, PageTable, Sv39Flags, activate, create_user_page_table,
init_kernel_page_table, map, translate,
PAGE_SIZE, PageTable, Sv39Flags, USER_VA_END, USER_VA_START, activate, active_root,
create_user_root, init_kernel_page_table, map_user_page, unmap_user_page,
};

185
crates/adapter-riscv/src/mmu/page_table.rs
View File

@@ -1,6 +1,8 @@
use super::allocator::alloc_page_table;
use crate::board;
use platform::{FrameSource, MapError};
use types::address::{PhysicalAddress, VirtualAddress};
use types::mapping::PagePerms;
pub use types::mapping::Mapping;
pub const PAGE_SIZE: usize = 4096;
@@ -32,29 +34,12 @@ impl Sv39Flags {
}
}
pub fn user_rx() -> Self {
Self {
bits: PTE_R | PTE_X | PTE_U,
}
}
pub fn user_rw() -> Self {
Self {
bits: PTE_R | PTE_W | PTE_U,
}
}
pub fn user(read: bool, write: bool, exec: bool) -> Self {
let mut bits = PTE_U;
if read {
bits |= PTE_R;
}
if write {
bits |= PTE_W;
}
if exec {
bits |= PTE_X;
}
pub fn user(perms: PagePerms) -> Self {
let bits = match perms {
PagePerms::ReadOnly => PTE_R | PTE_U,
PagePerms::ReadWrite => PTE_R | PTE_W | PTE_U,
PagePerms::ReadExecute => PTE_R | PTE_X | PTE_U,
};
Self { bits }
}
@@ -106,27 +91,6 @@ fn vpn(va: usize, level: usize) -> usize {
(va >> (12 + 9 * level)) & 0x1FF
}
pub fn translate(root: &PageTable, va: usize) -> Option<usize> {
let vpn2 = vpn(va, 2);
if !root.entries[vpn2].is_valid() {
return None;
}
let l1 = unsafe { &*((root.entries[vpn2].ppn() << 12) as *const PageTable) };
let vpn1 = vpn(va, 1);
if !l1.entries[vpn1].is_valid() {
return None;
}
let l0 = unsafe { &*((l1.entries[vpn1].ppn() << 12) as *const PageTable) };
let vpn0 = vpn(va, 0);
if !l0.entries[vpn0].is_valid() {
return None;
}
let pa = (l0.entries[vpn0].ppn() << 12) as usize | (va & 0xFFF);
Some(pa)
}
fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) {
let vpn2 = vpn(va, 2);
let vpn1 = vpn(va, 1);
@@ -246,34 +210,113 @@ pub fn activate(root: &PageTable) {
}
}
pub fn create_user_page_table(
kernel_root: &PageTable,
_program_pa: PhysicalAddress,
_program_pages: usize,
_user_va: VirtualAddress,
) -> &'static mut PageTable {
let root = alloc_page_table();
// ---- user address spaces ----
//
// User mappings live in [USER_VA_START, USER_VA_END), which is root slot 0,
// L1 slots 0..USER_L1_SLOTS. Every address space gets its own root and its
// own L1 for slot 0, and the kernel maps nothing in those L1 slots, so all
// page tables below the user window are private to one process. Kernel
// mappings elsewhere are shared by copying the kernel's entries.
for i in 0..512 {
if !kernel_root.entries[i].is_valid() {
continue;
}
if i == 0 {
// User mappings land in VPN[2]=0 — each process needs its own L1
let kernel_l1_addr = (kernel_root.entries[i].ppn() << 12) as *const PageTable;
let kernel_l1 = unsafe { &*kernel_l1_addr };
let new_l1 = alloc_page_table();
for j in 0..512 {
if kernel_l1.entries[j].is_valid() {
new_l1.entries[j] = kernel_l1.entries[j];
}
}
let ppn = (new_l1 as *const PageTable as u64) >> 12;
root.entries[i] = PageTableEntry::new_branch(ppn);
} else {
root.entries[i] = kernel_root.entries[i];
pub const USER_VA_START: usize = 0x1000;
pub const USER_VA_END: usize = 0x0200_0000;
const USER_L1_SLOTS: usize = USER_VA_END >> 21;
fn table_at(pa: usize) -> &'static mut PageTable {
// SAFETY: page tables are identity mapped in kernel space.
unsafe { &mut *(pa as *mut PageTable) }
}
fn fresh_table(frames: &mut dyn FrameSource) -> Result<usize, MapError> {
let pa = frames
.alloc_table_frame()
.ok_or(MapError::OutOfFrames)?
.raw();
table_at(pa).zero();
Ok(pa)
}
fn in_user_window(va: usize) -> bool {
va.is_multiple_of(PAGE_SIZE) && (USER_VA_START..USER_VA_END).contains(&va)
}
/// Builds a root table sharing the kernel's mappings, with a private L1 for
/// the user window. Returns the root's physical address.
pub fn create_user_root(
kernel_root: &PageTable,
frames: &mut dyn FrameSource,
) -> Result<usize, MapError> {
let root_pa = fresh_table(frames)?;
let l1_pa = fresh_table(frames)?;
let root = table_at(root_pa);
let l1 = table_at(l1_pa);
for (i, entry) in kernel_root.entries.iter().enumerate() {
if i != 0 && entry.is_valid() {
root.entries[i] = *entry;
}
}
root
if kernel_root.entries[0].is_valid() {
let kernel_l1 = table_at((kernel_root.entries[0].ppn() << 12) as usize);
for j in USER_L1_SLOTS..512 {
l1.entries[j] = kernel_l1.entries[j];
}
}
root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12);
Ok(root_pa)
}
fn user_leaf_table(
root_pa: usize,
va: usize,
frames: Option<&mut dyn FrameSource>,
) -> Result<&'static mut PageTable, MapError> {
let root = table_at(root_pa);
let l1 = table_at((root.entries[0].ppn() << 12) as usize);
let slot = &mut l1.entries[vpn(va, 1)];
if !slot.is_valid() {
let frames = frames.ok_or(MapError::NoAddressSpace)?;
let l0_pa = fresh_table(frames)?;
*slot = PageTableEntry::new_branch((l0_pa as u64) >> 12);
}
Ok(table_at((slot.ppn() << 12) as usize))
}
pub fn map_user_page(
root_pa: usize,
va: usize,
pa: usize,
flags: Sv39Flags,
frames: &mut dyn FrameSource,
) -> Result<(), MapError> {
if !in_user_window(va) || !pa.is_multiple_of(PAGE_SIZE) {
return Err(MapError::InvalidAddress);
}
let l0 = user_leaf_table(root_pa, va, Some(frames))?;
l0.entries[vpn(va, 0)] = PageTableEntry::new_leaf((pa as u64) >> 12, flags);
flush_tlb(va);
Ok(())
}
pub fn unmap_user_page(root_pa: usize, va: usize) {
if !in_user_window(va) {
return;
}
if let Ok(l0) = user_leaf_table(root_pa, va, None) {
l0.entries[vpn(va, 0)] = PageTableEntry::empty();
flush_tlb(va);
}
}
fn flush_tlb(va: usize) {
// SAFETY: sfence.vma only invalidates translation caches.
unsafe { core::arch::asm!("sfence.vma {0}, zero", in(reg) va) };
}
/// Physical address of the root table the hart is translating with.
pub fn active_root() -> usize {
let satp: usize;
// SAFETY: reading satp has no side effects.
unsafe { core::arch::asm!("csrr {0}, satp", out(reg) satp) };
(satp & ((1 << 44) - 1)) << 12
}

152
crates/adapter-riscv/src/platform.rs
View File

@@ -1,11 +1,9 @@
use domain::memory::MemoryRegion;
use domain::process::MAX_PROCESSES;
use platform::Platform;
use platform::{FrameSource, MapError, Platform};
use types::{
address::{EntryPoint, PhysicalAddress, StackPointer, VirtualAddress},
bounded::PageCount,
id::{CoreId, IrqId, ProcessId},
permission::{Permission, PermissionSet},
mapping::PagePerms,
time::{Ticks, TimerDuration},
};
@@ -17,6 +15,7 @@ use crate::trap;
pub struct RiscVPlatform {
num_cores: u8,
kernel_pt: usize,
/// Root table of each process's address space, 0 if it has none.
page_tables: [usize; MAX_PROCESSES],
}
@@ -33,40 +32,76 @@ impl RiscVPlatform {
self.page_tables[pid.raw() as usize]
}
pub fn set_page_table(&mut self, pid: ProcessId, addr: usize) {
self.page_tables[pid.raw() as usize] = addr;
}
fn ensure_page_table(&mut self, pid: ProcessId) -> &mut mmu::PageTable {
let idx = pid.raw() as usize;
if self.page_tables[idx] == 0 {
let kernel_pt = unsafe { &*(self.kernel_pt as *const mmu::PageTable) };
let user_pt = mmu::create_user_page_table(
kernel_pt,
PhysicalAddress::from_raw(0),
0,
VirtualAddress::from_raw(0x10000),
);
self.page_tables[idx] = user_pt as *const mmu::PageTable as usize;
fn root(&self, pid: ProcessId) -> Option<usize> {
match self.page_tables.get(pid.raw() as usize) {
Some(&0) | None => None,
Some(&root) => Some(root),
}
unsafe { &mut *(self.page_tables[idx] as *mut mmu::PageTable) }
}
}
impl Platform for RiscVPlatform {
fn configure_protection(&mut self, process: ProcessId, regions: &[MemoryRegion]) {
let pt = self.ensure_page_table(process);
for region in regions {
mmu::map(
pt,
mmu::Mapping {
va: VirtualAddress::from_raw(region.base().raw()),
pa: region.base(),
pages: region.size().value() as usize,
flags: mmu::Sv39Flags::user(true, true, true),
},
);
const USER_VA_START: usize = mmu::USER_VA_START;
const USER_VA_END: usize = mmu::USER_VA_END;
fn create_address_space(
&mut self,
process: ProcessId,
frames: &mut dyn FrameSource,
) -> Result<(), MapError> {
let slot = self
.page_tables
.get_mut(process.raw() as usize)
.ok_or(MapError::NoAddressSpace)?;
// SAFETY: the kernel page table is identity mapped and lives forever.
let kernel_root = unsafe { &*(self.kernel_pt as *const mmu::PageTable) };
*slot = mmu::create_user_root(kernel_root, frames)?;
Ok(())
}
fn destroy_address_space(&mut self, process: ProcessId) {
let Some(root) = self.root(process) else {
return;
};
// The caller may be tearing down the space we are running on; get
// off it before the kernel frees its frames.
if mmu::active_root() == root {
// SAFETY: the kernel page table is identity mapped and lives forever.
mmu::activate(unsafe { &*(self.kernel_pt as *const mmu::PageTable) });
}
self.page_tables[process.raw() as usize] = 0;
// SAFETY: flushing translations has no other effect.
unsafe { core::arch::asm!("sfence.vma zero, zero") };
}
fn map_page(
&mut self,
process: ProcessId,
va: VirtualAddress,
pa: PhysicalAddress,
perms: PagePerms,
frames: &mut dyn FrameSource,
) -> Result<(), MapError> {
let root = self.root(process).ok_or(MapError::NoAddressSpace)?;
mmu::map_user_page(
root,
va.raw(),
pa.raw(),
mmu::Sv39Flags::user(perms),
frames,
)
}
fn unmap_page(&mut self, process: ProcessId, va: VirtualAddress) {
if let Some(root) = self.root(process) {
mmu::unmap_user_page(root, va.raw());
}
}
fn zero_frame(&mut self, pa: PhysicalAddress) {
// SAFETY: pool frames are identity mapped kernel RAM and the kernel
// only zeroes frames it has just allocated.
unsafe { core::ptr::write_bytes(pa.raw() as *mut u8, 0, mmu::PAGE_SIZE) };
}
fn enable_user_mode(&mut self, entry: EntryPoint, stack: StackPointer) -> ! {
@@ -117,63 +152,10 @@ impl Platform for RiscVPlatform {
self.num_cores
}
fn map_shared(
&mut self,
process: ProcessId,
base: PhysicalAddress,
size: PageCount,
permissions: PermissionSet,
) {
let pt = self.ensure_page_table(process);
mmu::map(
pt,
mmu::Mapping {
va: VirtualAddress::from_raw(base.raw()),
pa: base,
pages: size.value() as usize,
flags: mmu::Sv39Flags::user(
permissions.allows(Permission::Read),
permissions.allows(Permission::Write),
permissions.allows(Permission::Execute),
),
},
);
}
fn debug_putchar(&self, byte: u8) {
crate::hw::uart::Uart::new().put_char(byte);
}
fn translate_va(&self, process: ProcessId, va: VirtualAddress) -> Option<PhysicalAddress> {
let pt_addr = self.page_tables[process.raw() as usize];
if pt_addr == 0 {
return None;
}
let pt = unsafe { &*(pt_addr as *const mmu::PageTable) };
mmu::translate(pt, va.raw()).map(PhysicalAddress::from_raw)
}
fn map_user_page(
&mut self,
process: ProcessId,
va: VirtualAddress,
pa: PhysicalAddress,
writable: bool,
executable: bool,
) {
let pt = self.ensure_page_table(process);
let flags = mmu::Sv39Flags::user(true, writable, executable);
mmu::map(
pt,
mmu::Mapping {
va,
pa,
pages: 1,
flags,
},
);
}
fn halt(&self) -> ! {
unsafe {
core::arch::asm!(

40
crates/adapter-riscv/src/trap/mod.rs
View File

@@ -7,16 +7,6 @@ pub struct TrapFrame {
pub sstatus: usize,
}
impl TrapFrame {
pub fn new() -> Self {
Self {
regs: [0; 32],
sepc: 0,
sstatus: 0,
}
}
}
global_asm!(
r#"
.align 4
@@ -134,7 +124,7 @@ pub fn init() {
fn trap_vector();
}
unsafe {
let addr = trap_vector as usize;
let addr = trap_vector as *const () as usize;
core::arch::asm!("csrw stvec, {0}", in(reg) addr);
core::arch::asm!("csrw sscratch, zero");
// SIE stays clear — interrupts are enabled only when entering userspace via sret
@@ -357,13 +347,13 @@ fn switch_to_thread(
apply_pending_result(frame, next_tid);
if next_pid != current_pid {
if let Some(pid) = next_pid {
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) };
crate::mmu::activate(pt);
}
if next_pid != current_pid
&& let Some(pid) = next_pid
{
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) };
crate::mmu::activate(pt);
}
}
}
@@ -388,13 +378,13 @@ fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) {
apply_pending_result(frame, next_tid);
if next_pid != current_pid {
if let Some(pid) = next_pid {
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) };
crate::mmu::activate(pt);
}
if next_pid != current_pid
&& let Some(pid) = next_pid
{
let pt_addr = kern.platform_mut().page_table_for(pid);
if pt_addr != 0 {
let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) };
crate::mmu::activate(pt);
}
}
} else {