diff --git a/crates/adapter-riscv/linker.ld b/crates/adapter-riscv/linker.ld index 754e5b9..7508c08 100644 --- a/crates/adapter-riscv/linker.ld +++ b/crates/adapter-riscv/linker.ld @@ -1,13 +1,20 @@ OUTPUT_ARCH(riscv) -ENTRY(_start) + +/* The kernel runs in the upper half: linked at its physical load address + plus PHYS_OFFSET (mmu::PHYS_OFFSET), loaded at the physical address. + Firmware jumps to the physical entry with the MMU off; _start switches + to the high mapping before any Rust code runs. */ +PHYS_OFFSET = 0xFFFFFFC000000000; +LOAD_ADDRESS = 0x80200000; +ENTRY(_start_phys) /* Every boundary below is page aligned: the kernel maps each range with its own permissions (W^X), see mmu::init_kernel_page_table. */ SECTIONS { - . = 0x80200000; + . = PHYS_OFFSET + LOAD_ADDRESS; _text_start = .; - .text : { + .text : AT(ADDR(.text) - PHYS_OFFSET) { *(.text.entry) *(.text .text.*) } @@ -15,24 +22,24 @@ SECTIONS { . = ALIGN(4096); _text_end = .; - .rodata : { + .rodata : AT(ADDR(.rodata) - PHYS_OFFSET) { *(.rodata .rodata.*) *(.srodata .srodata.*) } - .eh_frame : { + .eh_frame : AT(ADDR(.eh_frame) - PHYS_OFFSET) { *(.eh_frame) } . = ALIGN(4096); _rodata_end = .; - .data : { + .data : AT(ADDR(.data) - PHYS_OFFSET) { *(.data .data.*) *(.sdata .sdata.*) } . = ALIGN(4096); - .bss : { + .bss : AT(ADDR(.bss) - PHYS_OFFSET) { _bss_start = .; *(.bss .bss.*) *(.sbss .sbss.*) @@ -51,3 +58,5 @@ SECTIONS { _heap_start = .; } + +_start_phys = _start - PHYS_OFFSET; diff --git a/crates/adapter-riscv/src/boot/hardware.rs b/crates/adapter-riscv/src/boot/hardware.rs index ec26b1a..b3f6057 100644 --- a/crates/adapter-riscv/src/boot/hardware.rs +++ b/crates/adapter-riscv/src/boot/hardware.rs @@ -27,6 +27,7 @@ pub fn init(uart: &Uart) -> &'static mut mmu::PageTable { mmu::page_table::WxViolation::WritableCode(va) => ("writable code", va), mmu::page_table::WxViolation::ExecutableData(va) => ("executable data", va), mmu::page_table::WxViolation::MappedHole(va) => ("mapped hole", va), + mmu::page_table::WxViolation::LowerHalf(va) => ("lower-half mapping", va), }; uart.write_str("k-os: W^X violation: "); uart.write_str(what); diff --git a/crates/adapter-riscv/src/boot/loader.rs b/crates/adapter-riscv/src/boot/loader.rs index 5b3fa17..4bb8922 100644 --- a/crates/adapter-riscv/src/boot/loader.rs +++ b/crates/adapter-riscv/src/boot/loader.rs @@ -75,7 +75,7 @@ pub fn load_server( pub fn enter_userspace(pid: ProcessId, entry: usize, shared_buf_va: usize) -> ! { let pt_addr = kern().platform_mut().page_table_for(pid); if pt_addr != 0 { - mmu::activate(unsafe { &*(pt_addr as *const mmu::PageTable) }); + mmu::activate(mmu::page_table::table_at(pt_addr)); } let kernel_sp: usize; @@ -120,11 +120,11 @@ fn copy_to_user(pid: ProcessId, va: usize, data: &[u8]) -> bool { }; let len = (PAGE_SIZE - offset).min(data.len() - done); // SAFETY: `pa` is a pool frame the kernel just mapped for `pid`; - // pool frames are identity mapped kernel RAM. + // pool frames are RAM, mapped at `phys_to_virt`. unsafe { core::ptr::copy_nonoverlapping( data.as_ptr().add(done), - (pa.raw() as *mut u8).add(offset), + (mmu::phys_to_virt(pa.raw()) as *mut u8).add(offset), len, ); } diff --git a/crates/adapter-riscv/src/devices/gpu_setup.rs b/crates/adapter-riscv/src/devices/gpu_setup.rs index a54af1a..604bda0 100644 --- a/crates/adapter-riscv/src/devices/gpu_setup.rs +++ b/crates/adapter-riscv/src/devices/gpu_setup.rs @@ -33,11 +33,12 @@ pub fn configure(uart: &Uart, display_pid: ProcessId) { gpu.init_device(vq_phys); - let queue = - hw_virtio::queue::VirtioQueue::new(hw_virtio::queue::QueueAddrs::from_base(vq_phys)); + let queue = hw_virtio::queue::VirtioQueue::new(hw_virtio::queue::QueueAddrs::from_base( + crate::mmu::phys_to_virt(vq_phys), + )); let dma = hw_virtio::gpu::GpuDmaAddrs { cmd_pa: types::address::PhysicalAddress::from_raw(cmd_phys), - cmd_va: types::address::VirtualAddress::from_raw(cmd_phys), + cmd_va: types::address::VirtualAddress::from_raw(crate::mmu::phys_to_virt(cmd_phys)), resp_pa: types::address::PhysicalAddress::from_raw(cmd_phys + 512), }; diff --git a/crates/adapter-riscv/src/devices/input_setup.rs b/crates/adapter-riscv/src/devices/input_setup.rs index 1ff0b80..39565c2 100644 --- a/crates/adapter-riscv/src/devices/input_setup.rs +++ b/crates/adapter-riscv/src/devices/input_setup.rs @@ -14,8 +14,9 @@ pub fn configure(uart: &Uart, input_pid: ProcessId) { map_mmio(uart, input_pid, UART_VA, board::UART_BASE, 1); // Try to find VirtIO input device - if let Some(mmio) = hw_virtio::mmio::scan_for_device(hw_virtio::mmio::DEVICE_ID_INPUT) { - let base = mmio.base(); + let first = crate::mmu::phys_to_virt(hw_virtio::mmio::MMIO_FIRST); + if let Some(mmio) = hw_virtio::mmio::scan_for_device(first, hw_virtio::mmio::DEVICE_ID_INPUT) { + let base = crate::mmu::virt_to_phys(mmio.base()); map_mmio(uart, input_pid, INPUT_MMIO_VA, base, 1); let vq_phys = dma_buffer(uart, input_pid, INPUT_VQUEUE_VA, 2); let evbuf_phys = dma_buffer(uart, input_pid, INPUT_EVBUF_VA, 1); diff --git a/crates/adapter-riscv/src/devices/mod.rs b/crates/adapter-riscv/src/devices/mod.rs index 06d7701..129a1c6 100644 --- a/crates/adapter-riscv/src/devices/mod.rs +++ b/crates/adapter-riscv/src/devices/mod.rs @@ -82,10 +82,10 @@ pub(crate) fn info_page(uart: &Uart, pid: ProcessId, va: usize, words: &[u64]) { let pa = kern() .user_page_frame(pid, va) .unwrap_or_else(|| crate::halt(uart, "info page vanished")); - let info = pa.raw() as *mut u64; + let info = crate::mmu::phys_to_virt(pa.raw()) as *mut u64; for (i, &w) in words.iter().enumerate() { - // SAFETY: `pa` is a pool frame just mapped for `pid`, identity - // mapped in kernel space, and `words` fits in one page. + // SAFETY: `pa` is a pool frame just mapped for `pid`, reachable at + // `phys_to_virt`, and `words` fits in one page. unsafe { core::ptr::write_volatile(info.add(i), w) }; } } diff --git a/crates/adapter-riscv/src/devices/storage_setup.rs b/crates/adapter-riscv/src/devices/storage_setup.rs index a1a7892..533202a 100644 --- a/crates/adapter-riscv/src/devices/storage_setup.rs +++ b/crates/adapter-riscv/src/devices/storage_setup.rs @@ -31,6 +31,6 @@ pub fn configure(uart: &Uart, init_pid: ProcessId, storage_pid: ProcessId, virti } fn reinit_device(base: usize, vqueue_phys: usize) { - let mmio = hw_virtio::mmio::DirectMmio::new(base); + let mmio = hw_virtio::mmio::DirectMmio::new(crate::mmu::phys_to_virt(base)); hw_virtio::mmio::init_legacy_device(&mmio, vqueue_phys, hw_virtio::queue::QUEUE_SIZE as u32); } diff --git a/crates/adapter-riscv/src/hw/plic.rs b/crates/adapter-riscv/src/hw/plic.rs index 308d115..11f67b5 100644 --- a/crates/adapter-riscv/src/hw/plic.rs +++ b/crates/adapter-riscv/src/hw/plic.rs @@ -1,9 +1,10 @@ use crate::board; -const PLIC_PRIORITY: usize = board::PLIC_BASE; -const PLIC_S_ENABLE: usize = board::PLIC_BASE + 0x2080; -const PLIC_S_THRESHOLD: usize = board::PLIC_BASE + 0x201000; -const PLIC_S_CLAIM: usize = board::PLIC_BASE + 0x201004; +const PLIC: usize = crate::mmu::phys_to_virt(board::PLIC_BASE); +const PLIC_PRIORITY: usize = PLIC; +const PLIC_S_ENABLE: usize = PLIC + 0x2080; +const PLIC_S_THRESHOLD: usize = PLIC + 0x201000; +const PLIC_S_CLAIM: usize = PLIC + 0x201004; pub struct Plic; diff --git a/crates/adapter-riscv/src/hw/timer.rs b/crates/adapter-riscv/src/hw/timer.rs index 7726e71..df4d908 100644 --- a/crates/adapter-riscv/src/hw/timer.rs +++ b/crates/adapter-riscv/src/hw/timer.rs @@ -7,10 +7,13 @@ pub fn read_time() -> u64 { } fn sbi_set_timer(stime: u64) { + // SBI returns an error code in a0 and a value in a1; both are + // clobbered even though we ignore them. unsafe { core::arch::asm!( "ecall", - in("a0") stime, + inlateout("a0") stime => _, + lateout("a1") _, in("a6") 0usize, in("a7") 0x54494D45usize, ); diff --git a/crates/adapter-riscv/src/hw/uart.rs b/crates/adapter-riscv/src/hw/uart.rs index ee222f8..83daf48 100644 --- a/crates/adapter-riscv/src/hw/uart.rs +++ b/crates/adapter-riscv/src/hw/uart.rs @@ -7,7 +7,7 @@ pub struct Uart { impl Uart { pub fn new() -> Self { Self { - base: board::UART_BASE, + base: crate::mmu::phys_to_virt(board::UART_BASE), } } diff --git a/crates/adapter-riscv/src/hw/virtio_blk.rs b/crates/adapter-riscv/src/hw/virtio_blk.rs index 47fb3cb..51b8492 100644 --- a/crates/adapter-riscv/src/hw/virtio_blk.rs +++ b/crates/adapter-riscv/src/hw/virtio_blk.rs @@ -1,3 +1,4 @@ +use crate::mmu::{phys_to_virt, virt_to_phys}; use hw_virtio::block::{self, DmaAddrs}; use hw_virtio::mmio::{self, DirectMmio}; use hw_virtio::queue::{QUEUE_SIZE, QueueAddrs, VirtioQueue}; @@ -27,27 +28,38 @@ pub struct VirtioBlk { impl VirtioBlk { pub fn probe() -> Option { - let mmio = mmio::scan_for_device(mmio::DEVICE_ID_BLOCK)?; + let mmio = mmio::scan_for_device(phys_to_virt(mmio::MMIO_FIRST), mmio::DEVICE_ID_BLOCK)?; - let vq_phys = core::ptr::addr_of!(VQUEUE) as usize; - mmio::init_legacy_device(&mmio, vq_phys, QUEUE_SIZE as u32); + // The device sees physical addresses; the CPU uses kernel ones. + let vq_va = core::ptr::addr_of!(VQUEUE) as usize; + mmio::init_legacy_device(&mmio, virt_to_phys(vq_va), QUEUE_SIZE as u32); - let queue = VirtioQueue::new(QueueAddrs::from_base(vq_phys)); + let queue = VirtioQueue::new(QueueAddrs::from_base(vq_va)); + let pair = |va: usize| { + ( + PhysicalAddress::from_raw(virt_to_phys(va)), + VirtualAddress::from_raw(va), + ) + }; + let (header_pa, header_va) = pair(core::ptr::addr_of_mut!(REQ_HEADER) as usize); + let (status_pa, status_va) = pair(core::ptr::addr_of_mut!(REQ_STATUS) as usize); + let (data_pa, data_va) = pair(core::ptr::addr_of_mut!(SECTOR_BUF) as usize); let dma = DmaAddrs { - header_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(REQ_HEADER) as usize), - header_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(REQ_HEADER) as usize), - status_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(REQ_STATUS) as usize), - status_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(REQ_STATUS) as usize), - data_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(SECTOR_BUF) as usize), - data_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(SECTOR_BUF) as usize), + header_pa, + header_va, + status_pa, + status_va, + data_pa, + data_va, }; Some(Self { mmio, queue, dma }) } + /// Physical address of the device's MMIO window. pub fn base(&self) -> usize { - self.mmio.base() + virt_to_phys(self.mmio.base()) } pub fn read_sector(&mut self, sector: u32, buf: &mut [u8; SECTOR_SIZE]) -> bool { diff --git a/crates/adapter-riscv/src/hw/virtio_gpu.rs b/crates/adapter-riscv/src/hw/virtio_gpu.rs index c59d242..feb02e5 100644 --- a/crates/adapter-riscv/src/hw/virtio_gpu.rs +++ b/crates/adapter-riscv/src/hw/virtio_gpu.rs @@ -1,3 +1,4 @@ +use crate::mmu::{phys_to_virt, virt_to_phys}; use hw_virtio::gpu::{self, GpuDmaAddrs}; use hw_virtio::mmio::{self, DirectMmio}; use hw_virtio::queue::{QUEUE_SIZE, QueueAddrs, VirtioQueue}; @@ -10,12 +11,13 @@ pub struct VirtioGpu { impl VirtioGpu { pub fn probe() -> Option { - let mmio = mmio::scan_for_device(mmio::DEVICE_ID_GPU)?; + let mmio = mmio::scan_for_device(phys_to_virt(mmio::MMIO_FIRST), mmio::DEVICE_ID_GPU)?; Some(Self { mmio }) } + /// Physical address of the device's MMIO window. pub fn base(&self) -> usize { - self.mmio.base() + virt_to_phys(self.mmio.base()) } pub fn mmio(&self) -> &DirectMmio { @@ -34,10 +36,10 @@ impl VirtioGpu { width: Dimension, height: Dimension, ) -> bool { - let queue = VirtioQueue::new(QueueAddrs::from_base(vq_phys)); + let queue = VirtioQueue::new(QueueAddrs::from_base(phys_to_virt(vq_phys))); let dma = GpuDmaAddrs { cmd_pa: PhysicalAddress::from_raw(cmd_phys), - cmd_va: VirtualAddress::from_raw(cmd_phys), + cmd_va: VirtualAddress::from_raw(phys_to_virt(cmd_phys)), resp_pa: PhysicalAddress::from_raw(cmd_phys + 512), }; let ok1 = gpu::create_2d(&self.mmio, &queue, &dma, 1, width, height); diff --git a/crates/adapter-riscv/src/main.rs b/crates/adapter-riscv/src/main.rs index 706adfd..1f92f94 100644 --- a/crates/adapter-riscv/src/main.rs +++ b/crates/adapter-riscv/src/main.rs @@ -27,16 +27,52 @@ pub fn kernel_ptr() -> *mut kernel::Kernel { KERNEL.as_ptr() } +// Firmware enters `_start` at the physical load address with the MMU off, +// but the kernel is linked in the upper half. Using only PC-relative code, +// the stub turns on `boot_page_table`, which maps RAM twice (at its +// physical address, just so the next instruction still exists, and in the +// upper half) plus the MMIO window high, then jumps to its own high address. +// kmain replaces this table with the real one, which maps nothing low. global_asm!( r#" .section .text.entry .globl _start _start: - la sp, _stack_top + lla t0, boot_page_table + srli t0, t0, 12 + li t1, 8 << 60 + or t0, t0, t1 + csrw satp, t0 + sfence.vma + ld t0, start_high_address + jr t0 +start_high: + lla sp, _stack_top call kmain spin: wfi j spin + +.pushsection .rodata +.balign 8 +start_high_address: + .quad start_high +.popsection + +# Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with +# flags V|R|W|A|D = 0xc7, plus X = 0xcf. +.pushsection .data +.balign 4096 +boot_page_table: + .quad 0 + .quad 0 + .quad 0x200000cf # [2] 0x8000_0000 -> RAM, for the jump only + .fill 253, 8, 0 + .quad 0x000000c7 # [256] PHYS_OFFSET + 0 -> MMIO + .quad 0 + .quad 0x200000cf # [258] PHYS_OFFSET + 0x8000_0000 -> RAM + .fill 253, 8, 0 +.popsection "# ); diff --git a/crates/adapter-riscv/src/mmu/allocator.rs b/crates/adapter-riscv/src/mmu/allocator.rs index 8622e47..024e401 100644 --- a/crates/adapter-riscv/src/mmu/allocator.rs +++ b/crates/adapter-riscv/src/mmu/allocator.rs @@ -15,7 +15,7 @@ pub fn init_allocator(start: usize) { pub fn alloc_zeroed_page() -> usize { unsafe { let addr = NEXT_FREE; - if addr + PAGE_SIZE > board::USER_POOL_BASE { + if addr + PAGE_SIZE > super::phys_to_virt(board::USER_POOL_BASE) { crate::halt(&crate::hw::uart::Uart::new(), "boot allocator exhausted"); } NEXT_FREE += PAGE_SIZE; diff --git a/crates/adapter-riscv/src/mmu/mod.rs b/crates/adapter-riscv/src/mmu/mod.rs index 0791863..a2ba3bf 100644 --- a/crates/adapter-riscv/src/mmu/mod.rs +++ b/crates/adapter-riscv/src/mmu/mod.rs @@ -6,3 +6,20 @@ pub use page_table::{ PAGE_SIZE, PageTable, Sv39Flags, USER_VA_END, USER_VA_START, activate, active_root, check_wx, create_user_root, init_kernel_page_table, map_user_page, unmap_user_page, }; + +/// The kernel lives in the upper half of the address space: all of RAM and +/// the MMIO windows appear at their physical address plus this offset (the +/// start of Sv39's upper half), and the kernel image is linked there too, +/// so one offset converts any kernel address. Must match linker.ld. +pub const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000; + +/// Where the kernel sees physical address `pa`. +pub const fn phys_to_virt(pa: usize) -> usize { + pa + PHYS_OFFSET +} + +/// The physical address behind kernel address `va` (a static, the boot +/// heap, a page table). Only valid for addresses in the direct map. +pub const fn virt_to_phys(va: usize) -> usize { + va - PHYS_OFFSET +} diff --git a/crates/adapter-riscv/src/mmu/page_table.rs b/crates/adapter-riscv/src/mmu/page_table.rs index 42962a5..3a3302b 100644 --- a/crates/adapter-riscv/src/mmu/page_table.rs +++ b/crates/adapter-riscv/src/mmu/page_table.rs @@ -1,4 +1,5 @@ use super::allocator::alloc_page_table; +use super::{phys_to_virt, virt_to_phys}; use crate::board; use platform::{FrameSource, MapError}; use types::address::{PhysicalAddress, VirtualAddress}; @@ -103,27 +104,24 @@ fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) { let vpn0 = vpn(va, 0); if !root.entries[vpn2].is_valid() { - let table = alloc_page_table(); - let ppn = (table as *const PageTable as u64) >> 12; - root.entries[vpn2] = PageTableEntry::new_branch(ppn); + root.entries[vpn2] = PageTableEntry::new_branch(table_ppn(alloc_page_table())); } - - let l1_addr = (root.entries[vpn2].ppn() << 12) as *mut PageTable; - let l1 = unsafe { &mut *l1_addr }; + let l1 = table_at((root.entries[vpn2].ppn() << 12) as usize); if !l1.entries[vpn1].is_valid() { - let table = alloc_page_table(); - let ppn = (table as *const PageTable as u64) >> 12; - l1.entries[vpn1] = PageTableEntry::new_branch(ppn); + l1.entries[vpn1] = PageTableEntry::new_branch(table_ppn(alloc_page_table())); } - - let l0_addr = (l1.entries[vpn1].ppn() << 12) as *mut PageTable; - let l0 = unsafe { &mut *l0_addr }; + let l0 = table_at((l1.entries[vpn1].ppn() << 12) as usize); let ppn = (pa as u64) >> 12; l0.entries[vpn0] = PageTableEntry::new_leaf(ppn, flags); } +/// The physical page number of a page table the kernel holds a reference to. +fn table_ppn(table: &PageTable) -> u64 { + (virt_to_phys(table as *const PageTable as usize) as u64) >> 12 +} + pub fn map(root: &mut PageTable, mapping: Mapping) { for i in 0..mapping.pages { let offset = i * PAGE_SIZE; @@ -139,19 +137,21 @@ pub fn map(root: &mut PageTable, mapping: Mapping) { pub fn init_kernel_page_table() -> &'static mut PageTable { let root = alloc_page_table(); - let identity = |addr: usize| { + // Physical `pa` at its place in the upper-half direct map. + let direct = |pa: usize| { ( - VirtualAddress::from_raw(addr), - PhysicalAddress::from_raw(addr), + VirtualAddress::from_raw(phys_to_virt(pa)), + PhysicalAddress::from_raw(pa), ) }; - // The kernel image, then everything after it, each with only the - // access it needs: code is never writable, data never executable. - // RAM below the image belongs to the firmware and stays unmapped, as - // does the guard page under the kernel stack. + // The kernel image, then the rest of RAM, each with only the access it + // needs: code is never writable, data never executable. RAM below the + // image belongs to the firmware and stays unmapped, as does the guard + // page under the kernel stack. Nothing is mapped in the lower half, + // which is userspace's alone. let layout = KernelLayout::from_linker(); - let ram_end = board::RAM_BASE + board::RAM_SIZE; + let ram_end = phys_to_virt(board::RAM_BASE + board::RAM_SIZE); for (start, end, flags) in [ (layout.text_start, layout.text_end, Sv39Flags::kernel_rx()), (layout.text_end, layout.rodata_end, Sv39Flags::kernel_ro()), @@ -162,7 +162,7 @@ pub fn init_kernel_page_table() -> &'static mut PageTable { ), (layout.stack_bottom, ram_end, Sv39Flags::kernel_rw()), ] { - let (va, pa) = identity(start); + let (va, pa) = direct(virt_to_phys(start)); map( root, Mapping { @@ -174,49 +174,23 @@ pub fn init_kernel_page_table() -> &'static mut PageTable { ); } - let (va, pa) = identity(board::UART_BASE); - map( - root, - Mapping { - va, - pa, - pages: 1, - flags: Sv39Flags::kernel_rw(), - }, - ); - - let (va, pa) = identity(board::PLIC_BASE); - map( - root, - Mapping { - va, - pa, - pages: 0x400000 / PAGE_SIZE, - flags: Sv39Flags::kernel_rw(), - }, - ); - - let (va, pa) = identity(board::CLINT_BASE); - map( - root, - Mapping { - va, - pa, - pages: 0x10000 / PAGE_SIZE, - flags: Sv39Flags::kernel_rw(), - }, - ); - - let (va, pa) = identity(board::VIRTIO_BASE); - map( - root, - Mapping { - va, - pa, - pages: board::VIRTIO_SIZE / PAGE_SIZE, - flags: Sv39Flags::kernel_rw(), - }, - ); + for (base, size) in [ + (board::UART_BASE, PAGE_SIZE), + (board::PLIC_BASE, 0x40_0000), + (board::CLINT_BASE, 0x1_0000), + (board::VIRTIO_BASE, board::VIRTIO_SIZE), + ] { + let (va, pa) = direct(base); + map( + root, + Mapping { + va, + pa, + pages: size / PAGE_SIZE, + flags: Sv39Flags::kernel_rw(), + }, + ); + } root } @@ -260,19 +234,24 @@ pub enum WxViolation { ExecutableData(usize), /// A mapped page under the firmware, or the stack guard. MappedHole(usize), + /// A kernel mapping in the lower half, which belongs to userspace. + LowerHalf(usize), } -/// Walks every leaf of `root` and checks the W^X layout: nothing is both -/// writable and executable, only kernel text is executable, and the -/// firmware and the stack guard are unmapped. Returns the number of -/// executable pages. +/// Walks every leaf of `root` and checks the kernel layout: nothing is +/// both writable and executable, only kernel text is executable, the +/// firmware and the stack guard are unmapped, and nothing at all is mapped +/// in the lower half. Returns the number of executable pages. pub fn check_wx(root: &PageTable) -> Result { let layout = KernelLayout::from_linker(); let text = layout.text_start..layout.text_end; let guard = layout.stack_guard..layout.stack_bottom; - let firmware = board::RAM_BASE..layout.text_start; + let firmware = phys_to_virt(board::RAM_BASE)..layout.text_start; let mut executable = 0; for (va, bits) in leaves(root) { + if va < phys_to_virt(0) { + return Err(WxViolation::LowerHalf(va)); + } if bits & PTE_W != 0 && bits & PTE_X != 0 { return Err(WxViolation::WritableCode(va)); } @@ -311,6 +290,8 @@ fn leaves(root: &PageTable) -> impl Iterator + '_ { .filter(|(_, e)| e.is_valid()) .map(move |(i0, e0)| { let va = (i2 << 30) | (i1 << 21) | (i0 << 12); + // Sv39 addresses are sign extended from bit 38. + let va = if i2 >= 256 { va | !((1 << 39) - 1) } else { va }; (va, e0.0 & 0x3ff) }) }) @@ -318,8 +299,7 @@ fn leaves(root: &PageTable) -> impl Iterator + '_ { } pub fn activate(root: &PageTable) { - let ppn = (root as *const PageTable as u64) >> 12; - let satp = (8u64 << 60) | ppn; + let satp = (8u64 << 60) | table_ppn(root); unsafe { core::arch::asm!( "csrw satp, {0}", @@ -332,18 +312,19 @@ pub fn activate(root: &PageTable) { // ---- user address spaces ---- // // User mappings live in [USER_VA_START, USER_VA_END), which is root slot 0, -// L1 slots 0..USER_L1_SLOTS. Every address space gets its own root and its -// own L1 for slot 0, and the kernel maps nothing in those L1 slots, so all -// page tables below the user window are private to one process. Kernel -// mappings elsewhere are shared by copying the kernel's entries. +// the L1 slots below USER_VA_END. Every address space gets its own root and its +// own L1 for slot 0, and the kernel maps nothing in the lower half, so all +// page tables below the user window are private to one process. The +// kernel's upper-half entries are shared by copying them. pub const USER_VA_START: usize = 0x1000; pub const USER_VA_END: usize = 0x0200_0000; -const USER_L1_SLOTS: usize = USER_VA_END >> 21; -fn table_at(pa: usize) -> &'static mut PageTable { - // SAFETY: page tables are identity mapped in kernel space. - unsafe { &mut *(pa as *mut PageTable) } +/// The page table at physical address `pa`, through the direct map. +pub fn table_at(pa: usize) -> &'static mut PageTable { + // SAFETY: every page table lives in RAM, which the kernel maps at + // `phys_to_virt`. + unsafe { &mut *(phys_to_virt(pa) as *mut PageTable) } } fn fresh_table(frames: &mut dyn FrameSource) -> Result { @@ -368,18 +349,9 @@ pub fn create_user_root( let root_pa = fresh_table(frames)?; let l1_pa = fresh_table(frames)?; let root = table_at(root_pa); - let l1 = table_at(l1_pa); - for (i, entry) in kernel_root.entries.iter().enumerate() { - if i != 0 && entry.is_valid() { - root.entries[i] = *entry; - } - } - if kernel_root.entries[0].is_valid() { - let kernel_l1 = table_at((kernel_root.entries[0].ppn() << 12) as usize); - for j in USER_L1_SLOTS..512 { - l1.entries[j] = kernel_l1.entries[j]; - } + for (mine, kernels) in root.entries.iter_mut().zip(&kernel_root.entries).skip(256) { + *mine = *kernels; } root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12); Ok(root_pa) diff --git a/crates/adapter-riscv/src/platform.rs b/crates/adapter-riscv/src/platform.rs index 1357b42..d212382 100644 --- a/crates/adapter-riscv/src/platform.rs +++ b/crates/adapter-riscv/src/platform.rs @@ -53,7 +53,7 @@ impl Platform for RiscVPlatform { .page_tables .get_mut(process.raw() as usize) .ok_or(MapError::NoAddressSpace)?; - // SAFETY: the kernel page table is identity mapped and lives forever. + // SAFETY: the kernel page table lives in the boot heap forever. let kernel_root = unsafe { &*(self.kernel_pt as *const mmu::PageTable) }; *slot = mmu::create_user_root(kernel_root, frames)?; Ok(()) @@ -66,7 +66,7 @@ impl Platform for RiscVPlatform { // The caller may be tearing down the space we are running on; get // off it before the kernel frees its frames. if mmu::active_root() == root { - // SAFETY: the kernel page table is identity mapped and lives forever. + // SAFETY: the kernel page table lives in the boot heap forever. mmu::activate(unsafe { &*(self.kernel_pt as *const mmu::PageTable) }); } self.page_tables[process.raw() as usize] = 0; @@ -99,9 +99,11 @@ impl Platform for RiscVPlatform { } fn zero_frame(&mut self, pa: PhysicalAddress) { - // SAFETY: pool frames are identity mapped kernel RAM and the kernel - // only zeroes frames it has just allocated. - unsafe { core::ptr::write_bytes(pa.raw() as *mut u8, 0, mmu::PAGE_SIZE) }; + // SAFETY: pool frames are RAM, mapped at `phys_to_virt`, and the + // kernel only zeroes frames it has just allocated. + unsafe { + core::ptr::write_bytes(mmu::phys_to_virt(pa.raw()) as *mut u8, 0, mmu::PAGE_SIZE) + }; } fn enable_user_mode(&mut self, entry: EntryPoint, stack: StackPointer) -> ! { @@ -160,8 +162,8 @@ impl Platform for RiscVPlatform { unsafe { core::arch::asm!( "ecall", - in("a0") 0usize, - in("a1") 0usize, + inlateout("a0") 0usize => _, + inlateout("a1") 0usize => _, in("a7") 0x53525354usize, in("a6") 0usize, ); diff --git a/crates/adapter-riscv/src/trap/mod.rs b/crates/adapter-riscv/src/trap/mod.rs index 24df8a3..0983818 100644 --- a/crates/adapter-riscv/src/trap/mod.rs +++ b/crates/adapter-riscv/src/trap/mod.rs @@ -9,6 +9,7 @@ pub struct TrapFrame { global_asm!( r#" +.section .text .align 4 .globl trap_vector trap_vector: @@ -207,7 +208,7 @@ fn switch_to_thread( { let pt_addr = kern.platform_mut().page_table_for(pid); if pt_addr != 0 { - let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) }; + let pt = crate::mmu::page_table::table_at(pt_addr); crate::mmu::activate(pt); } } @@ -239,7 +240,7 @@ fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) { { let pt_addr = kern.platform_mut().page_table_for(pid); if pt_addr != 0 { - let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) }; + let pt = crate::mmu::page_table::table_at(pt_addr); crate::mmu::activate(pt); } } diff --git a/crates/hw-virtio/src/mmio.rs b/crates/hw-virtio/src/mmio.rs index 9143689..a08f6cd 100644 --- a/crates/hw-virtio/src/mmio.rs +++ b/crates/hw-virtio/src/mmio.rs @@ -87,9 +87,11 @@ pub const MMIO_FIRST: usize = 0x10001000; pub const MMIO_STRIDE: usize = 0x1000; pub const MMIO_SLOTS: usize = 8; -pub fn scan_for_device(device_id: u32) -> Option { +/// Probes the MMIO slots starting at `first`, the address the caller sees +/// slot 0 at (`MMIO_FIRST` itself, or wherever it is mapped). +pub fn scan_for_device(first: usize, device_id: u32) -> Option { for slot in 0..MMIO_SLOTS { - if let Some(mmio) = probe_slot(MMIO_FIRST + slot * MMIO_STRIDE, device_id) { + if let Some(mmio) = probe_slot(first + slot * MMIO_STRIDE, device_id) { return Some(mmio); } } diff --git a/docs/ADAPTERS.md b/docs/ADAPTERS.md index f8c14ba..93f4aad 100644 --- a/docs/ADAPTERS.md +++ b/docs/ADAPTERS.md @@ -14,6 +14,11 @@ This generates `crates/adapter-/` with the standard structure. Then: 2. Fill in `board.rs` with device addresses for the target board 3. Add boot assembly (`global_asm!` in `main.rs` or a `.S` file) 4. Update `linker.ld` for the target's entry address and memory layout. + The kernel runs in the upper half: link it at its physical load + address plus `mmu::PHYS_OFFSET` (with `AT()` load addresses and a + physical entry symbol), and have the boot stub enable a temporary + table mapping RAM both low and high before jumping high. Nothing in + the final kernel table may be mapped in the lower half. Keep text, read-only data and writable data on separate pages and export their page-aligned boundaries (`_text_start`, `_text_end`, `_rodata_end`, `_stack_guard`, `_stack_bottom`): the kernel maps each