feat(adapter): higher-half kernel (linked at PHYS_OFFSET, boot trampoline, no low mappings), fix SBI ecall clobbers

crates/adapter-riscv/linker.ld

@@ -1,13 +1,20 @@
 OUTPUT_ARCH(riscv)
-ENTRY(_start)
+
+/* The kernel runs in the upper half: linked at its physical load address
+   plus PHYS_OFFSET (mmu::PHYS_OFFSET), loaded at the physical address.
+   Firmware jumps to the physical entry with the MMU off; _start switches
+   to the high mapping before any Rust code runs. */
+PHYS_OFFSET = 0xFFFFFFC000000000;
+LOAD_ADDRESS = 0x80200000;
+ENTRY(_start_phys)
 
 /* Every boundary below is page aligned: the kernel maps each range with
    its own permissions (W^X), see mmu::init_kernel_page_table. */
 SECTIONS {
-    . = 0x80200000;
+    . = PHYS_OFFSET + LOAD_ADDRESS;
     _text_start = .;
 
-    .text : {
+    .text : AT(ADDR(.text) - PHYS_OFFSET) {
         *(.text.entry)
         *(.text .text.*)
     }
@@ -15,24 +22,24 @@ SECTIONS {
     . = ALIGN(4096);
     _text_end = .;
 
-    .rodata : {
+    .rodata : AT(ADDR(.rodata) - PHYS_OFFSET) {
         *(.rodata .rodata.*)
         *(.srodata .srodata.*)
     }
-    .eh_frame : {
+    .eh_frame : AT(ADDR(.eh_frame) - PHYS_OFFSET) {
         *(.eh_frame)
     }
 
     . = ALIGN(4096);
     _rodata_end = .;
 
-    .data : {
+    .data : AT(ADDR(.data) - PHYS_OFFSET) {
         *(.data .data.*)
         *(.sdata .sdata.*)
     }
 
     . = ALIGN(4096);
-    .bss : {
+    .bss : AT(ADDR(.bss) - PHYS_OFFSET) {
         _bss_start = .;
         *(.bss .bss.*)
         *(.sbss .sbss.*)
@@ -51,3 +58,5 @@ SECTIONS {
 
     _heap_start = .;
 }
+
+_start_phys = _start - PHYS_OFFSET;

crates/adapter-riscv/src/boot/hardware.rs

@@ -27,6 +27,7 @@ pub fn init(uart: &Uart) -> &'static mut mmu::PageTable {
                 mmu::page_table::WxViolation::WritableCode(va) => ("writable code", va),
                 mmu::page_table::WxViolation::ExecutableData(va) => ("executable data", va),
                 mmu::page_table::WxViolation::MappedHole(va) => ("mapped hole", va),
+                mmu::page_table::WxViolation::LowerHalf(va) => ("lower-half mapping", va),
             };
             uart.write_str("k-os: W^X violation: ");
             uart.write_str(what);

crates/adapter-riscv/src/boot/loader.rs

@@ -75,7 +75,7 @@ pub fn load_server(
 pub fn enter_userspace(pid: ProcessId, entry: usize, shared_buf_va: usize) -> ! {
     let pt_addr = kern().platform_mut().page_table_for(pid);
     if pt_addr != 0 {
-        mmu::activate(unsafe { &*(pt_addr as *const mmu::PageTable) });
+        mmu::activate(mmu::page_table::table_at(pt_addr));
     }
 
     let kernel_sp: usize;
@@ -120,11 +120,11 @@ fn copy_to_user(pid: ProcessId, va: usize, data: &[u8]) -> bool {
         };
         let len = (PAGE_SIZE - offset).min(data.len() - done);
         // SAFETY: `pa` is a pool frame the kernel just mapped for `pid`;
-        // pool frames are identity mapped kernel RAM.
+        // pool frames are RAM, mapped at `phys_to_virt`.
         unsafe {
             core::ptr::copy_nonoverlapping(
                 data.as_ptr().add(done),
-                (pa.raw() as *mut u8).add(offset),
+                (mmu::phys_to_virt(pa.raw()) as *mut u8).add(offset),
                 len,
             );
         }

crates/adapter-riscv/src/devices/gpu_setup.rs

@@ -33,11 +33,12 @@ pub fn configure(uart: &Uart, display_pid: ProcessId) {
 
     gpu.init_device(vq_phys);
 
-    let queue =
+    let queue = hw_virtio::queue::VirtioQueue::new(hw_virtio::queue::QueueAddrs::from_base(
-        hw_virtio::queue::VirtioQueue::new(hw_virtio::queue::QueueAddrs::from_base(vq_phys));
+        crate::mmu::phys_to_virt(vq_phys),
+    ));
     let dma = hw_virtio::gpu::GpuDmaAddrs {
         cmd_pa: types::address::PhysicalAddress::from_raw(cmd_phys),
-        cmd_va: types::address::VirtualAddress::from_raw(cmd_phys),
+        cmd_va: types::address::VirtualAddress::from_raw(crate::mmu::phys_to_virt(cmd_phys)),
         resp_pa: types::address::PhysicalAddress::from_raw(cmd_phys + 512),
     };
 

crates/adapter-riscv/src/devices/input_setup.rs

@@ -14,8 +14,9 @@ pub fn configure(uart: &Uart, input_pid: ProcessId) {
     map_mmio(uart, input_pid, UART_VA, board::UART_BASE, 1);
 
     // Try to find VirtIO input device
-    if let Some(mmio) = hw_virtio::mmio::scan_for_device(hw_virtio::mmio::DEVICE_ID_INPUT) {
+    let first = crate::mmu::phys_to_virt(hw_virtio::mmio::MMIO_FIRST);
-        let base = mmio.base();
+    if let Some(mmio) = hw_virtio::mmio::scan_for_device(first, hw_virtio::mmio::DEVICE_ID_INPUT) {
+        let base = crate::mmu::virt_to_phys(mmio.base());
         map_mmio(uart, input_pid, INPUT_MMIO_VA, base, 1);
         let vq_phys = dma_buffer(uart, input_pid, INPUT_VQUEUE_VA, 2);
         let evbuf_phys = dma_buffer(uart, input_pid, INPUT_EVBUF_VA, 1);

crates/adapter-riscv/src/devices/mod.rs

@@ -82,10 +82,10 @@ pub(crate) fn info_page(uart: &Uart, pid: ProcessId, va: usize, words: &[u64]) {
     let pa = kern()
         .user_page_frame(pid, va)
         .unwrap_or_else(|| crate::halt(uart, "info page vanished"));
-    let info = pa.raw() as *mut u64;
+    let info = crate::mmu::phys_to_virt(pa.raw()) as *mut u64;
     for (i, &w) in words.iter().enumerate() {
-        // SAFETY: `pa` is a pool frame just mapped for `pid`, identity
+        // SAFETY: `pa` is a pool frame just mapped for `pid`, reachable at
-        // mapped in kernel space, and `words` fits in one page.
+        // `phys_to_virt`, and `words` fits in one page.
         unsafe { core::ptr::write_volatile(info.add(i), w) };
     }
 }

crates/adapter-riscv/src/devices/storage_setup.rs

@@ -31,6 +31,6 @@ pub fn configure(uart: &Uart, init_pid: ProcessId, storage_pid: ProcessId, virti
 }
 
 fn reinit_device(base: usize, vqueue_phys: usize) {
-    let mmio = hw_virtio::mmio::DirectMmio::new(base);
+    let mmio = hw_virtio::mmio::DirectMmio::new(crate::mmu::phys_to_virt(base));
     hw_virtio::mmio::init_legacy_device(&mmio, vqueue_phys, hw_virtio::queue::QUEUE_SIZE as u32);
 }

crates/adapter-riscv/src/hw/plic.rs

@@ -1,9 +1,10 @@
 use crate::board;
 
-const PLIC_PRIORITY: usize = board::PLIC_BASE;
+const PLIC: usize = crate::mmu::phys_to_virt(board::PLIC_BASE);
-const PLIC_S_ENABLE: usize = board::PLIC_BASE + 0x2080;
+const PLIC_PRIORITY: usize = PLIC;
-const PLIC_S_THRESHOLD: usize = board::PLIC_BASE + 0x201000;
+const PLIC_S_ENABLE: usize = PLIC + 0x2080;
-const PLIC_S_CLAIM: usize = board::PLIC_BASE + 0x201004;
+const PLIC_S_THRESHOLD: usize = PLIC + 0x201000;
+const PLIC_S_CLAIM: usize = PLIC + 0x201004;
 
 pub struct Plic;
 

crates/adapter-riscv/src/hw/timer.rs

@@ -7,10 +7,13 @@ pub fn read_time() -> u64 {
 }
 
 fn sbi_set_timer(stime: u64) {
+    // SBI returns an error code in a0 and a value in a1; both are
+    // clobbered even though we ignore them.
     unsafe {
         core::arch::asm!(
             "ecall",
-            in("a0") stime,
+            inlateout("a0") stime => _,
+            lateout("a1") _,
             in("a6") 0usize,
             in("a7") 0x54494D45usize,
         );

crates/adapter-riscv/src/hw/uart.rs

@@ -7,7 +7,7 @@ pub struct Uart {
 impl Uart {
     pub fn new() -> Self {
         Self {
-            base: board::UART_BASE,
+            base: crate::mmu::phys_to_virt(board::UART_BASE),
         }
     }
 

crates/adapter-riscv/src/hw/virtio_blk.rs

@@ -1,3 +1,4 @@
+use crate::mmu::{phys_to_virt, virt_to_phys};
 use hw_virtio::block::{self, DmaAddrs};
 use hw_virtio::mmio::{self, DirectMmio};
 use hw_virtio::queue::{QUEUE_SIZE, QueueAddrs, VirtioQueue};
@@ -27,27 +28,38 @@ pub struct VirtioBlk {
 
 impl VirtioBlk {
     pub fn probe() -> Option<Self> {
-        let mmio = mmio::scan_for_device(mmio::DEVICE_ID_BLOCK)?;
+        let mmio = mmio::scan_for_device(phys_to_virt(mmio::MMIO_FIRST), mmio::DEVICE_ID_BLOCK)?;
 
-        let vq_phys = core::ptr::addr_of!(VQUEUE) as usize;
+        // The device sees physical addresses; the CPU uses kernel ones.
-        mmio::init_legacy_device(&mmio, vq_phys, QUEUE_SIZE as u32);
+        let vq_va = core::ptr::addr_of!(VQUEUE) as usize;
+        mmio::init_legacy_device(&mmio, virt_to_phys(vq_va), QUEUE_SIZE as u32);
 
-        let queue = VirtioQueue::new(QueueAddrs::from_base(vq_phys));
+        let queue = VirtioQueue::new(QueueAddrs::from_base(vq_va));
 
+        let pair = |va: usize| {
+            (
+                PhysicalAddress::from_raw(virt_to_phys(va)),
+                VirtualAddress::from_raw(va),
+            )
+        };
+        let (header_pa, header_va) = pair(core::ptr::addr_of_mut!(REQ_HEADER) as usize);
+        let (status_pa, status_va) = pair(core::ptr::addr_of_mut!(REQ_STATUS) as usize);
+        let (data_pa, data_va) = pair(core::ptr::addr_of_mut!(SECTOR_BUF) as usize);
         let dma = DmaAddrs {
-            header_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(REQ_HEADER) as usize),
+            header_pa,
-            header_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(REQ_HEADER) as usize),
+            header_va,
-            status_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(REQ_STATUS) as usize),
+            status_pa,
-            status_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(REQ_STATUS) as usize),
+            status_va,
-            data_pa: PhysicalAddress::from_raw(core::ptr::addr_of!(SECTOR_BUF) as usize),
+            data_pa,
-            data_va: VirtualAddress::from_raw(core::ptr::addr_of_mut!(SECTOR_BUF) as usize),
+            data_va,
         };
 
         Some(Self { mmio, queue, dma })
     }
 
+    /// Physical address of the device's MMIO window.
     pub fn base(&self) -> usize {
-        self.mmio.base()
+        virt_to_phys(self.mmio.base())
     }
 
     pub fn read_sector(&mut self, sector: u32, buf: &mut [u8; SECTOR_SIZE]) -> bool {

crates/adapter-riscv/src/hw/virtio_gpu.rs

@@ -1,3 +1,4 @@
+use crate::mmu::{phys_to_virt, virt_to_phys};
 use hw_virtio::gpu::{self, GpuDmaAddrs};
 use hw_virtio::mmio::{self, DirectMmio};
 use hw_virtio::queue::{QUEUE_SIZE, QueueAddrs, VirtioQueue};
@@ -10,12 +11,13 @@ pub struct VirtioGpu {
 
 impl VirtioGpu {
     pub fn probe() -> Option<Self> {
-        let mmio = mmio::scan_for_device(mmio::DEVICE_ID_GPU)?;
+        let mmio = mmio::scan_for_device(phys_to_virt(mmio::MMIO_FIRST), mmio::DEVICE_ID_GPU)?;
         Some(Self { mmio })
     }
 
+    /// Physical address of the device's MMIO window.
     pub fn base(&self) -> usize {
-        self.mmio.base()
+        virt_to_phys(self.mmio.base())
     }
 
     pub fn mmio(&self) -> &DirectMmio {
@@ -34,10 +36,10 @@ impl VirtioGpu {
         width: Dimension,
         height: Dimension,
     ) -> bool {
-        let queue = VirtioQueue::new(QueueAddrs::from_base(vq_phys));
+        let queue = VirtioQueue::new(QueueAddrs::from_base(phys_to_virt(vq_phys)));
         let dma = GpuDmaAddrs {
             cmd_pa: PhysicalAddress::from_raw(cmd_phys),
-            cmd_va: VirtualAddress::from_raw(cmd_phys),
+            cmd_va: VirtualAddress::from_raw(phys_to_virt(cmd_phys)),
             resp_pa: PhysicalAddress::from_raw(cmd_phys + 512),
         };
         let ok1 = gpu::create_2d(&self.mmio, &queue, &dma, 1, width, height);

crates/adapter-riscv/src/main.rs

@@ -27,16 +27,52 @@ pub fn kernel_ptr() -> *mut kernel::Kernel<platform::RiscVPlatform> {
     KERNEL.as_ptr()
 }
 
+// Firmware enters `_start` at the physical load address with the MMU off,
+// but the kernel is linked in the upper half. Using only PC-relative code,
+// the stub turns on `boot_page_table`, which maps RAM twice (at its
+// physical address, just so the next instruction still exists, and in the
+// upper half) plus the MMIO window high, then jumps to its own high address.
+// kmain replaces this table with the real one, which maps nothing low.
 global_asm!(
     r#"
 .section .text.entry
 .globl _start
 _start:
-    la sp, _stack_top
+    lla t0, boot_page_table
+    srli t0, t0, 12
+    li t1, 8 << 60
+    or t0, t0, t1
+    csrw satp, t0
+    sfence.vma
+    ld t0, start_high_address
+    jr t0
+start_high:
+    lla sp, _stack_top
     call kmain
 spin:
     wfi
     j spin
+
+.pushsection .rodata
+.balign 8
+start_high_address:
+    .quad start_high
+.popsection
+
+# Sv39 root of 1 GiB leaves; an entry is (PA >> 12) << 10 | flags, with
+# flags V|R|W|A|D = 0xc7, plus X = 0xcf.
+.pushsection .data
+.balign 4096
+boot_page_table:
+    .quad 0
+    .quad 0
+    .quad 0x200000cf    # [2]   0x8000_0000 -> RAM, for the jump only
+    .fill 253, 8, 0
+    .quad 0x000000c7    # [256] PHYS_OFFSET + 0 -> MMIO
+    .quad 0
+    .quad 0x200000cf    # [258] PHYS_OFFSET + 0x8000_0000 -> RAM
+    .fill 253, 8, 0
+.popsection
 "#
 );
 

crates/adapter-riscv/src/mmu/allocator.rs

@@ -15,7 +15,7 @@ pub fn init_allocator(start: usize) {
 pub fn alloc_zeroed_page() -> usize {
     unsafe {
         let addr = NEXT_FREE;
-        if addr + PAGE_SIZE > board::USER_POOL_BASE {
+        if addr + PAGE_SIZE > super::phys_to_virt(board::USER_POOL_BASE) {
             crate::halt(&crate::hw::uart::Uart::new(), "boot allocator exhausted");
         }
         NEXT_FREE += PAGE_SIZE;

crates/adapter-riscv/src/mmu/mod.rs

@@ -6,3 +6,20 @@ pub use page_table::{
     PAGE_SIZE, PageTable, Sv39Flags, USER_VA_END, USER_VA_START, activate, active_root, check_wx,
     create_user_root, init_kernel_page_table, map_user_page, unmap_user_page,
 };
+
+/// The kernel lives in the upper half of the address space: all of RAM and
+/// the MMIO windows appear at their physical address plus this offset (the
+/// start of Sv39's upper half), and the kernel image is linked there too,
+/// so one offset converts any kernel address. Must match linker.ld.
+pub const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000;
+
+/// Where the kernel sees physical address `pa`.
+pub const fn phys_to_virt(pa: usize) -> usize {
+    pa + PHYS_OFFSET
+}
+
+/// The physical address behind kernel address `va` (a static, the boot
+/// heap, a page table). Only valid for addresses in the direct map.
+pub const fn virt_to_phys(va: usize) -> usize {
+    va - PHYS_OFFSET
+}

crates/adapter-riscv/src/mmu/page_table.rs

@@ -1,4 +1,5 @@
 use super::allocator::alloc_page_table;
+use super::{phys_to_virt, virt_to_phys};
 use crate::board;
 use platform::{FrameSource, MapError};
 use types::address::{PhysicalAddress, VirtualAddress};
@@ -103,27 +104,24 @@ fn map_page(root: &mut PageTable, va: usize, pa: usize, flags: Sv39Flags) {
     let vpn0 = vpn(va, 0);
 
     if !root.entries[vpn2].is_valid() {
-        let table = alloc_page_table();
+        root.entries[vpn2] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
-        let ppn = (table as *const PageTable as u64) >> 12;
-        root.entries[vpn2] = PageTableEntry::new_branch(ppn);
     }
-
+    let l1 = table_at((root.entries[vpn2].ppn() << 12) as usize);
-    let l1_addr = (root.entries[vpn2].ppn() << 12) as *mut PageTable;
-    let l1 = unsafe { &mut *l1_addr };
 
     if !l1.entries[vpn1].is_valid() {
-        let table = alloc_page_table();
+        l1.entries[vpn1] = PageTableEntry::new_branch(table_ppn(alloc_page_table()));
-        let ppn = (table as *const PageTable as u64) >> 12;
-        l1.entries[vpn1] = PageTableEntry::new_branch(ppn);
     }
-
+    let l0 = table_at((l1.entries[vpn1].ppn() << 12) as usize);
-    let l0_addr = (l1.entries[vpn1].ppn() << 12) as *mut PageTable;
-    let l0 = unsafe { &mut *l0_addr };
 
     let ppn = (pa as u64) >> 12;
     l0.entries[vpn0] = PageTableEntry::new_leaf(ppn, flags);
 }
 
+/// The physical page number of a page table the kernel holds a reference to.
+fn table_ppn(table: &PageTable) -> u64 {
+    (virt_to_phys(table as *const PageTable as usize) as u64) >> 12
+}
+
 pub fn map(root: &mut PageTable, mapping: Mapping<Sv39Flags>) {
     for i in 0..mapping.pages {
         let offset = i * PAGE_SIZE;
@@ -139,19 +137,21 @@ pub fn map(root: &mut PageTable, mapping: Mapping<Sv39Flags>) {
 pub fn init_kernel_page_table() -> &'static mut PageTable {
     let root = alloc_page_table();
 
-    let identity = |addr: usize| {
+    // Physical `pa` at its place in the upper-half direct map.
+    let direct = |pa: usize| {
         (
-            VirtualAddress::from_raw(addr),
+            VirtualAddress::from_raw(phys_to_virt(pa)),
-            PhysicalAddress::from_raw(addr),
+            PhysicalAddress::from_raw(pa),
         )
     };
 
-    // The kernel image, then everything after it, each with only the
+    // The kernel image, then the rest of RAM, each with only the access it
-    // access it needs: code is never writable, data never executable.
+    // needs: code is never writable, data never executable. RAM below the
-    // RAM below the image belongs to the firmware and stays unmapped, as
+    // image belongs to the firmware and stays unmapped, as does the guard
-    // does the guard page under the kernel stack.
+    // page under the kernel stack. Nothing is mapped in the lower half,
+    // which is userspace's alone.
     let layout = KernelLayout::from_linker();
-    let ram_end = board::RAM_BASE + board::RAM_SIZE;
+    let ram_end = phys_to_virt(board::RAM_BASE + board::RAM_SIZE);
     for (start, end, flags) in [
         (layout.text_start, layout.text_end, Sv39Flags::kernel_rx()),
         (layout.text_end, layout.rodata_end, Sv39Flags::kernel_ro()),
@@ -162,7 +162,7 @@ pub fn init_kernel_page_table() -> &'static mut PageTable {
         ),
         (layout.stack_bottom, ram_end, Sv39Flags::kernel_rw()),
     ] {
-        let (va, pa) = identity(start);
+        let (va, pa) = direct(virt_to_phys(start));
         map(
             root,
             Mapping {
@@ -174,49 +174,23 @@ pub fn init_kernel_page_table() -> &'static mut PageTable {
         );
     }
 
-    let (va, pa) = identity(board::UART_BASE);
+    for (base, size) in [
-    map(
+        (board::UART_BASE, PAGE_SIZE),
-        root,
+        (board::PLIC_BASE, 0x40_0000),
-        Mapping {
+        (board::CLINT_BASE, 0x1_0000),
-            va,
+        (board::VIRTIO_BASE, board::VIRTIO_SIZE),
-            pa,
+    ] {
-            pages: 1,
+        let (va, pa) = direct(base);
-            flags: Sv39Flags::kernel_rw(),
+        map(
-        },
+            root,
-    );
+            Mapping {
-
+                va,
-    let (va, pa) = identity(board::PLIC_BASE);
+                pa,
-    map(
+                pages: size / PAGE_SIZE,
-        root,
+                flags: Sv39Flags::kernel_rw(),
-        Mapping {
+            },
-            va,
+        );
-            pa,
+    }
-            pages: 0x400000 / PAGE_SIZE,
-            flags: Sv39Flags::kernel_rw(),
-        },
-    );
-
-    let (va, pa) = identity(board::CLINT_BASE);
-    map(
-        root,
-        Mapping {
-            va,
-            pa,
-            pages: 0x10000 / PAGE_SIZE,
-            flags: Sv39Flags::kernel_rw(),
-        },
-    );
-
-    let (va, pa) = identity(board::VIRTIO_BASE);
-    map(
-        root,
-        Mapping {
-            va,
-            pa,
-            pages: board::VIRTIO_SIZE / PAGE_SIZE,
-            flags: Sv39Flags::kernel_rw(),
-        },
-    );
 
     root
 }
@@ -260,19 +234,24 @@ pub enum WxViolation {
     ExecutableData(usize),
     /// A mapped page under the firmware, or the stack guard.
     MappedHole(usize),
+    /// A kernel mapping in the lower half, which belongs to userspace.
+    LowerHalf(usize),
 }
 
-/// Walks every leaf of `root` and checks the W^X layout: nothing is both
+/// Walks every leaf of `root` and checks the kernel layout: nothing is
-/// writable and executable, only kernel text is executable, and the
+/// both writable and executable, only kernel text is executable, the
-/// firmware and the stack guard are unmapped. Returns the number of
+/// firmware and the stack guard are unmapped, and nothing at all is mapped
-/// executable pages.
+/// in the lower half. Returns the number of executable pages.
 pub fn check_wx(root: &PageTable) -> Result<usize, WxViolation> {
     let layout = KernelLayout::from_linker();
     let text = layout.text_start..layout.text_end;
     let guard = layout.stack_guard..layout.stack_bottom;
-    let firmware = board::RAM_BASE..layout.text_start;
+    let firmware = phys_to_virt(board::RAM_BASE)..layout.text_start;
     let mut executable = 0;
     for (va, bits) in leaves(root) {
+        if va < phys_to_virt(0) {
+            return Err(WxViolation::LowerHalf(va));
+        }
         if bits & PTE_W != 0 && bits & PTE_X != 0 {
             return Err(WxViolation::WritableCode(va));
         }
@@ -311,6 +290,8 @@ fn leaves(root: &PageTable) -> impl Iterator<Item = (usize, u64)> + '_ {
                         .filter(|(_, e)| e.is_valid())
                         .map(move |(i0, e0)| {
                             let va = (i2 << 30) | (i1 << 21) | (i0 << 12);
+                            // Sv39 addresses are sign extended from bit 38.
+                            let va = if i2 >= 256 { va | !((1 << 39) - 1) } else { va };
                             (va, e0.0 & 0x3ff)
                         })
                 })
@@ -318,8 +299,7 @@ fn leaves(root: &PageTable) -> impl Iterator<Item = (usize, u64)> + '_ {
 }
 
 pub fn activate(root: &PageTable) {
-    let ppn = (root as *const PageTable as u64) >> 12;
+    let satp = (8u64 << 60) | table_ppn(root);
-    let satp = (8u64 << 60) | ppn;
     unsafe {
         core::arch::asm!(
             "csrw satp, {0}",
@@ -332,18 +312,19 @@ pub fn activate(root: &PageTable) {
 // ---- user address spaces ----
 //
 // User mappings live in [USER_VA_START, USER_VA_END), which is root slot 0,
-// L1 slots 0..USER_L1_SLOTS. Every address space gets its own root and its
+// the L1 slots below USER_VA_END. Every address space gets its own root and its
-// own L1 for slot 0, and the kernel maps nothing in those L1 slots, so all
+// own L1 for slot 0, and the kernel maps nothing in the lower half, so all
-// page tables below the user window are private to one process. Kernel
+// page tables below the user window are private to one process. The
-// mappings elsewhere are shared by copying the kernel's entries.
+// kernel's upper-half entries are shared by copying them.
 
 pub const USER_VA_START: usize = 0x1000;
 pub const USER_VA_END: usize = 0x0200_0000;
-const USER_L1_SLOTS: usize = USER_VA_END >> 21;
 
-fn table_at(pa: usize) -> &'static mut PageTable {
+/// The page table at physical address `pa`, through the direct map.
-    // SAFETY: page tables are identity mapped in kernel space.
+pub fn table_at(pa: usize) -> &'static mut PageTable {
-    unsafe { &mut *(pa as *mut PageTable) }
+    // SAFETY: every page table lives in RAM, which the kernel maps at
+    // `phys_to_virt`.
+    unsafe { &mut *(phys_to_virt(pa) as *mut PageTable) }
 }
 
 fn fresh_table(frames: &mut dyn FrameSource) -> Result<usize, MapError> {
@@ -368,18 +349,9 @@ pub fn create_user_root(
     let root_pa = fresh_table(frames)?;
     let l1_pa = fresh_table(frames)?;
     let root = table_at(root_pa);
-    let l1 = table_at(l1_pa);
 
-    for (i, entry) in kernel_root.entries.iter().enumerate() {
+    for (mine, kernels) in root.entries.iter_mut().zip(&kernel_root.entries).skip(256) {
-        if i != 0 && entry.is_valid() {
+        *mine = *kernels;
-            root.entries[i] = *entry;
-        }
-    }
-    if kernel_root.entries[0].is_valid() {
-        let kernel_l1 = table_at((kernel_root.entries[0].ppn() << 12) as usize);
-        for j in USER_L1_SLOTS..512 {
-            l1.entries[j] = kernel_l1.entries[j];
-        }
     }
     root.entries[0] = PageTableEntry::new_branch((l1_pa as u64) >> 12);
     Ok(root_pa)

crates/adapter-riscv/src/platform.rs

@@ -53,7 +53,7 @@ impl Platform for RiscVPlatform {
             .page_tables
             .get_mut(process.raw() as usize)
             .ok_or(MapError::NoAddressSpace)?;
-        // SAFETY: the kernel page table is identity mapped and lives forever.
+        // SAFETY: the kernel page table lives in the boot heap forever.
         let kernel_root = unsafe { &*(self.kernel_pt as *const mmu::PageTable) };
         *slot = mmu::create_user_root(kernel_root, frames)?;
         Ok(())
@@ -66,7 +66,7 @@ impl Platform for RiscVPlatform {
         // The caller may be tearing down the space we are running on; get
         // off it before the kernel frees its frames.
         if mmu::active_root() == root {
-            // SAFETY: the kernel page table is identity mapped and lives forever.
+            // SAFETY: the kernel page table lives in the boot heap forever.
             mmu::activate(unsafe { &*(self.kernel_pt as *const mmu::PageTable) });
         }
         self.page_tables[process.raw() as usize] = 0;
@@ -99,9 +99,11 @@ impl Platform for RiscVPlatform {
     }
 
     fn zero_frame(&mut self, pa: PhysicalAddress) {
-        // SAFETY: pool frames are identity mapped kernel RAM and the kernel
+        // SAFETY: pool frames are RAM, mapped at `phys_to_virt`, and the
-        // only zeroes frames it has just allocated.
+        // kernel only zeroes frames it has just allocated.
-        unsafe { core::ptr::write_bytes(pa.raw() as *mut u8, 0, mmu::PAGE_SIZE) };
+        unsafe {
+            core::ptr::write_bytes(mmu::phys_to_virt(pa.raw()) as *mut u8, 0, mmu::PAGE_SIZE)
+        };
     }
 
     fn enable_user_mode(&mut self, entry: EntryPoint, stack: StackPointer) -> ! {
@@ -160,8 +162,8 @@ impl Platform for RiscVPlatform {
         unsafe {
             core::arch::asm!(
                 "ecall",
-                in("a0") 0usize,
+                inlateout("a0") 0usize => _,
-                in("a1") 0usize,
+                inlateout("a1") 0usize => _,
                 in("a7") 0x53525354usize,
                 in("a6") 0usize,
             );

crates/adapter-riscv/src/trap/mod.rs

@@ -9,6 +9,7 @@ pub struct TrapFrame {
 
 global_asm!(
     r#"
+.section .text
 .align 4
 .globl trap_vector
 trap_vector:
@@ -207,7 +208,7 @@ fn switch_to_thread(
     {
         let pt_addr = kern.platform_mut().page_table_for(pid);
         if pt_addr != 0 {
-            let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) };
+            let pt = crate::mmu::page_table::table_at(pt_addr);
             crate::mmu::activate(pt);
         }
     }
@@ -239,7 +240,7 @@ fn switch_to_next(frame: &mut TrapFrame, current_tid: types::id::ThreadId) {
     {
         let pt_addr = kern.platform_mut().page_table_for(pid);
         if pt_addr != 0 {
-            let pt = unsafe { &*(pt_addr as *const crate::mmu::PageTable) };
+            let pt = crate::mmu::page_table::table_at(pt_addr);
             crate::mmu::activate(pt);
         }
     }

crates/hw-virtio/src/mmio.rs

@@ -87,9 +87,11 @@ pub const MMIO_FIRST: usize = 0x10001000;
 pub const MMIO_STRIDE: usize = 0x1000;
 pub const MMIO_SLOTS: usize = 8;
 
-pub fn scan_for_device(device_id: u32) -> Option<DirectMmio> {
+/// Probes the MMIO slots starting at `first`, the address the caller sees
+/// slot 0 at (`MMIO_FIRST` itself, or wherever it is mapped).
+pub fn scan_for_device(first: usize, device_id: u32) -> Option<DirectMmio> {
     for slot in 0..MMIO_SLOTS {
-        if let Some(mmio) = probe_slot(MMIO_FIRST + slot * MMIO_STRIDE, device_id) {
+        if let Some(mmio) = probe_slot(first + slot * MMIO_STRIDE, device_id) {
             return Some(mmio);
         }
     }

docs/ADAPTERS.md

@@ -14,6 +14,11 @@ This generates `crates/adapter-<arch>/` with the standard structure. Then:
 2. Fill in `board.rs` with device addresses for the target board
 3. Add boot assembly (`global_asm!` in `main.rs` or a `.S` file)
 4. Update `linker.ld` for the target's entry address and memory layout.
+   The kernel runs in the upper half: link it at its physical load
+   address plus `mmu::PHYS_OFFSET` (with `AT()` load addresses and a
+   physical entry symbol), and have the boot stub enable a temporary
+   table mapping RAM both low and high before jumping high. Nothing in
+   the final kernel table may be mapped in the lower half.
    Keep text, read-only data and writable data on separate pages and
    export their page-aligned boundaries (`_text_start`, `_text_end`,
    `_rodata_end`, `_stack_guard`, `_stack_bottom`): the kernel maps each