refactor(kernel): split ipc/memory/lib/thread by concern; trim comments
This commit is contained in:
214
crates/kernel/src/memory/mod.rs
Normal file
214
crates/kernel/src/memory/mod.rs
Normal file
@@ -0,0 +1,214 @@
|
||||
//! Address-space bookkeeping. The kernel decides every user mapping from
|
||||
//! its own records (`FrameTable`, `RegionTable`, quotas) and only then tells
|
||||
//! the platform to apply it; it never asks the hardware what is mapped.
|
||||
|
||||
mod pages;
|
||||
mod region;
|
||||
|
||||
use domain::{
|
||||
error::KernelError,
|
||||
memory::{
|
||||
frame::{FrameTable, FrameUse, PAGE_SIZE},
|
||||
quota::Quota,
|
||||
},
|
||||
};
|
||||
use platform::{FrameSource, MapError, Platform};
|
||||
use types::{
|
||||
address::{PhysicalAddress, VirtualAddress},
|
||||
id::ProcessId,
|
||||
mapping::PagePerms,
|
||||
};
|
||||
|
||||
use crate::Kernel;
|
||||
|
||||
/// First VA the kernel considers when picking a spot for `RequestPages`.
|
||||
/// Below it sit the program image and boot-time device windows.
|
||||
pub(crate) const USER_HEAP_START: usize = 0x40000;
|
||||
|
||||
/// Page-table frames for one address space, charged to its owner.
|
||||
struct ChargedFrames<'a> {
|
||||
frames: &'a mut FrameTable,
|
||||
quota: &'a mut Quota,
|
||||
owner: ProcessId,
|
||||
}
|
||||
|
||||
impl FrameSource for ChargedFrames<'_> {
|
||||
fn alloc_table_frame(&mut self) -> Option<PhysicalAddress> {
|
||||
if !self.quota.charge(1) {
|
||||
return None;
|
||||
}
|
||||
match self.frames.alloc(FrameUse::PageTable { owner: self.owner }) {
|
||||
Some(pa) => Some(pa),
|
||||
None => {
|
||||
self.quota.uncharge(1);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_error(e: MapError) -> KernelError {
|
||||
match e {
|
||||
MapError::OutOfFrames => KernelError::OutOfMemory,
|
||||
MapError::InvalidAddress => KernelError::InvalidAddress,
|
||||
MapError::NoAddressSpace => KernelError::OutOfMemory,
|
||||
}
|
||||
}
|
||||
|
||||
impl<P: Platform> Kernel<P> {
|
||||
/// Whether `[va, va + pages)` is page aligned and inside the user window.
|
||||
pub(crate) fn in_user_window(va: VirtualAddress, pages: usize) -> bool {
|
||||
let start = va.raw();
|
||||
let Some(len) = pages.checked_mul(PAGE_SIZE) else {
|
||||
return false;
|
||||
};
|
||||
let Some(end) = start.checked_add(len) else {
|
||||
return false;
|
||||
};
|
||||
pages > 0
|
||||
&& start.is_multiple_of(PAGE_SIZE)
|
||||
&& start >= P::USER_VA_START
|
||||
&& end <= P::USER_VA_END
|
||||
}
|
||||
|
||||
/// `pid`'s user page at `va`: the platform's page table says where it
|
||||
/// points, the frame table confirms the frame really is that page.
|
||||
fn user_frame(
|
||||
&self,
|
||||
pid: ProcessId,
|
||||
va: VirtualAddress,
|
||||
) -> Option<(PhysicalAddress, PagePerms)> {
|
||||
let pa = self.platform.translate(pid, va)?;
|
||||
self.frames.user_at(pa, pid, va).map(|perms| (pa, perms))
|
||||
}
|
||||
|
||||
fn is_mapped(&self, pid: ProcessId, va: VirtualAddress) -> bool {
|
||||
self.user_frame(pid, va).is_some() || self.regions.covering(pid, va).is_some()
|
||||
}
|
||||
|
||||
fn range_is_free(&self, pid: ProcessId, va: VirtualAddress, pages: usize) -> bool {
|
||||
(0..pages).all(|i| !self.is_mapped(pid, page_at(va, i)))
|
||||
}
|
||||
|
||||
fn quota_mut(&mut self, pid: ProcessId) -> Result<&mut Quota, KernelError> {
|
||||
self.quotas
|
||||
.get_mut(pid.raw() as usize)
|
||||
.ok_or(KernelError::InvalidProcess(pid))
|
||||
}
|
||||
|
||||
pub fn page_usage(&self, pid: ProcessId) -> usize {
|
||||
self.quotas.get(pid.raw() as usize).map_or(0, Quota::used)
|
||||
}
|
||||
|
||||
pub fn set_page_limit(&mut self, pid: ProcessId, limit: usize) -> Result<(), KernelError> {
|
||||
self.processes.get(pid)?;
|
||||
if self.quota_mut(pid)?.set_limit(limit) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(KernelError::OutOfMemory)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn free_page_count(&self) -> usize {
|
||||
self.frames.free_count()
|
||||
}
|
||||
|
||||
/// Maps `pa` at `va` in `pid`, building page tables on `pid`'s quota.
|
||||
fn platform_map(
|
||||
&mut self,
|
||||
pid: ProcessId,
|
||||
va: VirtualAddress,
|
||||
pa: PhysicalAddress,
|
||||
perms: PagePerms,
|
||||
) -> Result<(), KernelError> {
|
||||
let quota = self
|
||||
.quotas
|
||||
.get_mut(pid.raw() as usize)
|
||||
.ok_or(KernelError::InvalidProcess(pid))?;
|
||||
let mut source = ChargedFrames {
|
||||
frames: &mut self.frames,
|
||||
quota,
|
||||
owner: pid,
|
||||
};
|
||||
self.platform
|
||||
.map_page(pid, va, pa, perms, &mut source)
|
||||
.map_err(map_error)
|
||||
}
|
||||
|
||||
pub(crate) fn create_address_space(&mut self, pid: ProcessId) -> Result<(), KernelError> {
|
||||
let quota = self
|
||||
.quotas
|
||||
.get_mut(pid.raw() as usize)
|
||||
.ok_or(KernelError::InvalidProcess(pid))?;
|
||||
*quota = Quota::default();
|
||||
let mut source = ChargedFrames {
|
||||
frames: &mut self.frames,
|
||||
quota,
|
||||
owner: pid,
|
||||
};
|
||||
let created = self
|
||||
.platform
|
||||
.create_address_space(pid, &mut source)
|
||||
.map_err(map_error);
|
||||
if created.is_err() {
|
||||
self.frames.free_owned_by(pid);
|
||||
if let Some(q) = self.quotas.get_mut(pid.raw() as usize) {
|
||||
*q = Quota::default();
|
||||
}
|
||||
}
|
||||
created
|
||||
}
|
||||
|
||||
/// Allocates, zeroes and maps one fresh page for `pid` at `va`.
|
||||
fn map_fresh_page(
|
||||
&mut self,
|
||||
pid: ProcessId,
|
||||
va: VirtualAddress,
|
||||
perms: PagePerms,
|
||||
) -> Result<PhysicalAddress, KernelError> {
|
||||
let usage = FrameUse::user(pid, va, perms).ok_or(KernelError::InvalidAddress)?;
|
||||
if !self.quota_mut(pid)?.charge(1) {
|
||||
return Err(KernelError::OutOfMemory);
|
||||
}
|
||||
let Some(pa) = self.frames.alloc(usage) else {
|
||||
self.quota_mut(pid)?.uncharge(1);
|
||||
return Err(KernelError::OutOfMemory);
|
||||
};
|
||||
self.platform.zero_frame(pa);
|
||||
if let Err(e) = self.platform_map(pid, va, pa, perms) {
|
||||
self.frames.free(pa);
|
||||
self.quota_mut(pid)?.uncharge(1);
|
||||
return Err(e);
|
||||
}
|
||||
Ok(pa)
|
||||
}
|
||||
|
||||
/// Removes one user page of `pid` and returns its frame to the pool.
|
||||
fn unmap_and_free(&mut self, pid: ProcessId, va: VirtualAddress) {
|
||||
if let Some((pa, _)) = self.user_frame(pid, va) {
|
||||
self.platform.unmap_page(pid, va);
|
||||
self.frames.free(pa);
|
||||
if let Some(q) = self.quotas.get_mut(pid.raw() as usize) {
|
||||
q.uncharge(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Releases everything `pid` holds in memory: its address space, page
|
||||
/// tables, user pages and owned regions.
|
||||
pub(crate) fn teardown_memory(&mut self, pid: ProcessId) {
|
||||
self.teardown_regions(pid);
|
||||
self.platform.destroy_address_space(pid);
|
||||
self.frames.free_owned_by(pid);
|
||||
if let Some(q) = self.quotas.get_mut(pid.raw() as usize) {
|
||||
*q = Quota::default();
|
||||
}
|
||||
if let Some(c) = self.heap_cursors.get_mut(pid.raw() as usize) {
|
||||
*c = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn page_at(base: VirtualAddress, i: usize) -> VirtualAddress {
|
||||
VirtualAddress::from_raw(base.raw() + i * PAGE_SIZE)
|
||||
}
|
||||
138
crates/kernel/src/memory/pages.rs
Normal file
138
crates/kernel/src/memory/pages.rs
Normal file
@@ -0,0 +1,138 @@
|
||||
//! Page moves, release and heap placement.
|
||||
|
||||
use domain::{
|
||||
error::KernelError,
|
||||
memory::frame::{FrameUse, PAGE_SIZE},
|
||||
};
|
||||
use platform::Platform;
|
||||
use types::{
|
||||
address::{PhysicalAddress, VirtualAddress},
|
||||
id::ProcessId,
|
||||
mapping::PagePerms,
|
||||
};
|
||||
|
||||
use super::{USER_HEAP_START, page_at};
|
||||
use crate::Kernel;
|
||||
|
||||
impl<P: Platform> Kernel<P> {
|
||||
/// Frame backing the user page at `va` in `pid`. Lets boot code fill
|
||||
/// pages it has just mapped for a process.
|
||||
pub fn user_page_frame(&self, pid: ProcessId, va: VirtualAddress) -> Option<PhysicalAddress> {
|
||||
self.user_frame(pid, va).map(|(pa, _)| pa)
|
||||
}
|
||||
|
||||
/// Maps `pages` fresh zeroed pages at `va`, all or nothing.
|
||||
pub fn map_fresh_range(
|
||||
&mut self,
|
||||
pid: ProcessId,
|
||||
va: VirtualAddress,
|
||||
pages: usize,
|
||||
perms: PagePerms,
|
||||
) -> Result<(), KernelError> {
|
||||
if !Self::in_user_window(va, pages) {
|
||||
return Err(KernelError::InvalidAddress);
|
||||
}
|
||||
if !self.range_is_free(pid, va, pages) {
|
||||
return Err(KernelError::AddressInUse);
|
||||
}
|
||||
for i in 0..pages {
|
||||
if let Err(e) = self.map_fresh_page(pid, page_at(va, i), perms) {
|
||||
for j in 0..i {
|
||||
self.unmap_and_free(pid, page_at(va, j));
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Next-fit search for `pages` free heap pages from `pid`'s last request, then from the bottom.
|
||||
pub(crate) fn find_free_heap_range(
|
||||
&self,
|
||||
pid: ProcessId,
|
||||
pages: usize,
|
||||
) -> Option<VirtualAddress> {
|
||||
let bottom = USER_HEAP_START.max(P::USER_VA_START);
|
||||
let cursor = self
|
||||
.heap_cursors
|
||||
.get(pid.raw() as usize)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
.max(bottom);
|
||||
self.first_fit_from(pid, cursor, pages)
|
||||
.or_else(|| self.first_fit_from(pid, bottom, pages))
|
||||
}
|
||||
|
||||
/// Records where `pid`'s next heap search starts.
|
||||
pub(crate) fn advance_heap_cursor(&mut self, pid: ProcessId, past: VirtualAddress) {
|
||||
if let Some(c) = self.heap_cursors.get_mut(pid.raw() as usize) {
|
||||
*c = past.raw();
|
||||
}
|
||||
}
|
||||
|
||||
fn first_fit_from(&self, pid: ProcessId, from: usize, pages: usize) -> Option<VirtualAddress> {
|
||||
let mut candidate = from;
|
||||
while Self::in_user_window(VirtualAddress::from_raw(candidate), pages) {
|
||||
let va = VirtualAddress::from_raw(candidate);
|
||||
match (0..pages).find(|&i| self.is_mapped(pid, page_at(va, i))) {
|
||||
None => return Some(va),
|
||||
Some(busy) => candidate += (busy + 1) * PAGE_SIZE,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Validates that every page of the range is a user page `pid` owns,
|
||||
/// then releases them all.
|
||||
pub(crate) fn release_range(
|
||||
&mut self,
|
||||
pid: ProcessId,
|
||||
va: VirtualAddress,
|
||||
pages: usize,
|
||||
) -> Result<(), KernelError> {
|
||||
if !Self::in_user_window(va, pages) {
|
||||
return Err(KernelError::InvalidAddress);
|
||||
}
|
||||
if !(0..pages).all(|i| self.user_frame(pid, page_at(va, i)).is_some()) {
|
||||
return Err(KernelError::InvalidAddress);
|
||||
}
|
||||
for i in 0..pages {
|
||||
self.unmap_and_free(pid, page_at(va, i));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Moves the page at `src` in `from` to `dst` in `to`. The frame and its
|
||||
/// quota charge change hands; `from` loses access.
|
||||
pub(crate) fn move_page(
|
||||
&mut self,
|
||||
from: ProcessId,
|
||||
src: VirtualAddress,
|
||||
to: ProcessId,
|
||||
dst: VirtualAddress,
|
||||
perms: PagePerms,
|
||||
) -> Result<(), KernelError> {
|
||||
if !Self::in_user_window(src, 1) || !Self::in_user_window(dst, 1) {
|
||||
return Err(KernelError::InvalidAddress);
|
||||
}
|
||||
let (pa, _) = self
|
||||
.user_frame(from, src)
|
||||
.ok_or(KernelError::InvalidAddress)?;
|
||||
self.processes.get(to)?;
|
||||
if !self.range_is_free(to, dst, 1) {
|
||||
return Err(KernelError::AddressInUse);
|
||||
}
|
||||
let usage = FrameUse::user(to, dst, perms).ok_or(KernelError::InvalidAddress)?;
|
||||
if !self.quota_mut(to)?.charge(1) {
|
||||
return Err(KernelError::OutOfMemory);
|
||||
}
|
||||
if let Err(e) = self.platform_map(to, dst, pa, perms) {
|
||||
self.quota_mut(to)?.uncharge(1);
|
||||
return Err(e);
|
||||
}
|
||||
self.platform.unmap_page(from, src);
|
||||
self.frames.retag(pa, usage);
|
||||
self.quota_mut(from)?.uncharge(1);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
201
crates/kernel/src/memory/region.rs
Normal file
201
crates/kernel/src/memory/region.rs
Normal file
@@ -0,0 +1,201 @@
|
||||
//! Shared and device regions and their mappings.
|
||||
|
||||
use domain::{
|
||||
error::KernelError,
|
||||
memory::{
|
||||
Region, RegionBacking, RegionMapping,
|
||||
frame::{FrameUse, PAGE_SIZE},
|
||||
},
|
||||
};
|
||||
use platform::Platform;
|
||||
use types::{
|
||||
address::{PhysicalAddress, VirtualAddress},
|
||||
bounded::PageCount,
|
||||
id::{ProcessId, RegionId},
|
||||
mapping::PagePerms,
|
||||
};
|
||||
|
||||
use super::page_at;
|
||||
use crate::Kernel;
|
||||
|
||||
impl<P: Platform> Kernel<P> {
|
||||
/// Allocates a physically contiguous, zeroed region charged to `owner`.
|
||||
pub fn create_pool_region(
|
||||
&mut self,
|
||||
owner: ProcessId,
|
||||
pages: PageCount,
|
||||
) -> Result<RegionId, KernelError> {
|
||||
self.processes.get(owner)?;
|
||||
let id = self.regions.next_free_id().ok_or(KernelError::NoFreeSlot)?;
|
||||
let count = pages.value() as usize;
|
||||
if !self.quota_mut(owner)?.charge(count) {
|
||||
return Err(KernelError::OutOfMemory);
|
||||
}
|
||||
let Some(base) = self
|
||||
.frames
|
||||
.alloc_contiguous(count, |_| FrameUse::Region { region: id })
|
||||
else {
|
||||
self.quota_mut(owner)?.uncharge(count);
|
||||
return Err(KernelError::OutOfMemory);
|
||||
};
|
||||
for i in 0..count {
|
||||
self.platform.zero_frame(base.offset(i));
|
||||
}
|
||||
self.regions
|
||||
.insert(Region::new(owner, RegionBacking::Pool, base, pages))
|
||||
.ok_or(KernelError::NoFreeSlot)
|
||||
}
|
||||
|
||||
/// Describes device memory at `base` so it can be mapped like a region.
|
||||
pub fn create_device_region(
|
||||
&mut self,
|
||||
owner: ProcessId,
|
||||
base: PhysicalAddress,
|
||||
pages: PageCount,
|
||||
) -> Result<RegionId, KernelError> {
|
||||
self.processes.get(owner)?;
|
||||
if !base.raw().is_multiple_of(PAGE_SIZE) {
|
||||
return Err(KernelError::InvalidAddress);
|
||||
}
|
||||
// Device memory must never alias frames the kernel hands out.
|
||||
let len = pages.value() as usize * PAGE_SIZE;
|
||||
let start = base.raw();
|
||||
let end = start.checked_add(len).ok_or(KernelError::InvalidAddress)?;
|
||||
let pool_start = self.frames.base().raw();
|
||||
let pool_end = pool_start + self.frames.capacity() * PAGE_SIZE;
|
||||
if start < pool_end && end > pool_start {
|
||||
return Err(KernelError::InvalidAddress);
|
||||
}
|
||||
// Nor another device's window: each device has one region, so
|
||||
// revoking or delegating it means the same thing everywhere.
|
||||
let overlaps = self.regions.iter().any(|(_, r)| {
|
||||
let r_start = r.base().raw();
|
||||
let r_end = r_start.saturating_add(r.pages().value() as usize * PAGE_SIZE);
|
||||
r.backing() == RegionBacking::Device && start < r_end && end > r_start
|
||||
});
|
||||
if overlaps {
|
||||
return Err(KernelError::AddressInUse);
|
||||
}
|
||||
self.regions
|
||||
.insert(Region::new(owner, RegionBacking::Device, base, pages))
|
||||
.ok_or(KernelError::NoFreeSlot)
|
||||
}
|
||||
|
||||
pub fn region_base(&self, id: RegionId) -> Option<PhysicalAddress> {
|
||||
self.regions.get(id).map(Region::base)
|
||||
}
|
||||
|
||||
pub fn region_pages(&self, id: RegionId) -> Option<PageCount> {
|
||||
self.regions.get(id).map(Region::pages)
|
||||
}
|
||||
|
||||
/// Maps region `id` for boot code, all or nothing; not tied to any capability.
|
||||
pub fn map_region(
|
||||
&mut self,
|
||||
id: RegionId,
|
||||
pid: ProcessId,
|
||||
va: VirtualAddress,
|
||||
perms: PagePerms,
|
||||
) -> Result<(), KernelError> {
|
||||
self.map_region_via(id, pid, va, perms, None)
|
||||
}
|
||||
|
||||
/// Maps region `id`, all or nothing; a mapping made `via` a capability dies with it.
|
||||
pub(crate) fn map_region_via(
|
||||
&mut self,
|
||||
id: RegionId,
|
||||
pid: ProcessId,
|
||||
va: VirtualAddress,
|
||||
perms: PagePerms,
|
||||
via: Option<domain::capability::CapRef>,
|
||||
) -> Result<(), KernelError> {
|
||||
self.processes.get(pid)?;
|
||||
let region = self.regions.get(id).ok_or(KernelError::InvalidRegion)?;
|
||||
let (base, pages) = (region.base(), region.pages().value() as usize);
|
||||
if region.mappings().any(|m| m.process == pid) {
|
||||
return Err(KernelError::AddressInUse);
|
||||
}
|
||||
// W^X holds across processes too: nobody may write what someone
|
||||
// else can execute.
|
||||
let clashes = |other: PagePerms| {
|
||||
matches!(
|
||||
(perms, other),
|
||||
(PagePerms::ReadWrite, PagePerms::ReadExecute)
|
||||
| (PagePerms::ReadExecute, PagePerms::ReadWrite)
|
||||
)
|
||||
};
|
||||
if region.mappings().any(|m| clashes(m.perms)) {
|
||||
return Err(KernelError::WritableAndExecutable);
|
||||
}
|
||||
if !Self::in_user_window(va, pages) {
|
||||
return Err(KernelError::InvalidAddress);
|
||||
}
|
||||
if !self.range_is_free(pid, va, pages) {
|
||||
return Err(KernelError::AddressInUse);
|
||||
}
|
||||
for i in 0..pages {
|
||||
if let Err(e) = self.platform_map(pid, page_at(va, i), base.offset(i), perms) {
|
||||
for j in 0..i {
|
||||
self.platform.unmap_page(pid, page_at(va, j));
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
let recorded = self.regions.get_mut(id).is_some_and(|r| {
|
||||
r.add_mapping(RegionMapping {
|
||||
process: pid,
|
||||
va,
|
||||
perms,
|
||||
via,
|
||||
})
|
||||
});
|
||||
if !recorded {
|
||||
for i in 0..pages {
|
||||
self.platform.unmap_page(pid, page_at(va, i));
|
||||
}
|
||||
return Err(KernelError::NoFreeSlot);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Removes `pid`'s mapping of region `id`, if it has one.
|
||||
pub(crate) fn unmap_region_from(&mut self, id: RegionId, pid: ProcessId) {
|
||||
let Some(region) = self.regions.get_mut(id) else {
|
||||
return;
|
||||
};
|
||||
let pages = region.pages().value() as usize;
|
||||
if let Some(m) = region.remove_mapping(pid) {
|
||||
for i in 0..pages {
|
||||
self.platform.unmap_page(pid, page_at(m.va, i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Frees regions owned by `pid` (unmapping them from every other
|
||||
/// process) and forgets `pid`'s mappings of other regions.
|
||||
pub(super) fn teardown_regions(&mut self, pid: ProcessId) {
|
||||
let mut owned = [None; domain::memory::MAX_REGIONS];
|
||||
for (slot, id) in owned.iter_mut().zip(self.regions.ids_owned_by(pid)) {
|
||||
*slot = Some(id);
|
||||
}
|
||||
for id in owned.into_iter().flatten() {
|
||||
let Some(region) = self.regions.remove(id) else {
|
||||
continue;
|
||||
};
|
||||
let pages = region.pages().value() as usize;
|
||||
for m in region.mappings().filter(|m| m.process != pid) {
|
||||
for i in 0..pages {
|
||||
self.platform.unmap_page(m.process, page_at(m.va, i));
|
||||
}
|
||||
}
|
||||
if region.backing() == RegionBacking::Pool {
|
||||
for i in 0..pages {
|
||||
self.frames.free(region.base().offset(i));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (_, region) in self.regions.iter_mut() {
|
||||
region.remove_mapping(pid);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user