use crate::cap::cnode; use crate::cap::object::{ EndpointData, FramebufferData, IrqHandlerData, IrqSource, NotificationData, ObjectData, ObjectTag, PciDeviceData, PortRange, ProcessObjectData, }; const COM1_PORTS: PortRange = match PortRange::new(0x3F8, 8) { Some(pr) => pr, None => unreachable!(), }; const PS2_KBD_PORTS: PortRange = match PortRange::new(0x60, 5) { Some(pr) => pr, None => unreachable!(), }; use crate::cap::pool::POOL; use crate::cap::table::{CapRef, Rights}; use crate::error::KernelError; use crate::types::Pid; const SLOT_SELF: u64 = 0; mod init_slots { pub const SERIAL_EP: u64 = 1; pub const FB_EP: u64 = 2; pub const INIT_NOTIF: u64 = 9; pub const NETSTACK_NOTIF: u64 = 10; pub const VFS_PROC: u64 = 15; pub const NETSTACK_RING_BASE: u64 = 288; pub const NETSTACK_RING_FRAMES: u16 = 2; } mod irq_driver_slots { pub const EP: u64 = 1; pub const IRQ: u64 = 2; pub const NOTIF: u64 = 3; } mod fbconsole_slots { pub const EP: u64 = 1; pub const FB: u64 = 2; pub const CONSOLE_RING_BASE: u64 = 64; } pub fn bootstrap_root_cnode(pid: Pid, size_bits: u8) -> Result<(), KernelError> { let allocator = &crate::mem::phys::BitmapFrameAllocator; let cnode_data = cnode::create_cnode(size_bits, allocator)?; let frame_count = cnode_data.frame_count; let (cnode_id, cnode_gen) = POOL.lock().allocate(ObjectData::CNode(cnode_data))?; let mut ptable = crate::proc::PROCESSES.lock(); let proc = ptable .get_mut(pid) .ok_or(KernelError::InvalidObject)?; proc.root_cnode = Some((cnode_id, cnode_gen)); proc.cnode_depth = size_bits; proc.charge_frames(frame_count as u16)?; Ok(()) } fn install_cap(pid: Pid, slot: u64, data: ObjectData) -> Result<(), KernelError> { let tag = data.tag(); let (object_id, generation) = POOL.lock().allocate(data)?; let cap = CapRef::new(tag, object_id, Rights::ALL, generation); let ptable = crate::proc::PROCESSES.lock(); let pool = POOL.lock_after(&ptable); match cnode::resolve_caller_insert(pid, slot, cap, &ptable, &pool) { Ok(()) => Ok(()), Err(e) => { drop(pool); let r = POOL.lock_after(&ptable).free(object_id, generation); debug_assert!(r.is_ok()); Err(e) } } } fn insert_cap_into( pid: Pid, slot: u64, cap: CapRef, ptable: &crate::sync::IrqMutexGuard<'_, crate::proc::ProcessManager, 0>, ) -> Result<(), KernelError> { let pool = POOL.lock_after(ptable); cnode::resolve_caller_insert(pid, slot, cap, ptable, &pool) } fn read_cap_from( pid: Pid, slot: u64, expected_tag: ObjectTag, required_rights: Rights, ptable: &crate::sync::IrqMutexGuard<'_, crate::proc::ProcessManager, 0>, ) -> Result { let pool = POOL.lock_after(ptable); cnode::resolve_caller_validate(pid, slot, expected_tag, required_rights, ptable, &pool) } use crate::cap::object::FrameObjectData; fn create_shared_ring_frames( pid_a: Pid, base_slot_a: u64, pid_b: Pid, base_slot_b: u64, frame_count: u16, ) -> Result<(), KernelError> { let allocator = crate::mem::phys::BitmapFrameAllocator; (0..frame_count).try_fold((), |(), i| { let owned = allocator.allocate().ok_or(KernelError::PoolExhausted)?; let phys = owned.phys_addr(); core::mem::forget(owned); crate::mem::addr::zero_frame(phys); crate::mem::refcount::increment(phys).map_err(|_| { crate::mem::phys::BitmapFrameAllocator::free_frame_by_addr(phys); KernelError::ResourceExhausted })?; let data = ObjectData::Frame(FrameObjectData::new(phys)); let tag = data.tag(); let (obj_id, obj_gen) = POOL.lock().allocate(data).inspect_err(|_| { let _ = crate::mem::refcount::decrement(phys); crate::mem::phys::BitmapFrameAllocator::free_frame_by_addr(phys); })?; let cap = CapRef::new(tag, obj_id, Rights::ALL, obj_gen); let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(pid_a, base_slot_a + i as u64, cap, &ptable).inspect_err(|_| { let r = POOL.lock_after(&ptable).free(obj_id, obj_gen); debug_assert!(r.is_ok()); let _ = crate::mem::refcount::decrement(phys); crate::mem::phys::BitmapFrameAllocator::free_frame_by_addr(phys); })?; POOL.lock_after(&ptable).inc_ref(obj_id, obj_gen)?; insert_cap_into(pid_b, base_slot_b + i as u64, cap, &ptable).inspect_err(|_| { POOL.lock_after(&ptable).dec_ref(obj_id, obj_gen); })?; Ok(()) }) } fn create_ring_frames_from_phys( pid: Pid, base_slot: u64, phys_addrs: &[x86_64::PhysAddr], ) -> Result<(), KernelError> { phys_addrs.iter().enumerate().try_fold((), |(), (i, &phys)| { crate::mem::refcount::increment(phys).map_err(|_| KernelError::ResourceExhausted)?; let data = ObjectData::Frame(FrameObjectData::new(phys)); install_cap(pid, base_slot + i as u64, data).inspect_err(|_| { let _ = crate::mem::refcount::decrement(phys); }) }) } pub fn bootstrap_self_cap(pid: Pid, cnode_size_bits: u8) -> Result<(), KernelError> { bootstrap_root_cnode(pid, cnode_size_bits)?; install_cap(pid, SLOT_SELF, ObjectData::Process(ProcessObjectData { pid })) } pub fn bootstrap_serial_driver( init_pid: Pid, driver_pid: Pid, com1_gsi: crate::arch::ioapic::Gsi, ) -> Result<(), KernelError> { let ep_data = ObjectData::Endpoint(EndpointData::new()); let ep_tag = ep_data.tag(); let (ep_id, ep_gen) = POOL.lock().allocate(ep_data)?; let ep_cap = CapRef::new(ep_tag, ep_id, Rights::ALL, ep_gen); { let ptable = crate::proc::PROCESSES.lock(); match insert_cap_into(init_pid, init_slots::SERIAL_EP, ep_cap, &ptable) { Ok(()) => {} Err(e) => { let r = POOL.lock_after(&ptable).free(ep_id, ep_gen); debug_assert!(r.is_ok()); return Err(e); } } POOL.lock_after(&ptable).inc_ref(ep_id, ep_gen)?; match insert_cap_into(driver_pid, irq_driver_slots::EP, ep_cap, &ptable) { Ok(()) => {} Err(e) => { POOL.lock_after(&ptable).dec_ref(ep_id, ep_gen); return Err(e); } } } crate::show!(boot, "endpoint obj {} shared pid {} slot {} <-> pid {} slot {}", ep_id.raw(), init_pid.raw(), init_slots::SERIAL_EP, driver_pid.raw(), irq_driver_slots::EP ); let com1_vector = crate::arch::idt::COM1_VECTOR; install_cap( driver_pid, irq_driver_slots::IRQ, ObjectData::IrqHandler(IrqHandlerData { vector: com1_vector, source: IrqSource::Ioapic { gsi: com1_gsi }, port_range: COM1_PORTS, }), )?; let notif_data = ObjectData::Notification(NotificationData::new()); let notif_tag = notif_data.tag(); let (notif_id, notif_gen) = POOL.lock().allocate(notif_data)?; let notif_cap = CapRef::new(notif_tag, notif_id, Rights::ALL, notif_gen); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(driver_pid, irq_driver_slots::NOTIF, notif_cap, &ptable) .inspect_err(|_| { let r = POOL.lock_after(&ptable).free(notif_id, notif_gen); debug_assert!(r.is_ok()); })?; } crate::irq::bind( com1_vector, IrqSource::Ioapic { gsi: com1_gsi }, notif_id, notif_gen, crate::types::NotificationBit::ZERO, )?; crate::show!(boot, "serial irq slot {} notif slot {} com1 gsi {} -> vec {}", irq_driver_slots::IRQ, irq_driver_slots::NOTIF, com1_gsi, com1_vector ); Ok(()) } pub fn bootstrap_ps2kbd( init_pid: Pid, ps2kbd_pid: Pid, kbd_gsi: crate::arch::ioapic::Gsi, ) -> Result<(), KernelError> { let ep_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(init_pid, init_slots::SERIAL_EP, ObjectTag::Endpoint, Rights::ALL, &ptable)? }; POOL.lock() .inc_ref(ep_cap.object_id(), ep_cap.generation())?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(ps2kbd_pid, irq_driver_slots::EP, ep_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(ep_cap.object_id(), ep_cap.generation()); })?; } let kbd_vector = crate::arch::idt::KBD_VECTOR; install_cap( ps2kbd_pid, irq_driver_slots::IRQ, ObjectData::IrqHandler(IrqHandlerData { vector: kbd_vector, source: IrqSource::Ioapic { gsi: kbd_gsi }, port_range: PS2_KBD_PORTS, }), )?; let notif_data = ObjectData::Notification(NotificationData::new()); let notif_tag = notif_data.tag(); let (notif_id, notif_gen) = POOL.lock().allocate(notif_data)?; let notif_cap = CapRef::new(notif_tag, notif_id, Rights::ALL, notif_gen); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(ps2kbd_pid, irq_driver_slots::NOTIF, notif_cap, &ptable) .inspect_err(|_| { let r = POOL.lock_after(&ptable).free(notif_id, notif_gen); debug_assert!(r.is_ok()); })?; } crate::irq::bind( kbd_vector, IrqSource::Ioapic { gsi: kbd_gsi }, notif_id, notif_gen, crate::types::NotificationBit::ZERO, )?; crate::show!(boot, "ps2 kbd caps installed irq slot {} notification slot {} kbd gsi {} -> vector {}", irq_driver_slots::IRQ, irq_driver_slots::NOTIF, kbd_gsi, kbd_vector, ); Ok(()) } mod nvme_slots { pub const EP: u64 = 1; pub const PCI: u64 = 2; pub const IRQ: u64 = 3; pub const NOTIF: u64 = 4; pub const CLIENT_NOTIF: u64 = 6; pub const BLOCK_RING_BASE: u64 = 64; pub const BLOCK_RING_FRAMES: u16 = 16; } mod fs_service_slots { pub const NOTIF: u64 = 3; pub const DRIVER_NOTIF: u64 = 4; pub const BLOCK_RING_BASE: u64 = 64; } mod fs_service_client_slots { pub const CLIENT_NOTIF: u64 = 6; pub const CLIENT_RING_BASE: u64 = 96; } mod vfs_slots { pub const NOTIF: u64 = 3; pub const LANCERFS_NOTIF: u64 = 4; pub const LANCERFS_RING_BASE: u64 = 64; pub const LANCERFS_RING_FRAMES: u16 = 16; pub const CLIENT0_RING_BASE: u64 = 160; pub const CLIENT0_RING_FRAMES: u16 = 16; pub const CLIENT0_NOTIF: u64 = 176; } mod ramfs_slots { pub const NOTIF: u64 = 3; pub const VFS_NOTIF: u64 = 4; pub const CLIENT_RING_BASE: u64 = 64; } mod vfs_ramfs_slots { pub const RAMFS_NOTIF: u64 = 6; pub const RAMFS_RING_BASE: u64 = 96; pub const RAMFS_RING_FRAMES: u16 = 16; } mod vfs_client_slots { pub const NOTIF: u64 = 12; pub const VFS_NOTIF: u64 = 13; pub const CLIENT_RING_BASE: u64 = 320; } mod virtio_net_slots { pub const EP: u64 = 1; pub const PCI: u64 = 2; pub const IRQ: u64 = 3; pub const NOTIF: u64 = 4; pub const NETSTACK_NOTIF: u64 = 7; pub const PACKET_RING_BASE: u64 = 64; pub const PACKET_RING_FRAMES: u16 = 16; } mod net_service_slots { pub const EP: u64 = 1; pub const NOTIF: u64 = 3; pub const DRIVER_NOTIF: u64 = 4; pub const SHELL_NOTIF: u64 = 6; pub const INIT_NOTIF: u64 = 8; pub const PACKET_RING_BASE: u64 = 64; pub const SHELL_RING_BASE: u64 = 96; pub const SHELL_RING_FRAMES: u16 = 4; pub const INIT_RING_BASE: u64 = 128; } mod remote_shell_slots { pub const EP: u64 = 1; pub const NOTIF: u64 = 3; pub const NETSTACK_NOTIF: u64 = 4; pub const SHELL_RING_BASE: u64 = 64; pub const VFS_RING_BASE: u64 = 128; pub const VFS_RING_FRAMES: u16 = 16; } pub fn bootstrap_fbconsole(init_pid: Pid, fbconsole_pid: Pid) -> Result<(), KernelError> { let fb = crate::arch::boot::framebuffer().ok_or(KernelError::NotFound)?; let ep_data = ObjectData::Endpoint(EndpointData::new()); let ep_tag = ep_data.tag(); let (ep_id, ep_gen) = POOL.lock().allocate(ep_data)?; let ep_cap = CapRef::new(ep_tag, ep_id, Rights::ALL, ep_gen); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(init_pid, init_slots::FB_EP, ep_cap, &ptable) .inspect_err(|_| { let r = POOL.lock_after(&ptable).free(ep_id, ep_gen); debug_assert!(r.is_ok()); })?; POOL.lock_after(&ptable).inc_ref(ep_id, ep_gen)?; insert_cap_into(fbconsole_pid, fbconsole_slots::EP, ep_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable).dec_ref(ep_id, ep_gen); })?; } crate::show!(boot, "fb endpoint obj {} shared pid {} slot {} <-> pid {} slot {}", ep_id.raw(), init_pid.raw(), init_slots::FB_EP, fbconsole_pid.raw(), fbconsole_slots::EP, ); let fb_phys = fb.addr() as u64 - crate::mem::addr::hhdm_offset(); let fb_byte_size = fb .pitch() .checked_mul(fb.height()) .ok_or(KernelError::InvalidParameter)?; let page_count = fb_byte_size.div_ceil(4096); crate::show!(boot, "fb phys {:#x} {} bytes {} pages pitch {} {}x{} {}bpp", fb_phys, fb_byte_size, page_count, fb.pitch(), fb.width(), fb.height(), fb.bpp(), ); let fb_data = FramebufferData::new( fb_phys, fb.width() as u32, fb.height() as u32, fb.pitch() as u32, fb.bpp(), fb_byte_size, ) .ok_or(KernelError::InvalidParameter)?; install_cap( fbconsole_pid, fbconsole_slots::FB, ObjectData::Framebuffer(fb_data), )?; let ring_frame_count = crate::console::phys_frame_count(); if ring_frame_count > 0 { let mut ring_phys: [x86_64::PhysAddr; 16] = [x86_64::PhysAddr::zero(); 16]; (0..ring_frame_count).fold((), |(), i| { ring_phys[i as usize] = crate::console::phys_addr(i).unwrap(); }); create_ring_frames_from_phys( fbconsole_pid, fbconsole_slots::CONSOLE_RING_BASE, &ring_phys[..ring_frame_count as usize], )?; crate::show!(boot, "console ring granted to fbconsole {} frames at base slot {}", ring_frame_count, fbconsole_slots::CONSOLE_RING_BASE, ); } crate::show!(boot, "fbconsole caps installed endpoint slot {} framebuffer slot {}", fbconsole_slots::EP, fbconsole_slots::FB, ); Ok(()) } pub fn bootstrap_nvme( init_pid: Pid, driver_pid: Pid, device_table_idx: u8, mapper: &mut x86_64::structures::paging::OffsetPageTable, allocator: &mut crate::mem::phys::BitmapFrameAllocator, hhdm_offset: u64, ) -> Result<(), KernelError> { let ep_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(init_pid, init_slots::SERIAL_EP, ObjectTag::Endpoint, Rights::ALL, &ptable)? }; POOL.lock() .inc_ref(ep_cap.object_id(), ep_cap.generation())?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(driver_pid, nvme_slots::EP, ep_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(ep_cap.object_id(), ep_cap.generation()); })?; } install_cap( driver_pid, nvme_slots::PCI, ObjectData::PciDevice(PciDeviceData { device_table_idx }), )?; crate::pci::enable_bus_master(device_table_idx)?; if crate::iommu::is_available() { crate::iommu::setup_device_context(device_table_idx, driver_pid, allocator, hhdm_offset)?; } let nvme_vector = crate::arch::idt::NVME_VECTOR; let dummy_ports = match PortRange::new(0, 1) { Some(pr) => pr, None => return Err(KernelError::InvalidParameter), }; crate::pci::msix::ensure_table_mapped(device_table_idx, mapper, allocator, hhdm_offset)?; crate::pci::msix::configure_entry(device_table_idx, 0, nvme_vector, 0)?; crate::pci::msix::enable_msix(device_table_idx)?; crate::pci::msix::unmask_entry(device_table_idx, 0)?; crate::show!(boot, "nvme msi-x configured entry 0 -> vector {}", nvme_vector); let irq_source = IrqSource::Msix { device_table_idx, entry_idx: 0, }; install_cap( driver_pid, nvme_slots::IRQ, ObjectData::IrqHandler(IrqHandlerData { vector: nvme_vector, source: irq_source, port_range: dummy_ports, }), )?; let notif_data = ObjectData::Notification(NotificationData::new()); let notif_tag = notif_data.tag(); let (notif_id, notif_gen) = POOL.lock().allocate(notif_data)?; let notif_cap = CapRef::new(notif_tag, notif_id, Rights::ALL, notif_gen); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(driver_pid, nvme_slots::NOTIF, notif_cap, &ptable) .inspect_err(|_| { let r = POOL.lock_after(&ptable).free(notif_id, notif_gen); debug_assert!(r.is_ok()); })?; } crate::irq::bind( nvme_vector, irq_source, notif_id, notif_gen, crate::types::NotificationBit::ZERO, )?; crate::show!(boot, "nvme caps installed pci slot {} irq slot {} notification slot {} vector {}", nvme_slots::PCI, nvme_slots::IRQ, nvme_slots::NOTIF, nvme_vector, ); Ok(()) } pub fn bootstrap_nvme_service(driver_pid: Pid, client_pid: Pid) -> Result<(), KernelError> { create_shared_ring_frames( driver_pid, nvme_slots::BLOCK_RING_BASE, client_pid, fs_service_slots::BLOCK_RING_BASE, nvme_slots::BLOCK_RING_FRAMES, )?; let client_notif_data = ObjectData::Notification(NotificationData::new()); let client_notif_tag = client_notif_data.tag(); let (client_notif_id, client_notif_gen) = POOL.lock().allocate(client_notif_data)?; let client_notif_cap = CapRef::new( client_notif_tag, client_notif_id, Rights::ALL, client_notif_gen, ); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(client_pid, fs_service_slots::NOTIF, client_notif_cap, &ptable) .inspect_err(|_| { let r = POOL .lock_after(&ptable) .free(client_notif_id, client_notif_gen); debug_assert!(r.is_ok()); })?; POOL.lock_after(&ptable) .inc_ref(client_notif_id, client_notif_gen)?; insert_cap_into(driver_pid, nvme_slots::CLIENT_NOTIF, client_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(client_notif_id, client_notif_gen); })?; } let driver_notif_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(driver_pid, nvme_slots::NOTIF, ObjectTag::Notification, Rights::ALL, &ptable)? }; POOL.lock() .inc_ref(driver_notif_cap.object_id(), driver_notif_cap.generation())?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(client_pid, fs_service_slots::DRIVER_NOTIF, driver_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(driver_notif_cap.object_id(), driver_notif_cap.generation()); })?; } crate::show!(boot, "nvme service caps ring base {}/{} notif slots {}/{} driver-notif slot {}", nvme_slots::BLOCK_RING_BASE, fs_service_slots::BLOCK_RING_BASE, nvme_slots::CLIENT_NOTIF, fs_service_slots::NOTIF, fs_service_slots::DRIVER_NOTIF, ); Ok(()) } pub fn bootstrap_vfs_lancerfs(vfs_pid: Pid, lancerfs_pid: Pid) -> Result<(), KernelError> { create_shared_ring_frames( vfs_pid, vfs_slots::LANCERFS_RING_BASE, lancerfs_pid, fs_service_client_slots::CLIENT_RING_BASE, vfs_slots::LANCERFS_RING_FRAMES, )?; let vfs_notif_data = ObjectData::Notification(NotificationData::new()); let vfs_notif_tag = vfs_notif_data.tag(); let (vfs_notif_id, vfs_notif_gen) = POOL.lock().allocate(vfs_notif_data)?; let vfs_notif_cap = CapRef::new(vfs_notif_tag, vfs_notif_id, Rights::ALL, vfs_notif_gen); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(vfs_pid, vfs_slots::NOTIF, vfs_notif_cap, &ptable) .inspect_err(|_| { let r = POOL.lock_after(&ptable).free(vfs_notif_id, vfs_notif_gen); debug_assert!(r.is_ok()); })?; POOL.lock_after(&ptable) .inc_ref(vfs_notif_id, vfs_notif_gen)?; insert_cap_into(lancerfs_pid, fs_service_client_slots::CLIENT_NOTIF, vfs_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(vfs_notif_id, vfs_notif_gen); })?; } let lancerfs_notif_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(lancerfs_pid, fs_service_slots::NOTIF, ObjectTag::Notification, Rights::ALL, &ptable)? }; POOL.lock().inc_ref( lancerfs_notif_cap.object_id(), lancerfs_notif_cap.generation(), )?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(vfs_pid, vfs_slots::LANCERFS_NOTIF, lancerfs_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable).dec_ref( lancerfs_notif_cap.object_id(), lancerfs_notif_cap.generation(), ); })?; } crate::show!(boot, "vfs-lancerfs caps ring base {}/{} vfs-notif slots {}/{} lancerfs-notif slot {}", vfs_slots::LANCERFS_RING_BASE, fs_service_client_slots::CLIENT_RING_BASE, vfs_slots::NOTIF, fs_service_client_slots::CLIENT_NOTIF, vfs_slots::LANCERFS_NOTIF, ); Ok(()) } pub fn bootstrap_vfs_ramfs(vfs_pid: Pid, ramfs_pid: Pid) -> Result<(), KernelError> { create_shared_ring_frames( vfs_pid, vfs_ramfs_slots::RAMFS_RING_BASE, ramfs_pid, ramfs_slots::CLIENT_RING_BASE, vfs_ramfs_slots::RAMFS_RING_FRAMES, )?; let vfs_notif_cap = { let ptable = crate::proc::PROCESSES.lock(); match read_cap_from(vfs_pid, vfs_slots::NOTIF, ObjectTag::Notification, Rights::ALL, &ptable) { Ok(cap) => { drop(ptable); POOL.lock().inc_ref(cap.object_id(), cap.generation())?; cap } Err(_) => { drop(ptable); let notif_data = ObjectData::Notification(NotificationData::new()); let notif_tag = notif_data.tag(); let (nid, ngen) = POOL.lock().allocate(notif_data)?; let cap = CapRef::new(notif_tag, nid, Rights::ALL, ngen); let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(vfs_pid, vfs_slots::NOTIF, cap, &ptable) .inspect_err(|_| { let r = POOL.lock_after(&ptable).free(nid, ngen); debug_assert!(r.is_ok()); })?; cap } } }; { POOL.lock() .inc_ref(vfs_notif_cap.object_id(), vfs_notif_cap.generation())?; let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(ramfs_pid, ramfs_slots::VFS_NOTIF, vfs_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(vfs_notif_cap.object_id(), vfs_notif_cap.generation()); })?; } let ramfs_notif_data = ObjectData::Notification(NotificationData::new()); let ramfs_notif_tag = ramfs_notif_data.tag(); let (ramfs_notif_id, ramfs_notif_gen) = POOL.lock().allocate(ramfs_notif_data)?; let ramfs_notif_cap = CapRef::new( ramfs_notif_tag, ramfs_notif_id, Rights::ALL, ramfs_notif_gen, ); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(ramfs_pid, ramfs_slots::NOTIF, ramfs_notif_cap, &ptable) .inspect_err(|_| { let r = POOL .lock_after(&ptable) .free(ramfs_notif_id, ramfs_notif_gen); debug_assert!(r.is_ok()); })?; POOL.lock_after(&ptable) .inc_ref(ramfs_notif_id, ramfs_notif_gen)?; insert_cap_into(vfs_pid, vfs_ramfs_slots::RAMFS_NOTIF, ramfs_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(ramfs_notif_id, ramfs_notif_gen); })?; } crate::show!(boot, "vfs-ramfs caps ring base {}/{} vfs-notif slot {} ramfs-notif slots {}/{}", vfs_ramfs_slots::RAMFS_RING_BASE, ramfs_slots::CLIENT_RING_BASE, ramfs_slots::VFS_NOTIF, ramfs_slots::NOTIF, vfs_ramfs_slots::RAMFS_NOTIF, ); Ok(()) } pub fn bootstrap_vfs_client(client_pid: Pid, vfs_pid: Pid) -> Result<(), KernelError> { create_shared_ring_frames( client_pid, vfs_client_slots::CLIENT_RING_BASE, vfs_pid, vfs_slots::CLIENT0_RING_BASE, vfs_slots::CLIENT0_RING_FRAMES, )?; let client_notif_data = ObjectData::Notification(NotificationData::new()); let client_notif_tag = client_notif_data.tag(); let (client_notif_id, client_notif_gen) = POOL.lock().allocate(client_notif_data)?; let client_notif_cap = CapRef::new( client_notif_tag, client_notif_id, Rights::ALL, client_notif_gen, ); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(client_pid, vfs_client_slots::NOTIF, client_notif_cap, &ptable) .inspect_err(|_| { let r = POOL .lock_after(&ptable) .free(client_notif_id, client_notif_gen); debug_assert!(r.is_ok()); })?; POOL.lock_after(&ptable) .inc_ref(client_notif_id, client_notif_gen)?; insert_cap_into(vfs_pid, vfs_slots::CLIENT0_NOTIF, client_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(client_notif_id, client_notif_gen); })?; } let vfs_notif_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(vfs_pid, vfs_slots::NOTIF, ObjectTag::Notification, Rights::ALL, &ptable)? }; POOL.lock() .inc_ref(vfs_notif_cap.object_id(), vfs_notif_cap.generation())?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(client_pid, vfs_client_slots::VFS_NOTIF, vfs_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(vfs_notif_cap.object_id(), vfs_notif_cap.generation()); })?; } crate::show!(boot, "vfs client caps ring base {}/{} client-notif slots {}/{} vfs-notif slot {}", vfs_client_slots::CLIENT_RING_BASE, vfs_slots::CLIENT0_RING_BASE, vfs_client_slots::NOTIF, vfs_slots::CLIENT0_NOTIF, vfs_client_slots::VFS_NOTIF, ); Ok(()) } pub fn bootstrap_init_vfs_control(init_pid: Pid, vfs_pid: Pid) -> Result<(), KernelError> { install_cap( init_pid, init_slots::VFS_PROC, ObjectData::Process(ProcessObjectData { pid: vfs_pid }), )?; crate::show!(boot, "init vfs control process cap for vfs at init slot {}", init_slots::VFS_PROC, ); Ok(()) } pub fn bootstrap_virtio_net( init_pid: Pid, driver_pid: Pid, virtio_net_gsi: crate::arch::ioapic::Gsi, device_table_idx: u8, mapper: &mut x86_64::structures::paging::OffsetPageTable, allocator: &mut crate::mem::phys::BitmapFrameAllocator, hhdm_offset: u64, ) -> Result<(), KernelError> { let ep_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(init_pid, init_slots::SERIAL_EP, ObjectTag::Endpoint, Rights::ALL, &ptable)? }; POOL.lock() .inc_ref(ep_cap.object_id(), ep_cap.generation())?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(driver_pid, virtio_net_slots::EP, ep_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(ep_cap.object_id(), ep_cap.generation()); })?; } install_cap( driver_pid, virtio_net_slots::PCI, ObjectData::PciDevice(PciDeviceData { device_table_idx }), )?; crate::pci::enable_bus_master(device_table_idx)?; if crate::iommu::is_available() { crate::iommu::setup_device_context(device_table_idx, driver_pid, allocator, hhdm_offset)?; } let virtio_net_vector = crate::arch::idt::VIRTIO_NET_VECTOR; let dummy_ports = match PortRange::new(0, 1) { Some(pr) => pr, None => return Err(KernelError::InvalidParameter), }; let has_msix = { let dev_table = crate::pci::DEVICE_TABLE.lock(); dev_table .get(device_table_idx as usize) .and_then(|d| d.msix_cap) .is_some() }; let irq_source = match has_msix { true => { crate::pci::msix::ensure_table_mapped( device_table_idx, mapper, allocator, hhdm_offset, )?; crate::pci::msix::configure_entry(device_table_idx, 0, virtio_net_vector, 0)?; crate::pci::msix::enable_msix(device_table_idx)?; crate::pci::msix::unmask_entry(device_table_idx, 0)?; crate::show!(boot, "virtio-net msi-x configured entry 0 -> vector {}", virtio_net_vector, ); IrqSource::Msix { device_table_idx, entry_idx: 0, } } false => IrqSource::Ioapic { gsi: virtio_net_gsi, }, }; install_cap( driver_pid, virtio_net_slots::IRQ, ObjectData::IrqHandler(IrqHandlerData { vector: virtio_net_vector, source: irq_source, port_range: dummy_ports, }), )?; let notif_data = ObjectData::Notification(NotificationData::new()); let notif_tag = notif_data.tag(); let (notif_id, notif_gen) = POOL.lock().allocate(notif_data)?; let notif_cap = CapRef::new(notif_tag, notif_id, Rights::ALL, notif_gen); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(driver_pid, virtio_net_slots::NOTIF, notif_cap, &ptable) .inspect_err(|_| { let r = POOL.lock_after(&ptable).free(notif_id, notif_gen); debug_assert!(r.is_ok()); })?; } crate::irq::bind( virtio_net_vector, irq_source, notif_id, notif_gen, crate::types::NotificationBit::ZERO, )?; crate::show!(boot, "virtio-net caps installed pci slot {} irq slot {} notification slot {} vector {}", virtio_net_slots::PCI, virtio_net_slots::IRQ, virtio_net_slots::NOTIF, virtio_net_vector, ); Ok(()) } pub fn bootstrap_net_service( init_pid: Pid, driver_pid: Pid, netstack_pid: Pid, ) -> Result<(), KernelError> { let ep_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(init_pid, init_slots::SERIAL_EP, ObjectTag::Endpoint, Rights::ALL, &ptable)? }; POOL.lock() .inc_ref(ep_cap.object_id(), ep_cap.generation())?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(netstack_pid, net_service_slots::EP, ep_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(ep_cap.object_id(), ep_cap.generation()); })?; } create_shared_ring_frames( driver_pid, virtio_net_slots::PACKET_RING_BASE, netstack_pid, net_service_slots::PACKET_RING_BASE, virtio_net_slots::PACKET_RING_FRAMES, )?; let netstack_notif_data = ObjectData::Notification(NotificationData::new()); let netstack_notif_tag = netstack_notif_data.tag(); let (netstack_notif_id, netstack_notif_gen) = POOL.lock().allocate(netstack_notif_data)?; let netstack_notif_cap = CapRef::new( netstack_notif_tag, netstack_notif_id, Rights::ALL, netstack_notif_gen, ); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(netstack_pid, net_service_slots::NOTIF, netstack_notif_cap, &ptable) .inspect_err(|_| { let r = POOL .lock_after(&ptable) .free(netstack_notif_id, netstack_notif_gen); debug_assert!(r.is_ok()); })?; POOL.lock_after(&ptable) .inc_ref(netstack_notif_id, netstack_notif_gen)?; insert_cap_into(driver_pid, virtio_net_slots::NETSTACK_NOTIF, netstack_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(netstack_notif_id, netstack_notif_gen); })?; } let driver_notif_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(driver_pid, virtio_net_slots::NOTIF, ObjectTag::Notification, Rights::ALL, &ptable)? }; POOL.lock() .inc_ref(driver_notif_cap.object_id(), driver_notif_cap.generation())?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(netstack_pid, net_service_slots::DRIVER_NOTIF, driver_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(driver_notif_cap.object_id(), driver_notif_cap.generation()); })?; } crate::show!(boot, "net service caps ring base {}/{} notif slots {}/{} driver-notif slot {}", virtio_net_slots::PACKET_RING_BASE, net_service_slots::PACKET_RING_BASE, virtio_net_slots::NETSTACK_NOTIF, net_service_slots::NOTIF, net_service_slots::DRIVER_NOTIF, ); Ok(()) } pub fn bootstrap_remote_shell( init_pid: Pid, netstack_pid: Pid, shell_pid: Pid, ) -> Result<(), KernelError> { let ep_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(init_pid, init_slots::SERIAL_EP, ObjectTag::Endpoint, Rights::ALL, &ptable)? }; POOL.lock() .inc_ref(ep_cap.object_id(), ep_cap.generation())?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(shell_pid, remote_shell_slots::EP, ep_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(ep_cap.object_id(), ep_cap.generation()); })?; } create_shared_ring_frames( netstack_pid, net_service_slots::SHELL_RING_BASE, shell_pid, remote_shell_slots::SHELL_RING_BASE, net_service_slots::SHELL_RING_FRAMES, )?; let shell_notif_data = ObjectData::Notification(NotificationData::new()); let shell_notif_tag = shell_notif_data.tag(); let (shell_notif_id, shell_notif_gen) = POOL.lock().allocate(shell_notif_data)?; let shell_notif_cap = CapRef::new( shell_notif_tag, shell_notif_id, Rights::ALL, shell_notif_gen, ); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(shell_pid, remote_shell_slots::NOTIF, shell_notif_cap, &ptable) .inspect_err(|_| { let r = POOL .lock_after(&ptable) .free(shell_notif_id, shell_notif_gen); debug_assert!(r.is_ok()); })?; POOL.lock_after(&ptable) .inc_ref(shell_notif_id, shell_notif_gen)?; insert_cap_into(netstack_pid, net_service_slots::SHELL_NOTIF, shell_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(shell_notif_id, shell_notif_gen); })?; } let netstack_notif_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(netstack_pid, net_service_slots::NOTIF, ObjectTag::Notification, Rights::ALL, &ptable)? }; POOL.lock().inc_ref( netstack_notif_cap.object_id(), netstack_notif_cap.generation(), )?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(shell_pid, remote_shell_slots::NETSTACK_NOTIF, netstack_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable).dec_ref( netstack_notif_cap.object_id(), netstack_notif_cap.generation(), ); })?; } { let ptable = crate::proc::PROCESSES.lock(); match read_cap_from(init_pid, vfs_client_slots::CLIENT_RING_BASE, ObjectTag::Frame, Rights::ALL, &ptable) { Ok(_) => { drop(ptable); (0..remote_shell_slots::VFS_RING_FRAMES).try_fold((), |(), i| { let src_slot = vfs_client_slots::CLIENT_RING_BASE + i as u64; let dst_slot = remote_shell_slots::VFS_RING_BASE + i as u64; let cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(init_pid, src_slot, ObjectTag::Frame, Rights::ALL, &ptable)? }; POOL.lock().inc_ref(cap.object_id(), cap.generation())?; let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(shell_pid, dst_slot, cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(cap.object_id(), cap.generation()); }) })?; [(vfs_client_slots::NOTIF, ObjectTag::Notification), (vfs_client_slots::VFS_NOTIF, ObjectTag::Notification), (init_slots::VFS_PROC, ObjectTag::Process)] .iter() .try_for_each(|&(slot, tag)| { let cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(init_pid, slot, tag, Rights::ALL, &ptable)? }; POOL.lock().inc_ref(cap.object_id(), cap.generation())?; let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(shell_pid, slot, cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(cap.object_id(), cap.generation()); }) })?; } Err(_) => { drop(ptable); } } } crate::show!(boot, "remote shell caps ring base {}/{} shell-notif slots {}/{} netstack-notif slot {}", net_service_slots::SHELL_RING_BASE, remote_shell_slots::SHELL_RING_BASE, net_service_slots::SHELL_NOTIF, remote_shell_slots::NOTIF, remote_shell_slots::NETSTACK_NOTIF, ); Ok(()) } pub fn bootstrap_init_netring(init_pid: Pid, netstack_pid: Pid) -> Result<(), KernelError> { create_shared_ring_frames( init_pid, init_slots::NETSTACK_RING_BASE, netstack_pid, net_service_slots::INIT_RING_BASE, init_slots::NETSTACK_RING_FRAMES, )?; let init_notif_data = ObjectData::Notification(NotificationData::new()); let init_notif_tag = init_notif_data.tag(); let (init_notif_id, init_notif_gen) = POOL.lock().allocate(init_notif_data)?; let init_notif_cap = CapRef::new(init_notif_tag, init_notif_id, Rights::ALL, init_notif_gen); { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(init_pid, init_slots::INIT_NOTIF, init_notif_cap, &ptable) .inspect_err(|_| { let r = POOL.lock_after(&ptable).free(init_notif_id, init_notif_gen); debug_assert!(r.is_ok()); })?; POOL.lock_after(&ptable) .inc_ref(init_notif_id, init_notif_gen)?; insert_cap_into(netstack_pid, net_service_slots::INIT_NOTIF, init_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable) .dec_ref(init_notif_id, init_notif_gen); })?; } let netstack_notif_cap = { let ptable = crate::proc::PROCESSES.lock(); read_cap_from(netstack_pid, net_service_slots::NOTIF, ObjectTag::Notification, Rights::ALL, &ptable)? }; POOL.lock().inc_ref( netstack_notif_cap.object_id(), netstack_notif_cap.generation(), )?; { let ptable = crate::proc::PROCESSES.lock(); insert_cap_into(init_pid, init_slots::NETSTACK_NOTIF, netstack_notif_cap, &ptable) .inspect_err(|_| { POOL.lock_after(&ptable).dec_ref( netstack_notif_cap.object_id(), netstack_notif_cap.generation(), ); })?; } crate::show!(boot, "init-netstack ring base {}/{} init-notif slots {}/{} netstack-notif slot {}", init_slots::NETSTACK_RING_BASE, net_service_slots::INIT_RING_BASE, init_slots::INIT_NOTIF, net_service_slots::INIT_NOTIF, init_slots::NETSTACK_NOTIF, ); Ok(()) }