use core::sync::atomic::Ordering; use crate::cap::cnode; use crate::cap::object::ObjectTag; use crate::cap::ops; use crate::cap::pool::POOL; use crate::mem::phys::BitmapFrameAllocator; use crate::proc::PROCESSES; use crate::ring::{ CompletionEntry, RING_OP_CAP_CREATE, RING_OP_NOP, RING_OP_NOTIFY_POLL, RING_OP_NOTIFY_SIGNAL, RingHeader, RingIndex, SubmissionEntry, ring_cq_offset, ring_sq_offset, }; use crate::types::Pid; fn setup_ring_page() -> (x86_64::PhysAddr, *mut u8) { let allocator = BitmapFrameAllocator; let frame = allocator.allocate().expect("alloc ring frame"); let phys = frame.phys_addr(); crate::mem::addr::zero_frame(phys); let virt = crate::mem::addr::phys_to_virt(phys); let _ = frame.inner(); (phys, virt.as_mut_ptr::()) } fn teardown_ring_page(phys: x86_64::PhysAddr) { BitmapFrameAllocator::free_frame_by_addr(phys); } unsafe fn write_sqe(ring_base: *mut u8, index: u32, sqe: SubmissionEntry) { let sq_base = unsafe { ring_base.add(ring_sq_offset()) }; let entry_ptr = unsafe { sq_base.add((index as usize) * core::mem::size_of::()) }; unsafe { core::ptr::write_volatile(entry_ptr as *mut SubmissionEntry, sqe) }; } unsafe fn read_cqe(ring_base: *const u8, index: u32) -> CompletionEntry { let cq_base = unsafe { ring_base.add(ring_cq_offset()) }; let entry_ptr = unsafe { cq_base.add((index as usize) * core::mem::size_of::()) }; unsafe { core::ptr::read_volatile(entry_ptr as *const CompletionEntry) } } unsafe fn set_sq_tail(ring_base: *mut u8, val: u32) { let header = unsafe { &*(ring_base as *const RingHeader) }; header.sq_tail.store(val, Ordering::Release); } fn alloc_test_process() -> ( crate::types::Pid, crate::sync::IrqMutexGuard<'static, crate::proc::ProcessManager, 0>, ) { let mut allocator = BitmapFrameAllocator; let mut ptable = PROCESSES.lock(); let created = ptable.allocate(&mut allocator).expect("alloc process"); ptable.start(created).expect("start"); let pid = created.pid(); bootstrap_test_cnode(pid, &mut ptable); (pid, ptable) } fn bootstrap_test_cnode(pid: Pid, ptable: &mut crate::proc::ProcessManager) { let size_bits = crate::proc::ROOT_CNODE_SIZE_BITS; let allocator = &crate::mem::phys::BitmapFrameAllocator; let cnode_data = crate::cap::cnode::create_cnode(size_bits, allocator).expect("create cnode"); let frame_count = cnode_data.frame_count; let (cnode_id, cnode_gen) = POOL.lock().allocate(crate::cap::object::ObjectData::CNode(cnode_data)).expect("alloc cnode"); let proc = ptable.get_mut(pid).expect("get proc"); proc.root_cnode = Some((cnode_id, cnode_gen)); proc.cnode_depth = size_bits; proc.charge_frames(frame_count as u16).expect("charge frames"); } crate::kernel_test!( fn ring_nop_returns_zero() { let (phys, ring_base) = setup_ring_page(); let (pid, ptable) = alloc_test_process(); drop(ptable); unsafe { write_sqe( ring_base, 0, SubmissionEntry { opcode: RING_OP_NOP, user_data: 42, ..SubmissionEntry::zeroed() }, ); set_sq_tail(ring_base, 1); } let result = crate::ring::process::ring_enter(phys, pid, 0); assert!(result == Ok(1), "should process 1 entry"); let cqe = unsafe { read_cqe(ring_base, 0) }; assert!(cqe.result == 0, "NOP should return 0"); assert!(cqe.user_data == 42, "user_data should be preserved"); let mut ptable = PROCESSES.lock(); ptable.destroy(pid, &mut BitmapFrameAllocator); drop(ptable); teardown_ring_page(phys); } ); crate::kernel_test!( fn ring_cap_create_via_ring() { let (phys, ring_base) = setup_ring_page(); let (pid, ptable) = alloc_test_process(); let (cnode_id, cnode_gen, depth) = cnode::cnode_coords(pid, &ptable).expect("cnode coords"); { let mut pool = POOL.lock_after(&ptable); ops::create_via_cnode(&mut pool, cnode_id, cnode_gen, 10, depth, ObjectTag::Endpoint) .expect("pre-create endpoint cap"); } drop(ptable); unsafe { write_sqe( ring_base, 0, SubmissionEntry { opcode: RING_OP_CAP_CREATE, arg0: ObjectTag::Endpoint as u8 as u64, cap_slot: 20, user_data: 100, ..SubmissionEntry::zeroed() }, ); set_sq_tail(ring_base, 1); } let result = crate::ring::process::ring_enter(phys, pid, 0); assert!(result == Ok(1), "should process 1 entry"); let cqe = unsafe { read_cqe(ring_base, 0) }; assert!( cqe.result >= 0, "cap_create should succeed, got {}", cqe.result ); assert!(cqe.user_data == 100, "user_data should be preserved"); let mut ptable = PROCESSES.lock(); ptable.destroy(pid, &mut BitmapFrameAllocator); drop(ptable); teardown_ring_page(phys); } ); crate::kernel_test!( fn ring_invalid_opcode_returns_error() { let (phys, ring_base) = setup_ring_page(); let (pid, ptable) = alloc_test_process(); drop(ptable); unsafe { write_sqe( ring_base, 0, SubmissionEntry { opcode: 0xFF, user_data: 7, ..SubmissionEntry::zeroed() }, ); set_sq_tail(ring_base, 1); } let result = crate::ring::process::ring_enter(phys, pid, 0); assert!(result == Ok(1), "should still process the entry"); let cqe = unsafe { read_cqe(ring_base, 0) }; assert!(cqe.result < 0, "invalid opcode should return error"); assert!(cqe.user_data == 7, "user_data should be preserved"); let mut ptable = PROCESSES.lock(); ptable.destroy(pid, &mut BitmapFrameAllocator); drop(ptable); teardown_ring_page(phys); } ); crate::kernel_test!( fn ring_empty_sq_returns_zero() { let (phys, _ring_base) = setup_ring_page(); let (pid, ptable) = alloc_test_process(); drop(ptable); let result = crate::ring::process::ring_enter(phys, pid, 0); assert!(result == Ok(0), "empty SQ should return 0 completions"); let mut ptable = PROCESSES.lock(); ptable.destroy(pid, &mut BitmapFrameAllocator); drop(ptable); teardown_ring_page(phys); } ); crate::kernel_test!( fn ring_cq_full_limits_processing() { let (phys, ring_base) = setup_ring_page(); let (pid, ptable) = alloc_test_process(); drop(ptable); let header = unsafe { &*(ring_base as *const RingHeader) }; header.cq_head.store(0, Ordering::Release); header .cq_tail .store(crate::ring::MAX_CQ_ENTRIES, Ordering::Release); { let mut ptable = PROCESSES.lock(); let proc = ptable.get_mut(pid).expect("get proc"); proc.ring_cq_tail = RingIndex::new(crate::ring::MAX_CQ_ENTRIES); } unsafe { write_sqe( ring_base, 0, SubmissionEntry { opcode: RING_OP_NOP, user_data: 1, ..SubmissionEntry::zeroed() }, ); set_sq_tail(ring_base, 1); } let result = crate::ring::process::ring_enter(phys, pid, 0); assert!(result == Ok(0), "should process 0 entries when CQ is full"); let mut ptable = PROCESSES.lock(); ptable.destroy(pid, &mut BitmapFrameAllocator); drop(ptable); teardown_ring_page(phys); } ); crate::kernel_test!( fn ring_batch_cap_limits_processing() { let (phys, ring_base) = setup_ring_page(); let (pid, ptable) = alloc_test_process(); drop(ptable); (0..32u32).for_each(|i| unsafe { write_sqe( ring_base, i, SubmissionEntry { opcode: RING_OP_NOP, user_data: i, ..SubmissionEntry::zeroed() }, ); }); unsafe { set_sq_tail(ring_base, 32) }; let result = crate::ring::process::ring_enter(phys, pid, 0).expect("ring_enter"); assert!( result <= 16, "batch cap should limit to MAX_RING_BATCH (16), got {}", result ); let mut ptable = PROCESSES.lock(); ptable.destroy(pid, &mut BitmapFrameAllocator); drop(ptable); teardown_ring_page(phys); } ); crate::kernel_test!( fn ring_notify_signal_and_poll_via_ring() { let (phys, ring_base) = setup_ring_page(); let (pid, ptable) = alloc_test_process(); let (cnode_id, cnode_gen, depth) = cnode::cnode_coords(pid, &ptable).expect("cnode coords"); { let mut pool = POOL.lock_after(&ptable); ops::create_via_cnode(&mut pool, cnode_id, cnode_gen, 5, depth, ObjectTag::Notification) .expect("create notification"); } drop(ptable); unsafe { write_sqe( ring_base, 0, SubmissionEntry { opcode: RING_OP_NOTIFY_SIGNAL, cap_slot: 5, arg0: 0xFF, user_data: 200, ..SubmissionEntry::zeroed() }, ); write_sqe( ring_base, 1, SubmissionEntry { opcode: RING_OP_NOTIFY_POLL, cap_slot: 5, user_data: 201, ..SubmissionEntry::zeroed() }, ); set_sq_tail(ring_base, 2); } let result = crate::ring::process::ring_enter(phys, pid, 0); assert!(result == Ok(2), "should process 2 entries"); let cqe_signal = unsafe { read_cqe(ring_base, 0) }; assert!(cqe_signal.result == 0, "signal should succeed"); assert!(cqe_signal.user_data == 200); let cqe_poll = unsafe { read_cqe(ring_base, 1) }; assert!(cqe_poll.result == 0, "poll should succeed"); assert!( cqe_poll.extra == 0xFF, "poll should return signaled bits, got {:#x}", cqe_poll.extra ); assert!(cqe_poll.user_data == 201); let mut ptable = PROCESSES.lock(); ptable.destroy(pid, &mut BitmapFrameAllocator); drop(ptable); teardown_ring_page(phys); } );