lewis.moe
Nothing to see here, move along
1use core::sync::atomic::Ordering;
2
3use crate::cap::cnode;
4use crate::cap::object::ObjectTag;
5use crate::cap::ops;
6use crate::cap::pool::POOL;
7use crate::cap::table::Rights;
8use crate::error::KernelError;
9use crate::ipc::message;
10use crate::ipc::{endpoint, notification};
11use crate::mem::addr;
12use crate::proc::context::IpcMessage;
13use crate::proc::{BlockedReason, PROCESSES};
14use crate::ring::{
15 CompletionEntry, MAX_CQ_ENTRIES, MAX_SQ_ENTRIES, RingHeader, RingIndex, RingOpcode,
16 SubmissionEntry,
17};
18use crate::types::Pid;
19
20const MAX_RING_BATCH: u32 = 16;
21
22fn _assert_from_bytes<T: zerocopy::FromBytes>() {}
23#[allow(dead_code)]
24fn _assert_ring_types_are_zerocopy() {
25 _assert_from_bytes::<SubmissionEntry>();
26 _assert_from_bytes::<CompletionEntry>();
27}
28
29struct UserSnapshot {
30 sq_tail: RingIndex,
31 cq_head: RingIndex,
32}
33
34unsafe fn snapshot_user_fields(ring_base: *const u8) -> UserSnapshot {
35 let header = unsafe { &*(ring_base as *const RingHeader) };
36 UserSnapshot {
37 sq_tail: RingIndex::new(header.sq_tail.load(Ordering::Acquire)),
38 cq_head: RingIndex::new(header.cq_head.load(Ordering::Acquire)),
39 }
40}
41
42unsafe fn commit_header(ring_base: *mut u8, new_sq_head: RingIndex, new_cq_tail: RingIndex) {
43 let header = unsafe { &*(ring_base as *const RingHeader) };
44 header.sq_head.store(new_sq_head.raw(), Ordering::Release);
45 header.cq_tail.store(new_cq_tail.raw(), Ordering::Release);
46}
47
48unsafe fn read_sq_entry(ring_base: *const u8, index: u32) -> SubmissionEntry {
49 let sq_base = unsafe { ring_base.add(super::ring_sq_offset()) };
50 let entry_ptr =
51 unsafe { sq_base.add((index as usize) * core::mem::size_of::<SubmissionEntry>()) };
52 unsafe { core::ptr::read_volatile(entry_ptr as *const SubmissionEntry) }
53}
54
55unsafe fn write_cq_entry(ring_base: *mut u8, index: u32, entry: CompletionEntry) {
56 let cq_base = unsafe { ring_base.add(super::ring_cq_offset()) };
57 let entry_ptr =
58 unsafe { cq_base.add((index as usize) * core::mem::size_of::<CompletionEntry>()) };
59 unsafe { core::ptr::write_volatile(entry_ptr as *mut CompletionEntry, entry) };
60}
61
62
63struct CqeResult {
64 value: i64,
65 extra: u64,
66}
67
68impl CqeResult {
69 const fn ok() -> Self {
70 Self { value: 0, extra: 0 }
71 }
72
73 const fn success(value: i64) -> Self {
74 Self { value, extra: 0 }
75 }
76
77 const fn with_extra(value: i64, extra: u64) -> Self {
78 Self { value, extra }
79 }
80}
81
82fn ring_cap_create(sqe: &SubmissionEntry, pid: Pid) -> Result<CqeResult, KernelError> {
83 let address = sqe.cap_slot as u64;
84 let tag = ObjectTag::try_from(sqe.arg0)?;
85
86 let ptable = PROCESSES.lock();
87 let (cnode_id, cnode_gen, depth) = cnode::cnode_coords(pid, &ptable)?;
88 let mut pool = POOL.lock_after(&ptable);
89 ops::create_via_cnode(&mut pool, cnode_id, cnode_gen, address, depth, tag)
90 .map(|id| CqeResult::success(id.raw() as i64))
91}
92
93fn ring_cap_derive(sqe: &SubmissionEntry, pid: Pid) -> Result<CqeResult, KernelError> {
94 let src_addr = sqe.cap_slot as u64;
95 let dest_addr = sqe.arg0;
96 let rights_mask = Rights::from_bits(sqe.arg1 as u16);
97
98 let ptable = PROCESSES.lock();
99 let (cnode_id, cnode_gen, depth) = cnode::cnode_coords(pid, &ptable)?;
100 let mut pool = POOL.lock_after(&ptable);
101 ops::derive_via_cnode(&mut pool, cnode_id, cnode_gen, src_addr, dest_addr, depth, rights_mask)
102 .map(|()| CqeResult::ok())
103}
104
105fn ring_ipc_send(sqe: &SubmissionEntry, pid: Pid) -> Result<CqeResult, KernelError> {
106 let address = sqe.cap_slot as u64;
107 let msg = IpcMessage::from_regs([sqe.arg0, sqe.arg1, sqe.arg2, 0, 0, 0]);
108
109 let mut ptable = PROCESSES.lock();
110 let cap = {
111 let pool = POOL.lock_after(&ptable);
112 cnode::resolve_caller_validate(
113 pid, address, ObjectTag::Endpoint, Rights::WRITE, &ptable, &pool,
114 )?
115 };
116
117 ptable[pid].ipc_message = msg;
118
119 let recv_pid = {
120 let mut pool = POOL.lock_after(&ptable);
121 let ep = pool
122 .get_mut(cap.object_id(), cap.generation())?
123 .as_endpoint_mut()?;
124 let dequeued = endpoint::dequeue_genuine_receiver(&mut ep.receivers, &mut ptable);
125 if let Some(ref br) = dequeued {
126 ep.holder = Some(br.pid());
127 }
128 dequeued
129 };
130
131 match recv_pid {
132 Some(blocked_recv) => {
133 let recv_pid_val = blocked_recv.pid();
134 let sender_prio = crate::sched::effective_priority_of(&ptable[pid]);
135
136 let recv = &mut ptable[recv_pid_val];
137 recv.ipc_message = msg;
138 recv.ipc_badge = pid.raw() as u64;
139 recv.unblock(blocked_recv)?;
140 message::inject_into_context(&mut recv.saved_context, &msg);
141 recv.saved_context.rax = pid.raw() as u64;
142 recv.seal_context();
143
144 crate::sched::boost_effective(&mut ptable, recv_pid_val, sender_prio);
145
146 Ok(CqeResult::ok())
147 }
148 None => Err(KernelError::WouldBlock),
149 }
150}
151
152fn ring_ipc_recv(sqe: &SubmissionEntry, pid: Pid) -> Result<CqeResult, KernelError> {
153 let address = sqe.cap_slot as u64;
154
155 let mut ptable = PROCESSES.lock();
156 let cap = {
157 let pool = POOL.lock_after(&ptable);
158 cnode::resolve_caller_validate(
159 pid, address, ObjectTag::Endpoint, Rights::READ, &ptable, &pool,
160 )?
161 };
162
163 let sender_pid = {
164 let mut pool = POOL.lock_after(&ptable);
165 let ep = pool
166 .get_mut(cap.object_id(), cap.generation())?
167 .as_endpoint_mut()?;
168 let dequeued = endpoint::dequeue(&mut ep.senders, &mut ptable);
169 if dequeued.is_some() {
170 ep.holder = Some(pid);
171 }
172 dequeued
173 };
174
175 match sender_pid {
176 Some(blocked_sender) => {
177 let sender_pid = blocked_sender.pid();
178 let sender_msg = ptable[sender_pid].ipc_message;
179 ptable[pid].ipc_message = sender_msg;
180 ptable[pid].ipc_badge = sender_pid.raw() as u64;
181
182 match ptable[sender_pid].blocked_reason() {
183 Some(BlockedReason::Calling(_, _)) => {
184 let mut pool = POOL.lock_after(&ptable);
185 match pool
186 .get_mut(cap.object_id(), cap.generation())
187 .and_then(|d| d.as_endpoint_mut())
188 {
189 Ok(ep) => {
190 match endpoint::enqueue(&mut ep.receivers, blocked_sender, &mut ptable)
191 {
192 Ok(()) => {
193 ptable[pid].reply_target = Some(sender_pid);
194 }
195 Err(_) => {
196 ptable[sender_pid].saved_context.rax =
197 crate::error::KernelError::ResourceExhausted.to_errno()
198 as u64;
199 ptable[sender_pid].seal_context();
200 {
201 let proof = ptable[sender_pid].blocked_proof();
202 let r = ptable[sender_pid].unblock(proof);
203 debug_assert!(r.is_ok());
204 }
205 }
206 }
207 }
208 Err(e) => {
209 ptable[sender_pid].saved_context.rax = e.to_errno() as u64;
210 ptable[sender_pid].seal_context();
211 {
212 let r = ptable[sender_pid].unblock(blocked_sender);
213 debug_assert!(r.is_ok());
214 }
215 }
216 }
217 }
218 _ => {
219 ptable[sender_pid].unblock(blocked_sender)?;
220 }
221 }
222
223 let sender_prio = crate::sched::effective_priority_of(&ptable[sender_pid]);
224 crate::sched::reset_effective(&mut ptable, pid);
225 crate::sched::boost_effective(&mut ptable, pid, sender_prio);
226
227 Ok(CqeResult::with_extra(
228 sender_pid.raw() as i64,
229 sender_msg.regs[0],
230 ))
231 }
232 None => Err(KernelError::WouldBlock),
233 }
234}
235
236fn ring_notify_signal(sqe: &SubmissionEntry, pid: Pid) -> Result<CqeResult, KernelError> {
237 let address = sqe.cap_slot as u64;
238 let bits = sqe.arg0;
239
240 let mut ptable = PROCESSES.lock();
241 let cap = {
242 let pool = POOL.lock_after(&ptable);
243 cnode::resolve_caller_validate(
244 pid, address, ObjectTag::Notification, Rights::WRITE, &ptable, &pool,
245 )?
246 };
247
248 notification::do_signal(&cap, bits, &mut ptable).map(|_| CqeResult::ok())
249}
250
251fn ring_notify_poll(sqe: &SubmissionEntry, pid: Pid) -> Result<CqeResult, KernelError> {
252 let address = sqe.cap_slot as u64;
253
254 let ptable = PROCESSES.lock();
255 let cap = {
256 let pool = POOL.lock_after(&ptable);
257 cnode::resolve_caller_validate(
258 pid, address, ObjectTag::Notification, Rights::READ, &ptable, &pool,
259 )?
260 };
261
262 drop(ptable);
263 notification::do_poll(&cap).map(|val| CqeResult::with_extra(0, val))
264}
265
266fn process_submission(sqe: &SubmissionEntry, pid: Pid) -> CompletionEntry {
267 let result = match RingOpcode::from_u8(sqe.opcode) {
268 Some(RingOpcode::Nop) => Ok(CqeResult::ok()),
269 Some(RingOpcode::CapCreate) => ring_cap_create(sqe, pid),
270 Some(RingOpcode::CapDerive) => ring_cap_derive(sqe, pid),
271 Some(RingOpcode::IpcSend) => ring_ipc_send(sqe, pid),
272 Some(RingOpcode::IpcRecv) => ring_ipc_recv(sqe, pid),
273 Some(RingOpcode::NotifySignal) => ring_notify_signal(sqe, pid),
274 Some(RingOpcode::NotifyPoll) => ring_notify_poll(sqe, pid),
275 None => Err(KernelError::InvalidParameter),
276 };
277
278 match result {
279 Ok(cqe) => CompletionEntry {
280 result: cqe.value,
281 user_data: sqe.user_data as u64,
282 extra: cqe.extra,
283 },
284 Err(e) => CompletionEntry {
285 result: e.to_errno(),
286 user_data: sqe.user_data as u64,
287 extra: 0,
288 },
289 }
290}
291
292pub fn ring_enter(
293 ring_phys_base: x86_64::PhysAddr,
294 pid: Pid,
295 _min_complete: u32,
296) -> Result<i64, KernelError> {
297 if !ring_phys_base.as_u64().is_multiple_of(4096) {
298 return Err(KernelError::InvalidAddress);
299 }
300
301 let max_phys = (crate::mem::phys::BitmapFrameAllocator::total_frames() as u64) * 4096;
302 let ring_end = ring_phys_base
303 .as_u64()
304 .checked_add(super::ring_total_size() as u64)
305 .ok_or(KernelError::InvalidAddress)?;
306 if ring_end > max_phys {
307 return Err(KernelError::InvalidAddress);
308 }
309
310 let hhdm = addr::hhdm_offset();
311 let ring_virt = (ring_phys_base.as_u64() + hhdm) as *mut u8;
312
313 let mut ptable = PROCESSES.lock();
314 let proc = ptable.get_mut(pid).ok_or(KernelError::InvalidObject)?;
315
316 let sq_head = proc.ring_sq_head;
317 let cq_tail = proc.ring_cq_tail;
318 let user_snap = unsafe { snapshot_user_fields(ring_virt) };
319
320 let pending = user_snap.sq_tail.distance(sq_head);
321 if pending > MAX_SQ_ENTRIES {
322 return Err(KernelError::InvalidParameter);
323 }
324
325 let cq_used = cq_tail.distance(user_snap.cq_head);
326 let cq_available = MAX_CQ_ENTRIES.saturating_sub(cq_used);
327 let to_process = pending.min(cq_available).min(MAX_RING_BATCH);
328
329 if to_process == 0 {
330 return Ok(0);
331 }
332 drop(ptable);
333
334 let (completed, cq_write) = (0..to_process).fold((0u32, cq_tail), |(done, cq_w), i| {
335 let sq_slot = sq_head.advance(i).slot(MAX_SQ_ENTRIES);
336 let sqe = unsafe { read_sq_entry(ring_virt, sq_slot) };
337 let cqe = process_submission(&sqe, pid);
338
339 let cq_slot = cq_w.slot(MAX_CQ_ENTRIES);
340 unsafe { write_cq_entry(ring_virt, cq_slot, cqe) };
341 (done + 1, cq_w.advance(1))
342 });
343
344 let new_sq_head = sq_head.advance(to_process);
345 unsafe { commit_header(ring_virt, new_sq_head, cq_write) };
346
347 let mut ptable = PROCESSES.lock();
348 if let Some(proc) = ptable.get_mut(pid) {
349 proc.ring_sq_head = new_sq_head;
350 proc.ring_cq_tail = cq_write;
351 }
352
353 Ok(completed as i64)
354}
355