Skip to content

Commit 86bb137

Browse files
FrankReholi-obk
authored andcommitted
epoll: add vector clock to the epoll ready_list
This adds a VClock to the epoll implementation's ready_list and has this VClock synced from the thread that updates an event in the ready_list and then has the VClocks of any threads being made runnable again, out of the calls to epoll_wait, synced from it.
1 parent 1b622f4 commit 86bb137

File tree

4 files changed

+67
-119
lines changed

4 files changed

+67
-119
lines changed

src/tools/miri/src/shims/unix/linux/epoll.rs

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use std::io;
44
use std::rc::{Rc, Weak};
55
use std::time::Duration;
66

7+
use crate::concurrency::VClock;
78
use crate::shims::unix::fd::{FdId, FileDescriptionRef, WeakFileDescriptionRef};
89
use crate::shims::unix::*;
910
use crate::*;
@@ -19,7 +20,7 @@ struct Epoll {
1920
/// and file descriptor value.
2021
// This is an Rc because EpollInterest need to hold a reference to update
2122
// it.
22-
ready_list: Rc<RefCell<BTreeMap<(FdId, i32), EpollEventInstance>>>,
23+
ready_list: Rc<ReadyList>,
2324
/// A list of thread ids blocked on this epoll instance.
2425
thread_id: RefCell<Vec<ThreadId>>,
2526
}
@@ -63,7 +64,7 @@ pub struct EpollEventInterest {
6364
/// <https://man7.org/linux/man-pages/man3/epoll_event.3type.html>
6465
data: u64,
6566
/// Ready list of the epoll instance under which this EpollEventInterest is registered.
66-
ready_list: Rc<RefCell<BTreeMap<(FdId, i32), EpollEventInstance>>>,
67+
ready_list: Rc<ReadyList>,
6768
/// The epoll file description that this EpollEventInterest is registered under.
6869
weak_epfd: WeakFileDescriptionRef,
6970
}
@@ -88,6 +89,12 @@ pub struct EpollReadyEvents {
8889
pub epollerr: bool,
8990
}
9091

92+
#[derive(Debug, Default)]
93+
struct ReadyList {
94+
mapping: RefCell<BTreeMap<(FdId, i32), EpollEventInstance>>,
95+
clock: RefCell<VClock>,
96+
}
97+
9198
impl EpollReadyEvents {
9299
pub fn new() -> Self {
93100
EpollReadyEvents {
@@ -127,7 +134,7 @@ impl EpollReadyEvents {
127134
}
128135

129136
impl Epoll {
130-
fn get_ready_list(&self) -> Rc<RefCell<BTreeMap<(FdId, i32), EpollEventInstance>>> {
137+
fn get_ready_list(&self) -> Rc<ReadyList> {
131138
Rc::clone(&self.ready_list)
132139
}
133140
}
@@ -374,7 +381,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
374381
drop(epoll_interest);
375382

376383
// Remove related epoll_interest from ready list.
377-
ready_list.borrow_mut().remove(&epoll_key);
384+
ready_list.mapping.borrow_mut().remove(&epoll_key);
378385

379386
// Remove dangling EpollEventInterest from its global table.
380387
// .unwrap() below should succeed because the file description id must have registered
@@ -469,7 +476,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
469476
.downcast::<Epoll>()
470477
.ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))?;
471478
let binding = epoll_file_description.get_ready_list();
472-
ready_list_empty = binding.borrow_mut().is_empty();
479+
ready_list_empty = binding.mapping.borrow_mut().is_empty();
473480
thread_ids = epoll_file_description.thread_id.borrow_mut();
474481
}
475482
if timeout == 0 || !ready_list_empty {
@@ -558,9 +565,15 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
558565
// holds a strong ref to epoll_interest.
559566
let epfd = epoll_interest.borrow().weak_epfd.upgrade().unwrap();
560567
// FIXME: We can randomly pick a thread to unblock.
561-
if let Some(thread_id) =
562-
epfd.downcast::<Epoll>().unwrap().thread_id.borrow_mut().pop()
563-
{
568+
569+
let epoll = epfd.downcast::<Epoll>().unwrap();
570+
571+
// Synchronize running thread to the epoll ready list.
572+
if let Some(clock) = &this.release_clock() {
573+
epoll.ready_list.clock.borrow_mut().join(clock);
574+
}
575+
576+
if let Some(thread_id) = epoll.thread_id.borrow_mut().pop() {
564577
waiter.push(thread_id);
565578
};
566579
}
@@ -614,7 +627,7 @@ fn check_and_update_one_event_interest<'tcx>(
614627
// insert an epoll_return to the ready list.
615628
if flags != 0 {
616629
let epoll_key = (id, epoll_event_interest.fd_num);
617-
let ready_list = &mut epoll_event_interest.ready_list.borrow_mut();
630+
let ready_list = &mut epoll_event_interest.ready_list.mapping.borrow_mut();
618631
let event_instance = EpollEventInstance::new(flags, epoll_event_interest.data);
619632
// Triggers the notification by inserting it to the ready list.
620633
ready_list.insert(epoll_key, event_instance);
@@ -641,7 +654,11 @@ fn blocking_epoll_callback<'tcx>(
641654
.ok_or_else(|| err_unsup_format!("non-epoll FD passed to `epoll_wait`"))?;
642655

643656
let ready_list = epoll_file_description.get_ready_list();
644-
let mut ready_list = ready_list.borrow_mut();
657+
658+
// Synchronize waking thread from the epoll ready list.
659+
ecx.acquire_clock(&ready_list.clock.borrow());
660+
661+
let mut ready_list = ready_list.mapping.borrow_mut();
645662
let mut num_of_events: i32 = 0;
646663
let mut array_iter = ecx.project_array_fields(events)?;
647664

src/tools/miri/tests/fail-dep/libc/libc-epoll-blocking.rs

Lines changed: 0 additions & 79 deletions
This file was deleted.

src/tools/miri/tests/fail-dep/libc/libc-epoll-blocking.stderr

Lines changed: 0 additions & 29 deletions
This file was deleted.

src/tools/miri/tests/pass-dep/libc/libc-epoll-blocking.rs

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
//@only-target: linux
2-
// test_epoll_block_then_unblock depends on a deterministic schedule.
2+
// test_epoll_block_then_unblock and test_epoll_race depend on a deterministic schedule.
33
//@compile-flags: -Zmiri-preemption-rate=0
44

55
use std::convert::TryInto;
@@ -12,6 +12,7 @@ fn main() {
1212
test_epoll_block_without_notification();
1313
test_epoll_block_then_unblock();
1414
test_notification_after_timeout();
15+
test_epoll_race();
1516
}
1617

1718
// Using `as` cast since `EPOLLET` wraps around
@@ -137,3 +138,41 @@ fn test_notification_after_timeout() {
137138
let expected_value = fds[0] as u64;
138139
check_epoll_wait::<1>(epfd, &[(expected_event, expected_value)], 10);
139140
}
141+
142+
// This test shows a data_race before epoll had vector clocks added.
143+
fn test_epoll_race() {
144+
// Create an epoll instance.
145+
let epfd = unsafe { libc::epoll_create1(0) };
146+
assert_ne!(epfd, -1);
147+
148+
// Create an eventfd instance.
149+
let flags = libc::EFD_NONBLOCK | libc::EFD_CLOEXEC;
150+
let fd = unsafe { libc::eventfd(0, flags) };
151+
152+
// Register eventfd with the epoll instance.
153+
let mut ev = libc::epoll_event { events: EPOLL_IN_OUT_ET, u64: fd as u64 };
154+
let res = unsafe { libc::epoll_ctl(epfd, libc::EPOLL_CTL_ADD, fd, &mut ev) };
155+
assert_eq!(res, 0);
156+
157+
static mut VAL: u8 = 0;
158+
let thread1 = thread::spawn(move || {
159+
// Write to the static mut variable.
160+
unsafe { VAL = 1 };
161+
// Write to the eventfd instance.
162+
let sized_8_data: [u8; 8] = 1_u64.to_ne_bytes();
163+
let res = unsafe { libc::write(fd, sized_8_data.as_ptr() as *const libc::c_void, 8) };
164+
// read returns number of bytes that have been read, which is always 8.
165+
assert_eq!(res, 8);
166+
});
167+
thread::yield_now();
168+
// epoll_wait for the event to happen.
169+
let expected_event = u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).unwrap();
170+
let expected_value = u64::try_from(fd).unwrap();
171+
check_epoll_wait::<8>(epfd, &[(expected_event, expected_value)], -1);
172+
// Read from the static mut variable.
173+
#[allow(static_mut_refs)]
174+
unsafe {
175+
assert_eq!(VAL, 1)
176+
};
177+
thread1.join().unwrap();
178+
}

0 commit comments

Comments
 (0)