On 32-bit x86, get the vsyscall address from the AUX vector

On 32-bit x86, get the vsyscall address from the AT_SYSINFO AUX vector
entry, rather than looking it up in the vDSO. This avoids the need to
link in all the vDSO code if it isn't otherwise needed. And, it's
simpler, avoiding the need for the `rustix_int_0x80` function.

Author: sunfishcode

src/backend/linux_raw/arch/mod.rs

@@ -51,11 +51,11 @@ pub(in crate::backend) mod asm;
 ))]
 pub(in crate::backend) use self::asm as choose;
 
-// On 32-bit x86, use vDSO wrappers for all syscalls. We could use the
-// architecture syscall instruction (`int 0x80`), but the vDSO kernel_vsyscall
+// On 32-bit x86, use the kernel_vsyscall mechanism for syscalls. We could use
+// the architecture syscall instruction (`int 0x80`), but the kernel_vsyscall
 // mechanism is much faster.
 #[cfg(target_arch = "x86")]
-pub(in crate::backend) use super::vdso_wrappers::x86_via_vdso as choose;
+pub(in crate::backend) use super::x86_vsyscall as choose;
 
 // This would be the code for always using `int 0x80` on 32-bit x86.
 //#[cfg(target_arch = "x86")]

src/backend/linux_raw/arch/x86.rs

@@ -15,7 +15,7 @@
 use crate::backend::reg::{
     ArgReg, FromAsm, RetReg, SyscallNumber, ToAsm, A0, A1, A2, A3, A4, A5, R0,
 };
-use crate::backend::vdso_wrappers::SyscallType;
+use crate::backend::x86_vsyscall::SyscallType;
 use core::arch::asm;
 
 #[inline]

src/backend/linux_raw/mod.rs

@@ -18,10 +18,12 @@
 mod arch;
 mod conv;
 mod reg;
-#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))]
+#[cfg(any(feature = "time", feature = "process"))]
 mod vdso;
-#[cfg(any(feature = "time", feature = "process", target_arch = "x86"))]
+#[cfg(any(feature = "time", feature = "process"))]
 mod vdso_wrappers;
+#[cfg(target_arch = "x86")]
+mod x86_vsyscall;
 
 #[cfg(feature = "event")]
 pub(crate) mod event;

src/backend/linux_raw/param/auxv.rs

@@ -21,6 +21,8 @@ use core::sync::atomic::AtomicU8;
 use core::sync::atomic::Ordering::Relaxed;
 use core::sync::atomic::{AtomicPtr, AtomicUsize};
 use linux_raw_sys::elf::*;
+#[cfg(target_arch = "x86")]
+use linux_raw_sys::general::AT_SYSINFO;
 use linux_raw_sys::general::{
     AT_BASE, AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_MINSIGSTKSZ, AT_NULL, AT_PAGESZ,
     AT_SYSINFO_EHDR,
@@ -38,8 +40,12 @@ pub(crate) fn page_size() -> usize {
     let mut page_size = PAGE_SIZE.load(Relaxed);
 
     if page_size == 0 {
-        init_auxv();
-        page_size = PAGE_SIZE.load(Relaxed);
+        #[cold]
+        fn compute_page_size() -> usize {
+            init_auxv();
+            PAGE_SIZE.load(Relaxed)
+        }
+        page_size = compute_page_size();
     }
 
     page_size
@@ -51,8 +57,12 @@ pub(crate) fn clock_ticks_per_second() -> u64 {
     let mut ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed);
 
     if ticks == 0 {
-        init_auxv();
-        ticks = CLOCK_TICKS_PER_SECOND.load(Relaxed);
+        #[cold]
+        fn compute_clock_ticks_per_second() -> usize {
+            init_auxv();
+            CLOCK_TICKS_PER_SECOND.load(Relaxed)
+        }
+        ticks = compute_clock_ticks_per_second();
     }
 
     ticks as u64
@@ -65,9 +75,12 @@ pub(crate) fn linux_hwcap() -> (usize, usize) {
     let mut hwcap2 = HWCAP2.load(Relaxed);
 
     if hwcap == 0 || hwcap2 == 0 {
-        init_auxv();
-        hwcap = HWCAP.load(Relaxed);
-        hwcap2 = HWCAP2.load(Relaxed);
+        #[cold]
+        fn compute_linux_hwcap() -> (usize, usize) {
+            init_auxv();
+            (HWCAP.load(Relaxed), HWCAP2.load(Relaxed))
+        }
+        (hwcap, hwcap2) = compute_linux_hwcap();
     }
 
     (hwcap, hwcap2)
@@ -92,8 +105,12 @@ pub(crate) fn linux_execfn() -> &'static CStr {
     let mut execfn = EXECFN.load(Relaxed);
 
     if execfn.is_null() {
-        init_auxv();
-        execfn = EXECFN.load(Relaxed);
+        #[cold]
+        fn compute_linux_execfn() -> *mut c::c_char {
+            init_auxv();
+            EXECFN.load(Relaxed)
+        }
+        execfn = compute_linux_execfn();
     }
 
     // SAFETY: We assume the `AT_EXECFN` value provided by the kernel is a
@@ -108,8 +125,12 @@ pub(crate) fn linux_secure() -> bool {
 
     // 0 means not initialized yet.
     if secure == 0 {
-        init_auxv();
-        secure = SECURE.load(Relaxed);
+        #[cold]
+        fn compute_linux_secure() -> u8 {
+            init_auxv();
+            SECURE.load(Relaxed)
+        }
+        secure = compute_linux_secure();
     }
 
     // 0 means not present. Libc `getauxval(AT_SECURE)` would return 0.
@@ -125,11 +146,13 @@ pub(crate) fn exe_phdrs() -> (*const c::c_void, usize, usize) {
     let mut phent = PHENT.load(Relaxed);
     let mut phnum = PHNUM.load(Relaxed);
 
-    if phdr.is_null() || phnum == 0 {
-        init_auxv();
-        phdr = PHDR.load(Relaxed);
-        phent = PHENT.load(Relaxed);
-        phnum = PHNUM.load(Relaxed);
+    if phdr.is_null() || phent == 0 || phnum == 0 {
+        #[cold]
+        fn compute_exe_phdrs() -> (*mut Elf_Phdr, usize, usize) {
+            init_auxv();
+            (PHDR.load(Relaxed), PHENT.load(Relaxed), PHNUM.load(Relaxed))
+        }
+        (phdr, phent, phnum) = compute_exe_phdrs();
     }
 
     (phdr.cast(), phent, phnum)
@@ -145,12 +168,16 @@ pub(in super::super) fn sysinfo_ehdr() -> *const Elf_Ehdr {
     let mut ehdr = SYSINFO_EHDR.load(Relaxed);
 
     if ehdr.is_null() {
-        // Use `maybe_init_auxv` to to read the aux vectors if it can, but do
-        // nothing if it can't. If it can't, then we'll get a null pointer
-        // here, which our callers are prepared to deal with.
-        maybe_init_auxv();
+        #[cold]
+        fn compute_sysinfo_ehdr() -> *mut Elf_Ehdr {
+            // Use `maybe_init_auxv` to to read the aux vectors if it can, but do
+            // nothing if it can't. If it can't, then we'll get a null pointer
+            // here, which our callers are prepared to deal with.
+            maybe_init_auxv();
+            SYSINFO_EHDR.load(Relaxed)
+        }
 
-        ehdr = SYSINFO_EHDR.load(Relaxed);
+        ehdr = compute_sysinfo_ehdr();
     }
 
     ehdr
@@ -162,8 +189,12 @@ pub(crate) fn entry() -> usize {
     let mut entry = ENTRY.load(Relaxed);
 
     if entry == 0 {
-        init_auxv();
-        entry = ENTRY.load(Relaxed);
+        #[cold]
+        fn compute_entry() -> usize {
+            init_auxv();
+            ENTRY.load(Relaxed)
+        }
+        entry = compute_entry();
     }
 
     entry
@@ -175,13 +206,34 @@ pub(crate) fn random() -> *const [u8; 16] {
     let mut random = RANDOM.load(Relaxed);
 
     if random.is_null() {
-        init_auxv();
-        random = RANDOM.load(Relaxed);
+        #[cold]
+        fn compute_random() -> *mut [u8; 16] {
+            init_auxv();
+            RANDOM.load(Relaxed)
+        }
+        random = compute_random();
     }
 
     random
 }
 
+#[cfg(target_arch = "x86")]
+#[inline]
+pub(crate) fn vsyscall() -> *const c::c_void {
+    let mut vsyscall = VSYSCALL.load(Relaxed);
+
+    if vsyscall.is_null() {
+        #[cold]
+        fn compute_vsyscall() -> *const c::c_void {
+            init_auxv();
+            VSYSCALL.load(Relaxed)
+        }
+        vsyscall = compute_vsyscall();
+    }
+
+    vsyscall
+}
+
 static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
 static CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0);
 static HWCAP: AtomicUsize = AtomicUsize::new(0);
@@ -201,6 +253,8 @@ static PHNUM: AtomicUsize = AtomicUsize::new(0);
 static ENTRY: AtomicUsize = AtomicUsize::new(0);
 #[cfg(feature = "runtime")]
 static RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(null_mut());
+#[cfg(feature = "x86")]
+static VSYSCALL: AtomicPtr<c::c_void> = AtomicPtr::new(null_mut());
 
 const PR_GET_AUXV: c::c_int = 0x4155_5856;
 
@@ -389,6 +443,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator<Item = Elf_auxv_t>) -> Opti
     let mut egid = None;
     #[cfg(feature = "runtime")]
     let mut random = null_mut();
+    #[cfg(target_arch = "x86")]
+    let mut vsyscall = null_mut();
 
     for Elf_auxv_t { a_type, a_val } in aux_iter {
         match a_type as _ {
@@ -428,6 +484,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator<Item = Elf_auxv_t>) -> Opti
             AT_ENTRY => entry = a_val as usize,
             #[cfg(feature = "runtime")]
             AT_RANDOM => random = check_raw_pointer::<[u8; 16]>(a_val as *mut _)?.as_ptr(),
+            #[cfg(target_arch = "x86")]
+            AT_SYSINFO => vsyscall = a_val.cast(),
 
             AT_NULL => break,
             _ => (),
@@ -464,6 +522,8 @@ unsafe fn init_from_aux_iter(aux_iter: impl Iterator<Item = Elf_auxv_t>) -> Opti
     ENTRY.store(entry, Relaxed);
     #[cfg(feature = "runtime")]
     RANDOM.store(random, Relaxed);
+    #[cfg(target_arch = "x86")]
+    VSYSCALL.store(vsyscall, Relaxed);
 
     Some(())
 }

src/backend/linux_raw/param/init.rs

@@ -14,6 +14,8 @@ use core::ptr::{null_mut, read, NonNull};
 use core::sync::atomic::AtomicBool;
 use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
 use linux_raw_sys::elf::*;
+#[cfg(target_arch = "x86")]
+use linux_raw_sys::general::AT_SYSINFO;
 use linux_raw_sys::general::{
     AT_CLKTCK, AT_EXECFN, AT_HWCAP, AT_HWCAP2, AT_MINSIGSTKSZ, AT_NULL, AT_PAGESZ, AT_SYSINFO_EHDR,
 };
@@ -96,6 +98,12 @@ pub(crate) fn random() -> *const [u8; 16] {
     unsafe { RANDOM.load(Ordering::Relaxed) }
 }
 
+#[cfg(target_arch = "x86")]
+#[inline]
+pub(crate) fn vsyscall() -> *const c_void {
+    unsafe { VSYSCALL.load(Ordering::Relaxed) }
+}
+
 static mut PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
 static mut CLOCK_TICKS_PER_SECOND: AtomicUsize = AtomicUsize::new(0);
 static mut HWCAP: AtomicUsize = AtomicUsize::new(0);
@@ -118,6 +126,8 @@ static mut PHNUM: AtomicUsize = AtomicUsize::new(0);
 static mut ENTRY: AtomicUsize = AtomicUsize::new(0);
 #[cfg(feature = "runtime")]
 static mut RANDOM: AtomicPtr<[u8; 16]> = AtomicPtr::new(NonNull::dangling().as_ptr());
+#[cfg(target_arch = "x86")]
+static mut VSYSCALL: AtomicPtr<c_void> = AtomicPtr::new(NonNull::dangling().as_ptr());
 
 /// When "use-explicitly-provided-auxv" is enabled, we export a function to be
 /// called during initialization, and passed a pointer to the original
@@ -170,6 +180,8 @@ unsafe fn init_from_auxp(mut auxp: *const Elf_auxv_t) {
             AT_ENTRY => ENTRY.store(a_val as usize, Ordering::Relaxed),
             #[cfg(feature = "runtime")]
             AT_RANDOM => RANDOM.store(a_val.cast::<[u8; 16]>(), Ordering::Relaxed),
+            #[cfg(feature = "x86")]
+            AT_SYSINFO => VSYSCALL.store(a_val.cast::<c_void>(), Ordering::Relaxed),
 
             AT_NULL => break,
             _ => (),

src/backend/linux_raw/param/libc_auxv.rs

@@ -11,6 +11,11 @@ use crate::ffi::CStr;
 #[cfg(not(feature = "runtime"))]
 use core::ptr::null;
 use linux_raw_sys::elf::*;
+#[cfg(target_arch = "x86")]
+use {
+    core::ffi::c_void, core::ptr::null_mut, core::sync::atomic::AtomicPtr,
+    core::sync::atomic::Ordering::Relaxed,
+};
 
 // `getauxval` wasn't supported in glibc until 2.16. Also this lets us use
 // `*mut` as the return type to preserve strict provenance.
@@ -38,6 +43,8 @@ const AT_RANDOM: c::c_ulong = 25;
 const AT_HWCAP2: c::c_ulong = 26;
 const AT_SECURE: c::c_ulong = 23;
 const AT_EXECFN: c::c_ulong = 31;
+#[cfg(target_arch = "x86")]
+const AT_SYSINFO: c::c_ulong = 32;
 const AT_SYSINFO_EHDR: c::c_ulong = 33;
 
 // Declare `sysconf` ourselves so that we don't depend on all of libc just for
@@ -72,6 +79,9 @@ fn test_abi() {
     const_assert_eq!(self::AT_ENTRY, ::libc::AT_ENTRY);
     #[cfg(feature = "runtime")]
     const_assert_eq!(self::AT_RANDOM, ::libc::AT_RANDOM);
+    // TODO: Upstream x86's `AT_SYSINFO` to libc.
+    #[cfg(target_arch = "x86")]
+    const_assert_eq!(self::AT_SYSINFO, ::linux_raw_sys::general::AT_SYSINFO);
 }
 
 #[cfg(feature = "param")]
@@ -192,3 +202,25 @@ pub(crate) fn entry() -> usize {
 pub(crate) fn random() -> *const [u8; 16] {
     unsafe { getauxval(AT_RANDOM) as *const [u8; 16] }
 }
+
+#[cfg(target_arch = "x86")]
+#[inline]
+pub(crate) fn vsyscall() -> *const c_void {
+    // We call this for every system call, so memoize the value.
+    static VSYSCALL: AtomicPtr<c_void> = AtomicPtr::new(null_mut());
+
+    let mut vsyscall = VSYSCALL.load(Relaxed);
+
+    if vsyscall.is_null() {
+        #[cold]
+        fn compute_vsyscall() -> *mut c_void {
+            let vsyscall = unsafe { getauxval(AT_SYSINFO) } as *mut c_void;
+            VSYSCALL.store(vsyscall, Relaxed);
+            vsyscall
+        }
+
+        vsyscall = compute_vsyscall();
+    }
+
+    vsyscall
+}