From c65cd8ea3216c53477dfe7a52829ff766e472d34 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 22 Apr 2024 23:46:48 +1000 Subject: [PATCH 01/12] Add back `preadv2` optimization for `try_acquire` on Linux Signed-off-by: Jiahao XU --- src/unix.rs | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/src/unix.rs b/src/unix.rs index 8f3905e..4aaf07f 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -285,6 +285,33 @@ impl Client { pub fn try_acquire(&self) -> io::Result> { let mut buf = [0]; + // On Linux, we can use preadv2 to do non-blocking read, + // even if `O_NONBLOCK` is not set. + // + // TODO: musl libc supports preadv2 since 1.2.5, but `libc` crate + // hasn't yet added it. + #[cfg(target_os = "linux")] + { + let read = self.read().as_raw_fd(); + loop { + match linux::non_blocking_read(read, &mut buf) { + Ok(1) => return Ok(Some(Acquired { byte: buf[0] })), + Ok(_) => { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "early EOF on jobserver pipe", + )) + } + + Err(e) if e.kind() == io::ErrorKind::WouldBlock => return Ok(None), + Err(e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) if e.kind() == io::ErrorKind::Unsupported => break, + + Err(err) => return Err(err), + } + } + } + let (mut fifo, is_non_blocking) = match self { Self::Fifo { file, @@ -368,6 +395,82 @@ impl Client { } } +// This should be available for all linux targets, +// though only [`non_blocking_read`] currently uses it so adding gnu cfg. +#[cfg(target_os = "linux")] +mod linux { + use super::*; + + use libc::{iovec, off_t, ssize_t, syscall, SYS_preadv2}; + + fn cvt_ssize(t: ssize_t) -> io::Result { + if t == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(t) + } + } + + unsafe fn preadv2( + fd: c_int, + iov: *const iovec, + iovcnt: c_int, + offset: off_t, + flags: c_int, + ) -> ssize_t { + #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] + let res = syscall(SYS_preadv2, fd, iov, iovcnt, offset, 0, flags); + + #[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] + let res = syscall(SYS_preadv2, fd, iov, iovcnt, offset, 0, flags); + + #[cfg(not(target_arch = "x86_64"))] + let res = syscall( + SYS_preadv2, + fd, + iov, + iovcnt, + offset as libc::c_long, + ((offset as u64) >> 32) as libc::c_long, + flags, + ); + + res.try_into().unwrap() + } + + pub fn non_blocking_read(fd: c_int, buf: &mut [u8]) -> io::Result { + static IS_NONBLOCKING_READ_UNSUPPORTED: AtomicBool = AtomicBool::new(false); + + if IS_NONBLOCKING_READ_UNSUPPORTED.load(Ordering::Relaxed) { + return Err(io::ErrorKind::Unsupported.into()); + } + + match cvt_ssize(unsafe { + preadv2( + fd, + &iovec { + iov_base: buf.as_ptr() as *mut _, + iov_len: buf.len(), + }, + 1, + -1, + libc::RWF_NOWAIT, + ) + }) { + Ok(cnt) => Ok(cnt.try_into().unwrap()), + Err(err) if err.raw_os_error() == Some(libc::EOPNOTSUPP) => { + IS_NONBLOCKING_READ_UNSUPPORTED.store(true, Ordering::Relaxed); + Err(io::ErrorKind::Unsupported.into()) + } + Err(err) if err.kind() == io::ErrorKind::Unsupported => { + IS_NONBLOCKING_READ_UNSUPPORTED.store(true, Ordering::Relaxed); + Err(err) + } + Err(err) => Err(err), + } + } +} + #[derive(Debug)] pub struct Helper { thread: JoinHandle<()>, From f34c2bc11cb77fd8774caab256f8a40ad6ce32c2 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 22 Apr 2024 23:58:20 +1000 Subject: [PATCH 02/12] SFix compilation on musl: Vendor `RWF_NOWAIT` constant Signed-off-by: Jiahao XU --- src/unix.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/unix.rs b/src/unix.rs index 4aaf07f..e1a45b7 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -403,6 +403,8 @@ mod linux { use libc::{iovec, off_t, ssize_t, syscall, SYS_preadv2}; + const RWF_NOWAIT: c_int = 0x00000008; + fn cvt_ssize(t: ssize_t) -> io::Result { if t == -1 { Err(io::Error::last_os_error()) @@ -454,7 +456,7 @@ mod linux { }, 1, -1, - libc::RWF_NOWAIT, + RWF_NOWAIT, ) }) { Ok(cnt) => Ok(cnt.try_into().unwrap()), From 0d6baba6715e46b15233b13b6515284576aab9b4 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 22 Apr 2024 23:59:43 +1000 Subject: [PATCH 03/12] Enable `preadv2` optimization on android Signed-off-by: Jiahao XU --- src/unix.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index e1a45b7..88fb76b 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -290,7 +290,7 @@ impl Client { // // TODO: musl libc supports preadv2 since 1.2.5, but `libc` crate // hasn't yet added it. - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "android"))] { let read = self.read().as_raw_fd(); loop { @@ -397,7 +397,7 @@ impl Client { // This should be available for all linux targets, // though only [`non_blocking_read`] currently uses it so adding gnu cfg. -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "android"))] mod linux { use super::*; From dd0f271e8fd3d1559df70f243a8a14d3c74c8d09 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 25 Apr 2024 21:34:28 +1000 Subject: [PATCH 04/12] Remove outdated comments Signed-off-by: Jiahao XU --- src/unix.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index 88fb76b..84ece1a 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -287,9 +287,6 @@ impl Client { // On Linux, we can use preadv2 to do non-blocking read, // even if `O_NONBLOCK` is not set. - // - // TODO: musl libc supports preadv2 since 1.2.5, but `libc` crate - // hasn't yet added it. #[cfg(any(target_os = "linux", target_os = "android"))] { let read = self.read().as_raw_fd(); @@ -395,8 +392,6 @@ impl Client { } } -// This should be available for all linux targets, -// though only [`non_blocking_read`] currently uses it so adding gnu cfg. #[cfg(any(target_os = "linux", target_os = "android"))] mod linux { use super::*; From 6ad33aa3d9b0b5c534416efb924383b49b27f235 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 25 Apr 2024 21:43:54 +1000 Subject: [PATCH 05/12] Fix `preadv2` implementation for x86_64 x32 Signed-off-by: Jiahao XU --- src/unix.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/unix.rs b/src/unix.rs index 84ece1a..9de48b2 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -419,7 +419,7 @@ mod linux { let res = syscall(SYS_preadv2, fd, iov, iovcnt, offset, 0, flags); #[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] - let res = syscall(SYS_preadv2, fd, iov, iovcnt, offset, 0, flags); + let res = syscall(SYS_preadv2, fd, iov, iovcnt, offset, flags); #[cfg(not(target_arch = "x86_64"))] let res = syscall( From 5602ef7bd624adc0f35924071ba32aa27289792c Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 25 Apr 2024 22:25:29 +1000 Subject: [PATCH 06/12] Hard code parameter `offset` Signed-off-by: Jiahao XU --- src/unix.rs | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index 9de48b2..e8f1fa5 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -396,7 +396,7 @@ impl Client { mod linux { use super::*; - use libc::{iovec, off_t, ssize_t, syscall, SYS_preadv2}; + use libc::{iovec, ssize_t, syscall, SYS_preadv2}; const RWF_NOWAIT: c_int = 0x00000008; @@ -408,18 +408,12 @@ mod linux { } } - unsafe fn preadv2( - fd: c_int, - iov: *const iovec, - iovcnt: c_int, - offset: off_t, - flags: c_int, - ) -> ssize_t { + unsafe fn preadv2(fd: c_int, iov: *const iovec, iovcnt: c_int, flags: c_int) -> ssize_t { #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] - let res = syscall(SYS_preadv2, fd, iov, iovcnt, offset, 0, flags); + let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1, 0, flags); #[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] - let res = syscall(SYS_preadv2, fd, iov, iovcnt, offset, flags); + let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1, flags); #[cfg(not(target_arch = "x86_64"))] let res = syscall( @@ -427,8 +421,8 @@ mod linux { fd, iov, iovcnt, - offset as libc::c_long, - ((offset as u64) >> 32) as libc::c_long, + -1 as libc::c_long, + ((-1 as u64) >> 32) as libc::c_long, flags, ); @@ -450,7 +444,6 @@ mod linux { iov_len: buf.len(), }, 1, - -1, RWF_NOWAIT, ) }) { From 5cf22b51f7dd01f74323ac6b412db78ed0a2f823 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 25 Apr 2024 22:39:32 +1000 Subject: [PATCH 07/12] Hardcode more values in `preadv2` Signed-off-by: Jiahao XU --- src/unix.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index e8f1fa5..98c48f8 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -398,6 +398,8 @@ mod linux { use libc::{iovec, ssize_t, syscall, SYS_preadv2}; + // TODO: Replace this with libc::RWF_NOWAIT once they have it for musl + // targets const RWF_NOWAIT: c_int = 0x00000008; fn cvt_ssize(t: ssize_t) -> io::Result { @@ -408,12 +410,14 @@ mod linux { } } - unsafe fn preadv2(fd: c_int, iov: *const iovec, iovcnt: c_int, flags: c_int) -> ssize_t { + unsafe fn preadv2(fd: c_int, iov: &iovec) -> ssize_t { + let iovcnt: c_int = 1; + #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] - let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1, 0, flags); + let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1, 0, RWF_NOWAIT); #[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] - let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1, flags); + let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1, RWF_NOWAIT); #[cfg(not(target_arch = "x86_64"))] let res = syscall( @@ -423,7 +427,7 @@ mod linux { iovcnt, -1 as libc::c_long, ((-1 as u64) >> 32) as libc::c_long, - flags, + RWF_NOWAIT, ); res.try_into().unwrap() @@ -443,8 +447,6 @@ mod linux { iov_base: buf.as_ptr() as *mut _, iov_len: buf.len(), }, - 1, - RWF_NOWAIT, ) }) { Ok(cnt) => Ok(cnt.try_into().unwrap()), From 686fe22d9c59d55dc3b63e5451dc21c023a18a45 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 25 Apr 2024 22:42:15 +1000 Subject: [PATCH 08/12] Apply code review change Co-authored-by: Jubilee Signed-off-by: Jiahao XU --- src/unix.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index 98c48f8..7f2efcd 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -450,14 +450,10 @@ mod linux { ) }) { Ok(cnt) => Ok(cnt.try_into().unwrap()), - Err(err) if err.raw_os_error() == Some(libc::EOPNOTSUPP) => { + Err(err) if matches!(err.raw_os_error(), Some(libc::EOPNOTSUPP | libc::ENOSYS)) => { IS_NONBLOCKING_READ_UNSUPPORTED.store(true, Ordering::Relaxed); Err(io::ErrorKind::Unsupported.into()) } - Err(err) if err.kind() == io::ErrorKind::Unsupported => { - IS_NONBLOCKING_READ_UNSUPPORTED.store(true, Ordering::Relaxed); - Err(err) - } Err(err) => Err(err), } } From 181591cd942911f0c2933f6e83022b2db88c4568 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 25 Apr 2024 22:48:25 +1000 Subject: [PATCH 09/12] Fix type of `offset` passed to `preadv2` syscall on x64 Signed-off-by: Jiahao XU --- src/unix.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index 7f2efcd..ba506d8 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -396,7 +396,7 @@ impl Client { mod linux { use super::*; - use libc::{iovec, ssize_t, syscall, SYS_preadv2}; + use libc::{iovec, off_t, ssize_t, syscall, SYS_preadv2}; // TODO: Replace this with libc::RWF_NOWAIT once they have it for musl // targets @@ -414,10 +414,18 @@ mod linux { let iovcnt: c_int = 1; #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] - let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1, 0, RWF_NOWAIT); + let res = syscall( + SYS_preadv2, + fd, + iov, + iovcnt, + -1 as off_t, + 0 as off_t, + RWF_NOWAIT, + ); #[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] - let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1, RWF_NOWAIT); + let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1 as off_t, RWF_NOWAIT); #[cfg(not(target_arch = "x86_64"))] let res = syscall( From 9087cbe151b75aeba20c8ea5f9bfab41bc8066c6 Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Thu, 25 Apr 2024 22:49:16 +1000 Subject: [PATCH 10/12] Mark `preadv2` a safe function to call Signed-off-by: Jiahao XU --- src/unix.rs | 60 +++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index ba506d8..4d48991 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -410,33 +410,37 @@ mod linux { } } - unsafe fn preadv2(fd: c_int, iov: &iovec) -> ssize_t { + fn preadv2(fd: c_int, iov: &iovec) -> ssize_t { let iovcnt: c_int = 1; #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] - let res = syscall( - SYS_preadv2, - fd, - iov, - iovcnt, - -1 as off_t, - 0 as off_t, - RWF_NOWAIT, - ); + let res = unsafe { + syscall( + SYS_preadv2, + fd, + iov, + iovcnt, + -1 as off_t, + 0 as off_t, + RWF_NOWAIT, + ) + }; #[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] - let res = syscall(SYS_preadv2, fd, iov, iovcnt, -1 as off_t, RWF_NOWAIT); + let res = unsafe { syscall(SYS_preadv2, fd, iov, iovcnt, -1 as off_t, RWF_NOWAIT) }; #[cfg(not(target_arch = "x86_64"))] - let res = syscall( - SYS_preadv2, - fd, - iov, - iovcnt, - -1 as libc::c_long, - ((-1 as u64) >> 32) as libc::c_long, - RWF_NOWAIT, - ); + let res = unsafe { + syscall( + SYS_preadv2, + fd, + iov, + iovcnt, + -1 as libc::c_long, + ((-1 as u64) >> 32) as libc::c_long, + RWF_NOWAIT, + ) + }; res.try_into().unwrap() } @@ -448,15 +452,13 @@ mod linux { return Err(io::ErrorKind::Unsupported.into()); } - match cvt_ssize(unsafe { - preadv2( - fd, - &iovec { - iov_base: buf.as_ptr() as *mut _, - iov_len: buf.len(), - }, - ) - }) { + match cvt_ssize(preadv2( + fd, + &iovec { + iov_base: buf.as_ptr() as *mut _, + iov_len: buf.len(), + }, + )) { Ok(cnt) => Ok(cnt.try_into().unwrap()), Err(err) if matches!(err.raw_os_error(), Some(libc::EOPNOTSUPP | libc::ENOSYS)) => { IS_NONBLOCKING_READ_UNSUPPORTED.store(true, Ordering::Relaxed); From 804310fbe45819ff0117fe5056153829fa91096b Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 29 Apr 2024 23:11:49 +1000 Subject: [PATCH 11/12] Refactor: Introduce variable `offset` to avoid hard-coded constants Signed-off-by: Jiahao XU --- src/unix.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index 4d48991..9096a45 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -412,22 +412,13 @@ mod linux { fn preadv2(fd: c_int, iov: &iovec) -> ssize_t { let iovcnt: c_int = 1; + let offset: off_t = -1; #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] - let res = unsafe { - syscall( - SYS_preadv2, - fd, - iov, - iovcnt, - -1 as off_t, - 0 as off_t, - RWF_NOWAIT, - ) - }; + let res = unsafe { syscall(SYS_preadv2, fd, iov, iovcnt, offset, 0 as off_t, RWF_NOWAIT) }; #[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] - let res = unsafe { syscall(SYS_preadv2, fd, iov, iovcnt, -1 as off_t, RWF_NOWAIT) }; + let res = unsafe { syscall(SYS_preadv2, fd, iov, iovcnt, offset, RWF_NOWAIT) }; #[cfg(not(target_arch = "x86_64"))] let res = unsafe { @@ -436,8 +427,8 @@ mod linux { fd, iov, iovcnt, - -1 as libc::c_long, - ((-1 as u64) >> 32) as libc::c_long, + offset as libc::c_long, + ((offset as u64) >> 32) as libc::c_long, RWF_NOWAIT, ) }; From 60cac974b62e10df744382ab6a87e333a2fc95ba Mon Sep 17 00:00:00 2001 From: Jiahao XU Date: Mon, 29 Apr 2024 23:14:21 +1000 Subject: [PATCH 12/12] Refactor: Use `cfg!` instead of `#[cfg]` Signed-off-by: Jiahao XU --- src/unix.rs | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/unix.rs b/src/unix.rs index 9096a45..4d5a80e 100644 --- a/src/unix.rs +++ b/src/unix.rs @@ -414,26 +414,25 @@ mod linux { let iovcnt: c_int = 1; let offset: off_t = -1; - #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] - let res = unsafe { syscall(SYS_preadv2, fd, iov, iovcnt, offset, 0 as off_t, RWF_NOWAIT) }; - - #[cfg(all(target_arch = "x86_64", target_pointer_width = "32"))] - let res = unsafe { syscall(SYS_preadv2, fd, iov, iovcnt, offset, RWF_NOWAIT) }; - - #[cfg(not(target_arch = "x86_64"))] - let res = unsafe { - syscall( - SYS_preadv2, - fd, - iov, - iovcnt, - offset as libc::c_long, - ((offset as u64) >> 32) as libc::c_long, - RWF_NOWAIT, - ) - }; - - res.try_into().unwrap() + if cfg!(all(target_arch = "x86_64", target_pointer_width = "64")) { + unsafe { syscall(SYS_preadv2, fd, iov, iovcnt, offset, 0 as off_t, RWF_NOWAIT) } + } else if cfg!(all(target_arch = "x86_64", target_pointer_width = "32")) { + unsafe { syscall(SYS_preadv2, fd, iov, iovcnt, offset, RWF_NOWAIT) } + } else { + unsafe { + syscall( + SYS_preadv2, + fd, + iov, + iovcnt, + offset as libc::c_long, + ((offset as u64) >> 32) as libc::c_long, + RWF_NOWAIT, + ) + } + } + .try_into() + .unwrap() } pub fn non_blocking_read(fd: c_int, buf: &mut [u8]) -> io::Result {