From 05ade464f7610879e35ee4216cabd5561e20299c Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Wed, 7 Feb 2024 10:53:38 +0100 Subject: [PATCH] Add repro for connections --- .gitignore | 2 +- tests/connect/Cargo.toml | 7 ++ tests/connect/src/main.rs | 89 +++++++++++++++ tests/sparse.rs | 226 ++++++++++++++++++++++++++++++++++---- 4 files changed, 303 insertions(+), 21 deletions(-) create mode 100644 tests/connect/Cargo.toml create mode 100644 tests/connect/src/main.rs diff --git a/.gitignore b/.gitignore index a6fb271..4ee8a41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/target +**/target **/*.rs.bk Cargo.lock tests/flock/target diff --git a/tests/connect/Cargo.toml b/tests/connect/Cargo.toml new file mode 100644 index 0000000..f958511 --- /dev/null +++ b/tests/connect/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "connect" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] diff --git a/tests/connect/src/main.rs b/tests/connect/src/main.rs new file mode 100644 index 0000000..ccd2b68 --- /dev/null +++ b/tests/connect/src/main.rs @@ -0,0 +1,89 @@ +//! Runs the sparse reuses_connection test to ensure we aren't creating too many +//! connections as a repro for https://github.com/EmbarkStudios/tame-index/issues/46 + +fn main() { + // Build the binary so we know it's up to date, and so we can use the one + // last modified in the target directory + assert!( + std::process::Command::new("cargo") + .args([ + "test", + "--no-run", + "--test", + "sparse", + "--features", + "sparse" + ]) + .status() + .unwrap() + .success(), + "failed to build test binary" + ); + + let mut latest = std::path::PathBuf::new(); + let mut ts = std::time::SystemTime::UNIX_EPOCH; + + for entry in std::fs::read_dir("target/debug/deps").expect("failed to read deps") { + let entry = entry.expect("failed to read entry"); + + if !entry + .file_name() + .as_os_str() + .to_str() + .unwrap() + .starts_with("sparse-") + { + continue; + } + + let md = entry.metadata().expect("failed to get metadata"); + + let mt = md.modified().expect("failed to get mod time"); + + if mt < ts { + continue; + } + + latest = entry.path(); + ts = mt; + } + + assert!( + std::process::Command::new("strace") + .args([ + "-f", + "-e", + "trace=connect", + "-o", + "/tmp/tame-index-connection-trace" + ]) + .arg(latest) + .arg("reuses_connection") + .status() + .unwrap() + .success(), + "failed to strace test" + ); + + let trace = std::fs::read_to_string("/tmp/tame-index-connection-trace") + .expect("failed to read strace output"); + + let connection_counts = trace + .lines() + .filter(|line| line.contains("connect(")) + .count(); + + // The connection count should be roughly the same as the processor count + let stdout = std::process::Command::new("nproc").output().unwrap().stdout; + + let proc_count: usize = std::str::from_utf8(&stdout) + .unwrap() + .trim() + .parse() + .unwrap(); + let max = proc_count + 5; + assert!( + connection_counts <= max, + "connection syscalls ({connection_counts}) should be lower than {max}" + ); +} diff --git a/tests/sparse.rs b/tests/sparse.rs index 4b38d6d..f065d0d 100644 --- a/tests/sparse.rs +++ b/tests/sparse.rs @@ -192,26 +192,212 @@ fn parse_modified_response() { } } -/// Ensure we can actually send a request to crates.io and parse the response -#[test] #[cfg(feature = "sparse")] -fn end_to_end() { - let td = utils::tempdir(); - let index = crates_io(&td); - let lock = &utils::unlocked(); - - let client = reqwest::blocking::Client::builder().build().unwrap(); - - let rsi = tame_index::index::RemoteSparseIndex::new(index, client); - - let spdx_krate = rsi - .krate("spdx".try_into().unwrap(), true, lock) - .expect("failed to retrieve spdx") - .expect("failed to find spdx"); +mod remote { + use super::*; + + /// Ensure we can actually send a request to crates.io and parse the response + #[test] + fn end_to_end() { + let td = utils::tempdir(); + let index = crates_io(&td); + let lock = &utils::unlocked(); + + let client = reqwest::blocking::Client::builder().build().unwrap(); + + let rsi = tame_index::index::RemoteSparseIndex::new(index, client); + + let spdx_krate = rsi + .krate("spdx".try_into().unwrap(), true, lock) + .expect("failed to retrieve spdx") + .expect("failed to find spdx"); + + spdx_krate + .versions + .iter() + .find(|iv| iv.version == "0.10.1") + .expect("failed to find expected version"); + } - spdx_krate - .versions - .iter() - .find(|iv| iv.version == "0.10.1") - .expect("failed to find expected version"); + /// Reuses connections. This test is intended to be run under strace to + /// validate that connections are not being created + /// https://github.com/EmbarkStudios/tame-index/issues/46 + #[test] + fn reuses_connection() { + // cargo metadata --format-version=1 | jq -r '.packages[].name | "\"\(.)\","' + const KRATES: &[&str] = &[ + "addr2line", + "adler", + "async-compression", + "autocfg", + "backtrace", + "base64", + "bitflags", + "bitflags", + "bumpalo", + "bytes", + "camino", + "cargo-platform", + "cargo_metadata", + "cc", + "cfg-if", + "core-foundation", + "core-foundation-sys", + "crc32fast", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-utils", + "either", + "encoding_rs", + "equivalent", + "errno", + "fastrand", + "flate2", + "fnv", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", + "getrandom", + "gimli", + "h2", + "hashbrown", + "hermit-abi", + "home", + "http", + "http-body", + "httparse", + "httpdate", + "hyper", + "hyper-rustls", + "idna", + "indexmap", + "ipnet", + "itoa", + "js-sys", + "libc", + "linux-raw-sys", + "log", + "memchr", + "mime", + "miniz_oxide", + "mio", + "num_cpus", + "object", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "proc-macro2", + "quote", + "rayon", + "rayon-core", + "reqwest", + "ring", + "rustc-demangle", + "rustix", + "rustls", + "rustls-pemfile", + "rustls-webpki", + "ryu", + "sct", + "semver", + "serde", + "serde_derive", + "serde_json", + "serde_spanned", + "serde_urlencoded", + "slab", + "smol_str", + "socket2", + "spin", + "static_assertions", + "syn", + "sync_wrapper", + "system-configuration", + "system-configuration-sys", + "tame-index", + "tempfile", + "thiserror", + "thiserror-impl", + "tiny-bench", + "tinyvec", + "tinyvec_macros", + "tokio", + "tokio-rustls", + "tokio-util", + "toml", + "toml_datetime", + "toml_edit", + "tower-service", + "tracing", + "tracing-core", + "try-lock", + "twox-hash", + "unicode-bidi", + "unicode-ident", + "unicode-normalization", + "untrusted", + "url", + "want", + "wasi", + "wasm-bindgen", + "wasm-bindgen-backend", + "wasm-bindgen-futures", + "wasm-bindgen-macro", + "wasm-bindgen-macro-support", + "wasm-bindgen-shared", + "web-sys", + "webpki-roots", + "windows-sys", + "windows-sys", + "windows-targets", + "windows-targets", + "windows_aarch64_gnullvm", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", + "windows_x86_64_msvc", + "winnow", + "winreg", + ]; + + let td = utils::tempdir(); + let index = crates_io(&td); + let lock = &utils::unlocked(); + + let client = reqwest::blocking::Client::builder().build().unwrap(); + let rsi = tame_index::index::RemoteSparseIndex::new(index, client); + + let results = rsi.krates( + KRATES.into_iter().map(|s| s.to_string()).collect(), + false, + lock, + ); + + use std::fmt::Write; + let mut errors = String::new(); + + for (name, res) in results { + match res { + Ok(Some(_)) => continue, + Ok(None) => writeln!(&mut errors, "{name}:\tfailed to locate").unwrap(), + Err(err) => writeln!(&mut errors, "{name}:\t{err}").unwrap(), + } + } + + assert!(errors.is_empty(), "{errors}"); + } }