Add repro for connections

.gitignore

@@ -1,4 +1,4 @@
-/target
+**/target
 **/*.rs.bk
 Cargo.lock
 tests/flock/target

tests/connect/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "connect"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]

tests/connect/src/main.rs

@@ -0,0 +1,89 @@
+//! Runs the sparse reuses_connection test to ensure we aren't creating too many
+//! connections as a repro for https://github.com/EmbarkStudios/tame-index/issues/46
+
+fn main() {
+    // Build the binary so we know it's up to date, and so we can use the one
+    // last modified in the target directory
+    assert!(
+        std::process::Command::new("cargo")
+            .args([
+                "test",
+                "--no-run",
+                "--test",
+                "sparse",
+                "--features",
+                "sparse"
+            ])
+            .status()
+            .unwrap()
+            .success(),
+        "failed to build test binary"
+    );
+
+    let mut latest = std::path::PathBuf::new();
+    let mut ts = std::time::SystemTime::UNIX_EPOCH;
+
+    for entry in std::fs::read_dir("target/debug/deps").expect("failed to read deps") {
+        let entry = entry.expect("failed to read entry");
+
+        if !entry
+            .file_name()
+            .as_os_str()
+            .to_str()
+            .unwrap()
+            .starts_with("sparse-")
+        {
+            continue;
+        }
+
+        let md = entry.metadata().expect("failed to get metadata");
+
+        let mt = md.modified().expect("failed to get mod time");
+
+        if mt < ts {
+            continue;
+        }
+
+        latest = entry.path();
+        ts = mt;
+    }
+
+    assert!(
+        std::process::Command::new("strace")
+            .args([
+                "-f",
+                "-e",
+                "trace=connect",
+                "-o",
+                "/tmp/tame-index-connection-trace"
+            ])
+            .arg(latest)
+            .arg("reuses_connection")
+            .status()
+            .unwrap()
+            .success(),
+        "failed to strace test"
+    );
+
+    let trace = std::fs::read_to_string("/tmp/tame-index-connection-trace")
+        .expect("failed to read strace output");
+
+    let connection_counts = trace
+        .lines()
+        .filter(|line| line.contains("connect("))
+        .count();
+
+    // The connection count should be roughly the same as the processor count
+    let stdout = std::process::Command::new("nproc").output().unwrap().stdout;
+
+    let proc_count: usize = std::str::from_utf8(&stdout)
+        .unwrap()
+        .trim()
+        .parse()
+        .unwrap();
+    let max = proc_count + 5;
+    assert!(
+        connection_counts <= max,
+        "connection syscalls ({connection_counts}) should be lower than {max}"
+    );
+}

tests/sparse.rs

@@ -192,26 +192,212 @@ fn parse_modified_response() {
     }
 }
 
-/// Ensure we can actually send a request to crates.io and parse the response
-#[test]
 #[cfg(feature = "sparse")]
-fn end_to_end() {
-    let td = utils::tempdir();
-    let index = crates_io(&td);
-    let lock = &utils::unlocked();
-
-    let client = reqwest::blocking::Client::builder().build().unwrap();
-
-    let rsi = tame_index::index::RemoteSparseIndex::new(index, client);
-
-    let spdx_krate = rsi
-        .krate("spdx".try_into().unwrap(), true, lock)
-        .expect("failed to retrieve spdx")
-        .expect("failed to find spdx");
+mod remote {
+    use super::*;
+
+    /// Ensure we can actually send a request to crates.io and parse the response
+    #[test]
+    fn end_to_end() {
+        let td = utils::tempdir();
+        let index = crates_io(&td);
+        let lock = &utils::unlocked();
+
+        let client = reqwest::blocking::Client::builder().build().unwrap();
+
+        let rsi = tame_index::index::RemoteSparseIndex::new(index, client);
+
+        let spdx_krate = rsi
+            .krate("spdx".try_into().unwrap(), true, lock)
+            .expect("failed to retrieve spdx")
+            .expect("failed to find spdx");
+
+        spdx_krate
+            .versions
+            .iter()
+            .find(|iv| iv.version == "0.10.1")
+            .expect("failed to find expected version");
+    }
 
-    spdx_krate
-        .versions
-        .iter()
-        .find(|iv| iv.version == "0.10.1")
-        .expect("failed to find expected version");
+    /// Reuses connections. This test is intended to be run under strace to
+    /// validate that <n> connections are not being created
+    /// https://github.com/EmbarkStudios/tame-index/issues/46
+    #[test]
+    fn reuses_connection() {
+        // cargo metadata --format-version=1 | jq -r '.packages[].name | "\"\(.)\","'
+        const KRATES: &[&str] = &[
+            "addr2line",
+            "adler",
+            "async-compression",
+            "autocfg",
+            "backtrace",
+            "base64",
+            "bitflags",
+            "bitflags",
+            "bumpalo",
+            "bytes",
+            "camino",
+            "cargo-platform",
+            "cargo_metadata",
+            "cc",
+            "cfg-if",
+            "core-foundation",
+            "core-foundation-sys",
+            "crc32fast",
+            "crossbeam-deque",
+            "crossbeam-epoch",
+            "crossbeam-utils",
+            "either",
+            "encoding_rs",
+            "equivalent",
+            "errno",
+            "fastrand",
+            "flate2",
+            "fnv",
+            "form_urlencoded",
+            "futures-channel",
+            "futures-core",
+            "futures-io",
+            "futures-sink",
+            "futures-task",
+            "futures-util",
+            "getrandom",
+            "gimli",
+            "h2",
+            "hashbrown",
+            "hermit-abi",
+            "home",
+            "http",
+            "http-body",
+            "httparse",
+            "httpdate",
+            "hyper",
+            "hyper-rustls",
+            "idna",
+            "indexmap",
+            "ipnet",
+            "itoa",
+            "js-sys",
+            "libc",
+            "linux-raw-sys",
+            "log",
+            "memchr",
+            "mime",
+            "miniz_oxide",
+            "mio",
+            "num_cpus",
+            "object",
+            "once_cell",
+            "percent-encoding",
+            "pin-project-lite",
+            "pin-utils",
+            "proc-macro2",
+            "quote",
+            "rayon",
+            "rayon-core",
+            "reqwest",
+            "ring",
+            "rustc-demangle",
+            "rustix",
+            "rustls",
+            "rustls-pemfile",
+            "rustls-webpki",
+            "ryu",
+            "sct",
+            "semver",
+            "serde",
+            "serde_derive",
+            "serde_json",
+            "serde_spanned",
+            "serde_urlencoded",
+            "slab",
+            "smol_str",
+            "socket2",
+            "spin",
+            "static_assertions",
+            "syn",
+            "sync_wrapper",
+            "system-configuration",
+            "system-configuration-sys",
+            "tame-index",
+            "tempfile",
+            "thiserror",
+            "thiserror-impl",
+            "tiny-bench",
+            "tinyvec",
+            "tinyvec_macros",
+            "tokio",
+            "tokio-rustls",
+            "tokio-util",
+            "toml",
+            "toml_datetime",
+            "toml_edit",
+            "tower-service",
+            "tracing",
+            "tracing-core",
+            "try-lock",
+            "twox-hash",
+            "unicode-bidi",
+            "unicode-ident",
+            "unicode-normalization",
+            "untrusted",
+            "url",
+            "want",
+            "wasi",
+            "wasm-bindgen",
+            "wasm-bindgen-backend",
+            "wasm-bindgen-futures",
+            "wasm-bindgen-macro",
+            "wasm-bindgen-macro-support",
+            "wasm-bindgen-shared",
+            "web-sys",
+            "webpki-roots",
+            "windows-sys",
+            "windows-sys",
+            "windows-targets",
+            "windows-targets",
+            "windows_aarch64_gnullvm",
+            "windows_aarch64_gnullvm",
+            "windows_aarch64_msvc",
+            "windows_aarch64_msvc",
+            "windows_i686_gnu",
+            "windows_i686_gnu",
+            "windows_i686_msvc",
+            "windows_i686_msvc",
+            "windows_x86_64_gnu",
+            "windows_x86_64_gnu",
+            "windows_x86_64_gnullvm",
+            "windows_x86_64_gnullvm",
+            "windows_x86_64_msvc",
+            "windows_x86_64_msvc",
+            "winnow",
+            "winreg",
+        ];
+
+        let td = utils::tempdir();
+        let index = crates_io(&td);
+        let lock = &utils::unlocked();
+
+        let client = reqwest::blocking::Client::builder().build().unwrap();
+        let rsi = tame_index::index::RemoteSparseIndex::new(index, client);
+
+        let results = rsi.krates(
+            KRATES.into_iter().map(|s| s.to_string()).collect(),
+            false,
+            lock,
+        );
+
+        use std::fmt::Write;
+        let mut errors = String::new();
+
+        for (name, res) in results {
+            match res {
+                Ok(Some(_)) => continue,
+                Ok(None) => writeln!(&mut errors, "{name}:\tfailed to locate").unwrap(),
+                Err(err) => writeln!(&mut errors, "{name}:\t{err}").unwrap(),
+            }
+        }
+
+        assert!(errors.is_empty(), "{errors}");
+    }
 }