From 8f90c1fbf20d2d8848a22fb72dd62a8f01af04b0 Mon Sep 17 00:00:00 2001 From: Cormac Relf Date: Tue, 7 Jan 2025 19:10:20 -0800 Subject: [PATCH] Middle-ground vendoring option using a local registry (#46) Summary: The problem is a goldilocks one. 1. `vendor = false` is quick & easy to manage deps & buckify, but pretty bad day to day. - Doesn't work offline as buck2 issues HEAD requests constantly - Terrible DX annoyance with "too many open files" errors due to buck trying to download 1000 crates at once. The standard start to your day looks like "run buck2 build about 15 times until by random chance the scheduler manages to get past those errors" - Those crates get downloaded again, and again, and again - `reindeer buckify` takes 2 seconds or so. Pretty convenient. 2. `[vendor] ...` is slow to manage deps & buckify. - Neat for small projects - Also probably neat for Meta with y'all's funky EdenFS etc. - But middle ground is bad - Middle = `vendor` directory of 1000 crates, 1.2 GB, 50k source files. Mostly from dupes of the windows crates which can't be pinned to one single version etc. - `reindeer vendor` takes 35 seconds - `reindeer buckify` takes 20 seconds - `git status` takes 150ms - The vendor folder wrecks git performance simply by its existence. - Build experience is perfect, works offline, etc. I think we need a solution for the middle ground: - `vendor = "local-registry"`, using https://github.com/dhovart/cargo-local-registry - `reindeer vendor` ultimately just writes a bunch of .crate files into vendor/, which are just gzipped tarballs - .crate files stored in git, but using git-lfs if you like. Suddenly `windows-0.48.0.crate` is just a blob. Your diffs are much simpler when you modify deps. Etc. - Some buck2 rule to extract them. (There is no prelude rule that can do this with `strip_prefix` and `sub_targets` support, but prelude's `extract_archive` could probably have that added.) Outcomes: - Offline works (~~although doesn't handle cargo `package = { git = "..." }` deps yet~~). - `reindeer vendor` and `reindeer buckify` both take 2 seconds - `git status` takes 20ms - Buck builds are a compromise, but a pretty great one. It still has to extract the tarballs when you want to build things. But at least buck won't ever actually extract `windows-0.48.0.crate` on linux, and you only pay for what you build. - The DX annoyance factor during builds is back to zero. No more too many open files errors. - DX annoyance when updating deps is acceptable. Problems: - Relies on https://github.com/dhovart/cargo-local-registry being installed. Note however this is a single-file binary. I think if you rewrote it without the dependency on the `cargo` crate it would be maybe a 2-file crate. And we could use it as a library. - I think storing the local registry's `index` folder (nested `ab/ ac/ ah/ ...` folders) might be a little bit annoying if you're making concurrent edits on different branches. But you can always regenerate. Pull Request resolved: https://github.com/facebookincubator/reindeer/pull/46 Reviewed By: zertosh Differential Revision: D67925711 Pulled By: dtolnay fbshipit-source-id: 2203d939cf4381c722abf6ea88e3acaf86a30c24 --- src/buck.rs | 38 +++++++++++++++++ src/buckify.rs | 94 +++++++++++++++++++++++++++++++---------- src/cargo.rs | 3 +- src/config.rs | 53 ++++++++++++++++------- src/fixups.rs | 19 +++++++-- src/main.rs | 10 ++++- src/remap.rs | 7 +++- src/vendor.rs | 111 ++++++++++++++++++++++++++++++++++++------------- 8 files changed, 260 insertions(+), 75 deletions(-) diff --git a/src/buck.rs b/src/buck.rs index 5f052d0a..f17ad9e2 100644 --- a/src/buck.rs +++ b/src/buck.rs @@ -448,6 +448,38 @@ impl Serialize for HttpArchive { } } +#[derive(Debug)] +pub struct ExtractArchive { + pub name: Name, + pub src: BuckPath, + pub strip_prefix: String, + pub sub_targets: BTreeSet, + pub visibility: Visibility, + pub sort_key: Name, +} + +impl Serialize for ExtractArchive { + fn serialize(&self, ser: S) -> Result { + let Self { + name, + src, + strip_prefix, + sub_targets, + visibility, + sort_key: _, + } = self; + let mut map = ser.serialize_map(None)?; + map.serialize_entry("name", name)?; + map.serialize_entry("src", src)?; + map.serialize_entry("strip_prefix", strip_prefix)?; + if !sub_targets.is_empty() { + map.serialize_entry("sub_targets", sub_targets)?; + } + map.serialize_entry("visibility", visibility)?; + map.end() + } +} + #[derive(Debug)] pub struct GitFetch { pub name: Name, @@ -1030,6 +1062,7 @@ impl Serialize for PrebuiltCxxLibrary { pub enum Rule { Alias(Alias), Filegroup(Filegroup), + ExtractArchive(ExtractArchive), HttpArchive(HttpArchive), GitFetch(GitFetch), Binary(RustBinary), @@ -1071,6 +1104,7 @@ fn rule_sort_key(rule: &Rule) -> impl Ord + '_ { // Make the alias rule come before the actual rule. Note that aliases // emitted by reindeer are always to a target within the same package. Rule::Alias(Alias { actual, .. }) => RuleSortKey::Other(actual, 0), + Rule::ExtractArchive(ExtractArchive { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), Rule::HttpArchive(HttpArchive { sort_key, .. }) => RuleSortKey::Other(sort_key, 1), Rule::GitFetch(GitFetch { name, .. }) => RuleSortKey::GitFetch(name), Rule::Filegroup(_) @@ -1096,6 +1130,7 @@ impl Rule { Rule::Alias(Alias { name, .. }) | Rule::Filegroup(Filegroup { name, .. }) | Rule::HttpArchive(HttpArchive { name, .. }) + | Rule::ExtractArchive(ExtractArchive { name, .. }) | Rule::GitFetch(GitFetch { name, .. }) | Rule::Binary(RustBinary { common: @@ -1148,6 +1183,9 @@ impl Rule { Rule::Filegroup(filegroup) => { FunctionCall::new(&config.filegroup, filegroup).serialize(Serializer) } + Rule::ExtractArchive(compressed_crate) => { + FunctionCall::new(&config.extract_archive, compressed_crate).serialize(Serializer) + } Rule::HttpArchive(http_archive) => { FunctionCall::new(&config.http_archive, http_archive).serialize(Serializer) } diff --git a/src/buckify.rs b/src/buckify.rs index 1fdaca9f..ff4bf313 100644 --- a/src/buckify.rs +++ b/src/buckify.rs @@ -33,6 +33,7 @@ use crate::buck; use crate::buck::Alias; use crate::buck::BuckPath; use crate::buck::Common; +use crate::buck::ExtractArchive; use crate::buck::Filegroup; use crate::buck::GitFetch; use crate::buck::HttpArchive; @@ -56,6 +57,7 @@ use crate::cargo::Source; use crate::cargo::TargetReq; use crate::collection::SetOrMap; use crate::config::Config; +use crate::config::VendorConfig; use crate::fixups::ExportSources; use crate::fixups::Fixups; use crate::glob::Globs; @@ -219,7 +221,7 @@ fn generate_rules<'scope>( for rule in rules { let _ = rule_tx.send(Ok(rule)); } - if context.config.vendor.is_none() { + if !matches!(context.config.vendor, VendorConfig::Source(_)) { deps.push((pkg, TargetReq::Sources)); } } @@ -258,10 +260,18 @@ fn generate_nonvendored_sources_archive<'scope>( match &lockfile_package.source { Source::Local => Ok(None), - Source::CratesIo => generate_http_archive(context, pkg, lockfile_package).map(Some), + Source::CratesIo => match context.config.vendor { + VendorConfig::Off => generate_http_archive(context, pkg, lockfile_package).map(Some), + VendorConfig::LocalRegistry => generate_extract_archive(pkg).map(Some), + VendorConfig::Source(_) => unreachable!(), + }, Source::Git { repo, commit_hash, .. - } => generate_git_fetch(repo, commit_hash).map(Some), + } => match context.config.vendor { + VendorConfig::Off => generate_git_fetch(repo, commit_hash).map(Some), + VendorConfig::LocalRegistry => generate_extract_archive(pkg).map(Some), + VendorConfig::Source(_) => unreachable!(), + }, Source::Unrecognized(_) => { bail!( "`vendor = false` mode is supported only with exclusively crates.io and https git dependencies. \"{}\" {} is coming from some other source", @@ -272,6 +282,20 @@ fn generate_nonvendored_sources_archive<'scope>( } } +fn generate_extract_archive(pkg: &Manifest) -> anyhow::Result { + Ok(Rule::ExtractArchive(ExtractArchive { + name: Name(format!("{}-{}.crate", pkg.name, pkg.version)), + src: BuckPath(PathBuf::from(format!( + "vendor/{}-{}.crate", + pkg.name, pkg.version, + ))), + strip_prefix: format!("{}-{}", pkg.name, pkg.version), + sub_targets: BTreeSet::new(), // populated later after all fixups are constructed + visibility: Visibility::Private, + sort_key: Name(format!("{}-{}", pkg.name, pkg.version)), + })) +} + fn generate_http_archive<'scope>( context: &'scope RuleContext<'scope>, pkg: &'scope Manifest, @@ -413,22 +437,25 @@ fn generate_target_rules<'scope>( log::debug!("pkg {} target {} fixups {:#?}", pkg, tgt.name, fixups); let manifest_dir = pkg.manifest_dir(); - let mapped_manifest_dir = - if context.config.vendor.is_some() || matches!(pkg.source, Source::Local) { - relative_path(&paths.third_party_dir, manifest_dir) - } else if let Source::Git { repo, .. } = &pkg.source { - let git_fetch = short_name_for_git_repo(repo)?; - let repository_root = find_repository_root(manifest_dir)?; - let path_within_repo = relative_path(repository_root, manifest_dir); - PathBuf::from(git_fetch).join(path_within_repo) - } else { - PathBuf::from(format!("{}-{}.crate", pkg.name, pkg.version)) - }; + let mapped_manifest_dir = if matches!(config.vendor, VendorConfig::Source(_)) + || matches!(pkg.source, Source::Local) + { + relative_path(&paths.third_party_dir, manifest_dir) + } else if let VendorConfig::LocalRegistry = config.vendor { + PathBuf::from(format!("{}-{}.crate", pkg.name, pkg.version)) + } else if let Source::Git { repo, .. } = &pkg.source { + let git_fetch = short_name_for_git_repo(repo)?; + let repository_root = find_repository_root(manifest_dir)?; + let path_within_repo = relative_path(repository_root, manifest_dir); + PathBuf::from(git_fetch).join(path_within_repo) + } else { + PathBuf::from(format!("{}-{}.crate", pkg.name, pkg.version)) + }; let crate_root = mapped_manifest_dir.join(relative_path(manifest_dir, &tgt.src_path)); let edition = tgt.edition.unwrap_or(pkg.edition); let mut licenses = BTreeSet::new(); - if config.vendor.is_none() { + if !matches!(config.vendor, VendorConfig::Source(_)) { // The `licenses` attribute takes `attrs.source()` which is the file // containing the custom license text. For `vendor = false` mode, we // don't have such a file on disk, and we don't have a Buck label either @@ -458,7 +485,8 @@ fn generate_target_rules<'scope>( // filename, or a list of globs. // If we're configured to get precise sources and we're using 2018+ edition source, then // parse the crate to see what files are actually used. - let mut srcs = if (config.vendor.is_some() || matches!(pkg.source, Source::Local)) + let mut srcs = if (matches!(config.vendor, VendorConfig::Source(_)) + || matches!(pkg.source, Source::Local)) && fixups.precise_srcs() && edition >= Edition::Rust2018 { @@ -504,7 +532,7 @@ fn generate_target_rules<'scope>( ) .context("rustc_flags")?; - if config.vendor.is_some() || matches!(pkg.source, Source::Local) { + if matches!(config.vendor, VendorConfig::Source(_)) || matches!(pkg.source, Source::Local) { unzip_platform( config, &mut base, @@ -516,6 +544,10 @@ fn generate_target_rules<'scope>( fixups.compute_srcs(srcs)?, ) .context("srcs")?; + } else if let VendorConfig::LocalRegistry = config.vendor { + let extract_archive_target = format!(":{}-{}.crate", pkg.name, pkg.version); + base.srcs + .insert(BuckPath(PathBuf::from(extract_archive_target))); } else if let Source::Git { repo, .. } = &pkg.source { let short_name = short_name_for_git_repo(repo)?; let git_fetch_target = format!(":{}.git", short_name); @@ -887,7 +919,9 @@ fn generate_target_rules<'scope>( // For non-disk sources (i.e. non-vendor mode git_fetch and // http_archive), `srcs` and `exclude` are ignored because // we can't look at the files to match globs. - let srcs = if config.vendor.is_some() || matches!(pkg.source, Source::Local) { + let srcs = if matches!(config.vendor, VendorConfig::Source(_)) + || matches!(pkg.source, Source::Local) + { // e.g. {"src/lib.rs": "vendor/foo-1.0.0/src/lib.rs"} let mut globs = Globs::new(srcs, exclude).context("export sources")?; let srcs = globs @@ -901,6 +935,14 @@ fn generate_target_rules<'scope>( globs.check_all_globs_used()?; } srcs + } else if let VendorConfig::LocalRegistry = config.vendor { + // e.g. {":foo-1.0.0.git": "foo-1.0.0"} + let extract_archive_target = format!(":{}-{}.crate", pkg.name, pkg.version); + [( + BuckPath(mapped_manifest_dir.clone()), + SubtargetOrPath::Path(BuckPath(PathBuf::from(extract_archive_target))), + )] + .into() } else if let Source::Git { repo, .. } = &pkg.source { // e.g. {":foo-123.git": "foo-123"} let short_name = short_name_for_git_repo(repo)?; @@ -1002,7 +1044,7 @@ fn buckify_for_universe( // Fill in all http_archive rules with all the sub_targets which got // mentioned by fixups. - if config.vendor.is_none() { + if !matches!(config.vendor, VendorConfig::Source(_)) { let mut need_subtargets = HashMap::>::new(); let mut insert = |subtarget_or_path: &SubtargetOrPath| { if let SubtargetOrPath::Subtarget(subtarget) = subtarget_or_path { @@ -1043,10 +1085,18 @@ fn buckify_for_universe( rules = rules .into_iter() .map(|mut rule| { - if let Rule::HttpArchive(rule) = &mut rule { - if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { - rule.sub_targets = need_subtargets; + match &mut rule { + Rule::HttpArchive(rule) => { + if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { + rule.sub_targets = need_subtargets; + } + } + Rule::ExtractArchive(rule) => { + if let Some(need_subtargets) = need_subtargets.remove(&rule.name) { + rule.sub_targets = need_subtargets; + } } + _ => {} } rule }) diff --git a/src/cargo.rs b/src/cargo.rs index 7de18023..13b8a02e 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -30,6 +30,7 @@ use serde::Deserializer; use serde::Serialize; use crate::config::Config; +use crate::config::VendorConfig; use crate::lockfile::Lockfile; use crate::platform::PlatformExpr; use crate::Args; @@ -60,7 +61,7 @@ pub fn cargo_get_lockfile_and_metadata( let cargo_home; let lockfile; - if config.vendor.is_none() { + if !matches!(config.vendor, VendorConfig::Source(_)) { cargo_home = None; // Whether or not there is a Cargo.lock already, do not read it yet. diff --git a/src/config.rs b/src/config.rs index 268bd2b3..91658527 100644 --- a/src/config.rs +++ b/src/config.rs @@ -77,11 +77,8 @@ pub struct Config { #[serde(default)] pub buck: BuckConfig, - #[serde( - default = "default_vendor_config", - deserialize_with = "deserialize_vendor_config" - )] - pub vendor: Option, + #[serde(default, deserialize_with = "deserialize_vendor_config")] + pub vendor: VendorConfig, #[serde(default = "default_platforms")] pub platform: HashMap, @@ -128,6 +125,8 @@ pub struct BuckConfig { /// Rule name for http_archive #[serde(default)] pub http_archive: StringWithDefault, + #[serde(default)] + pub extract_archive: StringWithDefault, /// Rule name for git_fetch #[serde(default)] pub git_fetch: StringWithDefault, @@ -150,9 +149,17 @@ pub struct BuckConfig { pub buildscript_genrule: StringWithDefault, } +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub enum VendorConfig { + Off, + LocalRegistry, + Source(VendorSourceConfig), +} + #[derive(Debug, Default, Clone, Deserialize)] #[serde(deny_unknown_fields)] -pub struct VendorConfig { +pub struct VendorSourceConfig { /// List of .gitignore files to use to filter checksum files, relative to /// this config file. #[serde(default)] @@ -162,6 +169,12 @@ pub struct VendorConfig { pub checksum_exclude: HashSet, } +impl Default for VendorConfig { + fn default() -> Self { + VendorConfig::Source(Default::default()) + } +} + #[derive(Clone)] pub struct StringWithDefault { pub value: String, @@ -236,10 +249,6 @@ impl From for StringWithDefault { } } -fn default_vendor_config() -> Option { - Some(VendorConfig::default()) -} - fn default_platforms() -> HashMap { const DEFAULT_PLATFORMS_TOML: &str = include_str!("default_platforms.toml"); @@ -259,14 +268,14 @@ fn default_universes() -> BTreeMap { map } -fn deserialize_vendor_config<'de, D>(deserializer: D) -> Result, D::Error> +fn deserialize_vendor_config<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { struct VendorConfigVisitor; impl<'de> Visitor<'de> for VendorConfigVisitor { - type Value = Option; + type Value = VendorConfig; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("[vendor] section, or `vendor = false`") @@ -278,14 +287,30 @@ where { // `vendor = true`: default configuration with vendoring. // `vendor = false`: do not vendor. - Ok(value.then(VendorConfig::default)) + Ok(if value { + VendorConfig::default() + } else { + VendorConfig::Off + }) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + if v == "local-registry" { + Ok(VendorConfig::LocalRegistry) + } else { + Err(E::custom("unknown vendor type")) + } } fn visit_map(self, map: M) -> Result where M: MapAccess<'de>, { - VendorConfig::deserialize(MapAccessDeserializer::new(map)).map(Some) + VendorSourceConfig::deserialize(MapAccessDeserializer::new(map)) + .map(VendorConfig::Source) } } diff --git a/src/fixups.rs b/src/fixups.rs index b8e767a5..66420762 100644 --- a/src/fixups.rs +++ b/src/fixups.rs @@ -42,6 +42,7 @@ use crate::cargo::NodeDepKind; use crate::cargo::Source; use crate::collection::SetOrMap; use crate::config::Config; +use crate::config::VendorConfig; use crate::glob::Globs; use crate::glob::SerializableGlobSet as GlobSet; use crate::glob::NO_EXCLUDE; @@ -167,7 +168,9 @@ impl<'meta> Fixups<'meta> { &self, relative_to_manifest_dir: &Path, ) -> anyhow::Result { - if self.config.vendor.is_some() || matches!(self.package.source, Source::Local) { + if matches!(self.config.vendor, VendorConfig::Source(_)) + || matches!(self.package.source, Source::Local) + { // Path to vendored file looks like "vendor/foo-1.0.0/src/lib.rs" let manifest_dir = relative_path(&self.third_party_dir, self.manifest_dir); let path = manifest_dir.join(relative_to_manifest_dir); @@ -309,7 +312,7 @@ impl<'meta> Fixups<'meta> { }; for fix in fixes { - if self.config.vendor.is_none() { + if !matches!(self.config.vendor, VendorConfig::Source(_)) { if let Source::Git { repo, .. } = &self.package.source { // Cxx_library fixups only work if the sources are vendored // or from an http_archive. They do not work with sources @@ -868,13 +871,18 @@ impl<'meta> Fixups<'meta> { for cargo_env in config.cargo_env.iter() { let v = match cargo_env { CargoEnv::CARGO_MANIFEST_DIR => { - if self.config.vendor.is_some() + if matches!(self.config.vendor, VendorConfig::Source(_)) || matches!(self.package.source, Source::Local) { StringOrPath::Path(BuckPath(relative_path( &self.third_party_dir, self.manifest_dir, ))) + } else if let VendorConfig::LocalRegistry = self.config.vendor { + StringOrPath::String(format!( + "{}-{}.crate", + self.package.name, self.package.version, + )) } else if let Source::Git { repo, .. } = &self.package.source { let short_name = short_name_for_git_repo(repo)?; StringOrPath::String(short_name.to_owned()) @@ -929,7 +937,10 @@ impl<'meta> Fixups<'meta> { // This function is only used in vendoring mode, so it's guaranteed that // manifest_dir is a subdirectory of third_party_dir. - assert!(self.config.vendor.is_some() || matches!(self.package.source, Source::Local)); + assert!( + matches!(self.config.vendor, VendorConfig::Source(_)) + || matches!(self.package.source, Source::Local) + ); let manifest_rel = relative_path(&self.third_party_dir, self.manifest_dir); let srcs_globs: Vec = srcs diff --git a/src/main.rs b/src/main.rs index a22c9860..4c08179b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,8 @@ use std::path::PathBuf; use clap::Parser; use clap::Subcommand; +use crate::config::VendorConfig; + mod audit_sec; mod buck; mod buckify; @@ -145,10 +147,14 @@ fn try_main() -> anyhow::Result<()> { } SubCommand::Buckify { stdout } => { - if config.vendor.is_some() && !vendor::is_vendored(&paths)? { + if matches!( + config.vendor, + VendorConfig::LocalRegistry | VendorConfig::Source(_) + ) && !vendor::is_vendored(&config, &paths)? + { // If you ran `reindeer buckify` without `reindeer vendor`, then // default to generating non-vendored targets. - config.vendor = None; + config.vendor = VendorConfig::Off; } buckify::buckify(&config, &args, &paths, *stdout)?; } diff --git a/src/remap.rs b/src/remap.rs index e939aa4a..dde35977 100644 --- a/src/remap.rs +++ b/src/remap.rs @@ -11,14 +11,15 @@ use std::path::PathBuf; use serde::Deserialize; use serde::Serialize; -#[derive(Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Default)] pub struct RemapConfig { #[serde(rename = "source", default)] pub sources: Map, } -#[derive(Serialize, Deserialize, Default)] +#[derive(Debug, Serialize, Deserialize, Default)] pub struct RemapSource { + pub registry: Option, pub directory: Option, pub git: Option, pub rev: Option, @@ -26,4 +27,6 @@ pub struct RemapSource { pub tag: Option, #[serde(rename = "replace-with")] pub replace_with: Option, + #[serde(rename = "local-registry")] + pub local_registry: Option, } diff --git a/src/vendor.rs b/src/vendor.rs index ee8d5ce9..693e0012 100644 --- a/src/vendor.rs +++ b/src/vendor.rs @@ -22,7 +22,9 @@ use crate::buckify::relative_path; use crate::cargo; use crate::config::Config; use crate::config::VendorConfig; +use crate::config::VendorSourceConfig; use crate::remap::RemapConfig; +use crate::remap::RemapSource; use crate::Args; use crate::Paths; @@ -42,36 +44,76 @@ pub(crate) fn cargo_vendor( ) -> anyhow::Result<()> { let vendordir = Path::new("vendor"); // relative to third_party_dir - let mut cmdline = vec![ - "vendor", - "--manifest-path", - paths.manifest_path.to_str().unwrap(), - vendordir.to_str().unwrap(), - "--versioned-dirs", - ]; - if no_delete { - cmdline.push("--no-delete"); - } + if let VendorConfig::LocalRegistry = config.vendor { + let mut cmdline = vec![ + "local-registry", + "-s", + paths.lockfile_path.to_str().unwrap(), + vendordir.to_str().unwrap(), + "--git", + ]; + if no_delete { + cmdline.push("--no-delete"); + } + log::info!("Running cargo {:?}", cmdline); + let _ = cargo::run_cargo( + config, + Some(&paths.cargo_home), + &paths.third_party_dir, + args, + &cmdline, + )?; + let mut remap = RemapConfig::default(); + remap.sources.insert( + "crates-io".to_owned(), + RemapSource { + registry: Some("sparse+https://index.crates.io/".to_owned()), + replace_with: Some("local-registry".to_owned()), + ..RemapSource::default() + }, + ); + remap.sources.insert( + "local-registry".to_owned(), + RemapSource { + local_registry: Some(vendordir.to_owned()), + ..RemapSource::default() + }, + ); + let config_toml = toml::to_string(&remap).context("failed to serialize config.toml")?; + fs::write(paths.cargo_home.join("config.toml"), config_toml)?; + assert!(is_vendored(config, paths)?); + } else { + let mut cmdline = vec![ + "vendor", + "--manifest-path", + paths.manifest_path.to_str().unwrap(), + vendordir.to_str().unwrap(), + "--versioned-dirs", + ]; + if no_delete { + cmdline.push("--no-delete"); + } - fs::create_dir_all(&paths.cargo_home)?; + fs::create_dir_all(&paths.cargo_home)?; - log::info!("Running cargo {:?}", cmdline); - let cargoconfig = cargo::run_cargo( - config, - Some(&paths.cargo_home), - &paths.third_party_dir, - args, - &cmdline, - )?; + log::info!("Running cargo {:?}", cmdline); + let cargoconfig = cargo::run_cargo( + config, + Some(&paths.cargo_home), + &paths.third_party_dir, + args, + &cmdline, + )?; - fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; - if !cargoconfig.is_empty() { - assert!(is_vendored(paths)?); - } + fs::write(paths.cargo_home.join("config.toml"), &cargoconfig)?; + if !cargoconfig.is_empty() { + assert!(is_vendored(config, paths)?); + } - if let Some(vendor_config) = &config.vendor { - filter_checksum_files(&paths.third_party_dir, vendordir, vendor_config)?; - write_excluded_build_scripts(&paths.third_party_dir, vendordir)?; + if let VendorConfig::Source(source_config) = &config.vendor { + filter_checksum_files(&paths.third_party_dir, vendordir, source_config)?; + write_excluded_build_scripts(&paths.third_party_dir, vendordir)?; + } } if audit_sec { @@ -81,7 +123,7 @@ pub(crate) fn cargo_vendor( Ok(()) } -pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { +pub(crate) fn is_vendored(config: &Config, paths: &Paths) -> anyhow::Result { // .cargo/config.toml is Cargo's preferred name for the config, but .cargo/config // is the older name so it takes priority if present. let mut cargo_config_path = paths.cargo_home.join("config"); @@ -108,8 +150,17 @@ pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { let remap_config: RemapConfig = toml::from_str(&content) .context(format!("Failed to parse {}", cargo_config_path.display()))?; - match remap_config.sources.get("vendored-sources") { - Some(vendored_sources) => Ok(vendored_sources.directory.is_some()), + let source_name = match config.vendor { + VendorConfig::LocalRegistry => "local-registry", + VendorConfig::Source(_) => "vendored-sources", + _ => return Ok(false), + }; + match remap_config.sources.get(source_name) { + Some(source) => match config.vendor { + VendorConfig::LocalRegistry => Ok(source.local_registry.is_some()), + VendorConfig::Source(_) => Ok(source.directory.is_some()), + VendorConfig::Off => Ok(false), + }, None => Ok(false), } } @@ -117,7 +168,7 @@ pub(crate) fn is_vendored(paths: &Paths) -> anyhow::Result { fn filter_checksum_files( third_party_dir: &Path, vendordir: &Path, - config: &VendorConfig, + config: &VendorSourceConfig, ) -> anyhow::Result<()> { if config.checksum_exclude.is_empty() && config.gitignore_checksum_exclude.is_empty() { return Ok(());