diff --git a/hacking/cargo-manifest-management/Cargo.lock b/hacking/cargo-manifest-management/Cargo.lock index 1ccc22cf8..504612d37 100644 --- a/hacking/cargo-manifest-management/Cargo.lock +++ b/hacking/cargo-manifest-management/Cargo.lock @@ -369,9 +369,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "log" @@ -388,9 +388,6 @@ dependencies = [ "serde_json", "similar", "toml", - "toml-normalize", - "toml-path-regex", - "toml_edit", ] [[package]] @@ -526,18 +523,18 @@ checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "serde" -version = "1.0.190" +version = "1.0.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +checksum = "a834c4821019838224821468552240d4d95d14e751986442c816572d39a080c9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.190" +version = "1.0.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +checksum = "46fa52d5646bce91b680189fe5b1c049d2ea38dabb4e2e7c8d00ca12cfbfbcfd" dependencies = [ "proc-macro2", "quote", @@ -618,9 +615,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "2.0.38" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -678,14 +675,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff9e3abce27ee2c9a37f9ad37238c1bdd4e789c84ba37df76aa4d528f5072cc" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.21.0", ] [[package]] @@ -698,7 +695,7 @@ dependencies = [ "serde_with", "toml", "toml-path-regex", - "toml_edit", + "toml_edit 0.20.7", ] [[package]] @@ -725,6 +722,17 @@ name = "toml_edit" version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ "indexmap 2.1.0", "serde", @@ -894,9 +902,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.18" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176b6138793677221d420fd2f0aeeced263f197688b36484660da767bca2fa32" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" dependencies = [ "memchr", ] diff --git a/hacking/cargo-manifest-management/Cargo.toml b/hacking/cargo-manifest-management/Cargo.toml index 108d6322f..34c610df5 100644 --- a/hacking/cargo-manifest-management/Cargo.toml +++ b/hacking/cargo-manifest-management/Cargo.toml @@ -9,4 +9,5 @@ resolver = "2" members = [ "crates/toml-path-regex", "crates/toml-normalize", + "crates/manage-cargo-manifests", ] diff --git a/hacking/cargo-manifest-management/Makefile b/hacking/cargo-manifest-management/Makefile index b01ee42fe..ac8cd5cde 100644 --- a/hacking/cargo-manifest-management/Makefile +++ b/hacking/cargo-manifest-management/Makefile @@ -4,17 +4,29 @@ # SPDX-License-Identifier: BSD-2-Clause # +blueprint := $$(nix-build -A blueprintJSON --no-out-link) + +target_dir := target + +bin_dir := $(target_dir)/debug + +run := PATH=$(bin_dir):$$PATH $(bin_dir)/manage-cargo-manifests --blueprint $(blueprint) + .PHONY: none none: .PHONY: clean clean: - rm -rf target + rm -rf $(target_dir) + +.INTERMEDIATE: bins +bins: + cargo build .PHONY: update -update: - script=$$(nix-build -A updateScript --no-out-link) && $$script +update: bins + $(run) .PHONY: check -check: - nix-build -A witness --no-out-link +check: bins + $(run) --just-check diff --git a/hacking/cargo-manifest-management/crates/manage-cargo-manifests/Cargo.toml b/hacking/cargo-manifest-management/crates/manage-cargo-manifests/Cargo.toml new file mode 100644 index 000000000..64dcbd261 --- /dev/null +++ b/hacking/cargo-manifest-management/crates/manage-cargo-manifests/Cargo.toml @@ -0,0 +1,19 @@ +# +# Copyright 2023, Colias Group, LLC +# +# SPDX-License-Identifier: BSD-2-Clause +# + +[package] +name = "manage-cargo-manifests" +version = "0.1.0" +authors = ["Nick Spinale "] +edition = "2021" +license = "BSD-2-Clause" + +[dependencies] +clap = { version = "4.4.6", features = [ "derive" ] } +serde = { version = "1.0", features = [ "derive" ] } +serde_json = "1.0" +toml = "0.8.6" +similar = "2.3" diff --git a/hacking/cargo-manifest-management/crates/manage-cargo-manifests/src/main.rs b/hacking/cargo-manifest-management/crates/manage-cargo-manifests/src/main.rs new file mode 100644 index 000000000..484a8a9da --- /dev/null +++ b/hacking/cargo-manifest-management/crates/manage-cargo-manifests/src/main.rs @@ -0,0 +1,142 @@ +// +// Copyright 2023, Colias Group, LLC +// +// SPDX-License-Identifier: BSD-2-Clause +// + +use std::fs::{self, File}; +use std::io::Write; +use std::path::PathBuf; +use std::process::{self, Command, Stdio}; +use std::str; + +use clap::Parser; +use serde::Deserialize; +use serde_json::Value as JsonValue; +use similar::{ChangeTag, TextDiff}; +use toml::Table as TomlTable; + +#[derive(Debug, Parser)] +struct Args { + #[arg(long)] + blueprint: PathBuf, + + #[arg(long)] + just_check: bool, +} + +fn main() { + let args = Args::parse(); + let blueprint_file = File::open(&args.blueprint).unwrap(); + let blueprint: Blueprint = serde_json::from_reader(blueprint_file).unwrap(); + blueprint.execute(args.just_check); +} + +#[derive(Debug, Deserialize)] +#[serde(transparent)] +pub struct Blueprint { + pub entries: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct Entry { + #[serde(rename = "absolutePath")] + pub absolute_path: PathBuf, + #[serde(rename = "manifestValue")] + pub manifest_value: TomlTable, + #[serde(rename = "frontmatter")] + pub frontmatter: Option, + #[serde(rename = "formatPolicyOverrides")] + pub format_policy_overrides: Vec, + #[serde(rename = "justEnsureEquivalence")] + pub just_ensure_equivalence: bool, +} + +impl Blueprint { + pub fn execute(&self, just_check: bool) { + for entry in self.entries.iter() { + entry.execute(just_check); + } + } +} + +impl Entry { + fn execute(&self, just_check: bool) { + let manifest_path = self.absolute_path.join("Cargo.toml"); + let rendered = self.render(&format!("{}", self.manifest_value)); + let mut write = true; + if manifest_path.is_file() { + let existing = fs::read_to_string(&manifest_path).unwrap(); + if self.just_ensure_equivalence { + let existing_rendered = self.render(&existing); + if existing_rendered != rendered { + eprintln!( + "error: {} is out of date (note that this is a structural comparison):", + manifest_path.display() + ); + eprintln!("{}", format_diff(&rendered, &existing_rendered)); + die(); + } + } else { + if existing == rendered { + write = false; + } else if just_check { + eprintln!("error: {} is out of date:", manifest_path.display()); + eprintln!("{}", format_diff(&rendered, &existing)); + die(); + } + } + } else if just_check || self.just_ensure_equivalence { + eprintln!("error: {} does not exist", manifest_path.display()); + die(); + } + if write { + eprintln!("writing {}", manifest_path.display()); + fs::write(manifest_path, rendered).unwrap(); + } + } + + fn render(&self, unformatted_toml: &str) -> String { + let mut cmd = Command::new("toml-normalize"); + cmd.stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .arg("--builtin-policy=cargo-manifest"); + for policy in self.format_policy_overrides.iter() { + cmd.arg(format!("--inline-policy={}", policy)); + } + let mut child = cmd.spawn().unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(unformatted_toml.as_bytes()) + .unwrap(); + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + let mut s = String::new(); + if let Some(frontmatter) = self.frontmatter.as_ref() { + s.push_str(frontmatter); + s.push('\n'); + } + s.push_str(str::from_utf8(&output.stdout).unwrap()); + s + } +} + +fn format_diff(a: &str, b: &str) -> String { + let d = TextDiff::from_lines(a, b); + let mut s = String::new(); + for change in d.iter_all_changes() { + let sign = match change.tag() { + ChangeTag::Delete => "-", + ChangeTag::Insert => "+", + ChangeTag::Equal => " ", + }; + s.push_str(&format!("{}{}", sign, change)) + } + s +} + +fn die() -> ! { + process::exit(1) +} diff --git a/hacking/cargo-manifest-management/crates/toml-normalize/src/main.rs b/hacking/cargo-manifest-management/crates/toml-normalize/src/main.rs index b29114afd..50601b42b 100644 --- a/hacking/cargo-manifest-management/crates/toml-normalize/src/main.rs +++ b/hacking/cargo-manifest-management/crates/toml-normalize/src/main.rs @@ -22,6 +22,9 @@ struct Args { #[arg(long, value_name = "POLICY_FILE")] policy: Vec, + #[arg(long, value_name = "POLICY")] + inline_policy: Vec, + #[arg(long)] builtin_policy: Vec, } @@ -77,6 +80,17 @@ fn main() { policies.push((policy, index)); } + for (s, index) in args.inline_policy.iter().zip( + matches + .indices_of("inline_policy") + .map(Iterator::collect) + .unwrap_or_else(Vec::new), + ) { + let policy = serde_json::from_str(&s) + .unwrap_or_else(|err| panic!("error deserializing policy {}: {:?}", s, err)); + policies.push((policy, index)); + } + for (name, index) in args.builtin_policy.iter().zip( matches .indices_of("builtin_policy") diff --git a/hacking/cargo-manifest-management/default.nix b/hacking/cargo-manifest-management/default.nix index a4456980e..19f41fde2 100644 --- a/hacking/cargo-manifest-management/default.nix +++ b/hacking/cargo-manifest-management/default.nix @@ -25,11 +25,13 @@ rec { workspaceDirFilter = relativePathSegments: lib.head relativePathSegments == "crates"; }; - inherit (workspace) generatedManifestsList; + inherit (workspace) blueprint debug; - x = pkgs.callPackage ./x.nix { - inherit generatedManifestsList; - }; + prettyJSON = name: value: with pkgs; runCommand name { + nativeBuildInputs = [ jq ]; + } '' + jq < ${builtins.toFile name (builtins.toJSON value)} > $out + ''; - inherit (x) witness updateScript debug; + blueprintJSON = prettyJSON "blueprint.json" blueprint; } diff --git a/hacking/cargo-manifest-management/tool.nix b/hacking/cargo-manifest-management/tool.nix index c2a957c54..d45b32ae1 100644 --- a/hacking/cargo-manifest-management/tool.nix +++ b/hacking/cargo-manifest-management/tool.nix @@ -132,6 +132,20 @@ let (lib.flip lib.concatMap generatedManifestsList (manifest: lib.optional (manifest.packageName != null) (lib.nameValuePair manifest.packageName manifest))); -in { - inherit generatedManifestsList; +in rec { + blueprint = generatedManifestsList; + + debug = { + byPath = lib.listToAttrs (lib.forEach blueprint (attrs: { + name = attrs.absolutePath; + value = attrs; + })); + byCrateName = lib.listToAttrs + (lib.forEach + (lib.filter (attrs: attrs.manifestValue.package.name or null != null) blueprint) + (attrs: { + name = attrs.manifestValue.package.name; + value = attrs; + })); + }; } diff --git a/hacking/cargo-manifest-management/x.nix b/hacking/cargo-manifest-management/x.nix deleted file mode 100644 index 89b529fe9..000000000 --- a/hacking/cargo-manifest-management/x.nix +++ /dev/null @@ -1,107 +0,0 @@ -{ lib, formats -, runCommand, writeText, writeShellApplication -, generatedManifestsList -}: - -let - tomlNormalize = ./target/debug/toml-normalize; - - generateJSON = (formats.json {}).generate; - - generateTOML = (formats.toml {}).generate; - - appendNewline = s: "${s}\n"; - - normalize = { inputManifestTOML, formatPolicyOverrides, frontmatter }: - let - frontmatterArg = lib.optionalString (frontmatter != null) ( - builtins.toFile "frontmatter.txt" (appendNewline frontmatter) - ); - in - runCommand "Cargo.toml" {} '' - ${tomlNormalize} \ - ${inputManifestTOML} \ - --builtin-policy cargo-manifest \ - ${lib.concatMapStrings (policy: "--policy ${generateJSON "policy.json" policy}") formatPolicyOverrides} \ - | cat ${frontmatterArg} - > $out - ''; - - format = { manifestValue, formatPolicyOverrides, frontmatter }: normalize { - inherit formatPolicyOverrides frontmatter; - inputManifestTOML = generateTOML "Cargo.toml" manifestValue; - }; - - augmentedGeneratedManifestsList = lib.forEach generatedManifestsList (attrs: attrs // rec { - - origManifestPath = "${attrs.absolutePath}/Cargo.toml"; - - origManifest = builtins.path { - path = origManifestPath; - }; - - origManifestExists = builtins.pathExists origManifestPath; - - normalizedOrigManifest = normalize { - inherit (attrs) frontmatter; - inputManifestTOML = origManifest; - }; - - normalizedManifest = format { - inherit (attrs) manifestValue formatPolicyOverrides frontmatter; - }; - - witness = if attrs.justEnsureEquivalence then witnessEquivalent else witnessIdentical; - - witnessEquivalent = runCommand "witness-structurally-equivalent" {} '' - if ! diff -u --label ${origManifestPath} ${normalizedOrigManifest} ${normalizedManifest}; then - echo "${origManifestPath} is not structurally equivalent to ${normalizedManifest}" >&2 - exit 1 - fi - touch $out - ''; - - witnessIdentical = runCommand "witness-identical" {} '' - if ! diff -u --label ${origManifestPath} ${origManifest} ${normalizedManifest}; then - echo "${origManifestPath} is not identical to ${normalizedManifest}" >&2 - exit 1 - fi - touch $out - ''; - }); - - witness = writeText "witness" (lib.flip lib.concatMapStrings augmentedGeneratedManifestsList (attrs: with attrs; - appendNewline (if justEnsureEquivalence then witnessEquivalent else witnessIdentical) - )); - - updateScript = writeShellApplication { - name = "update"; - text = lib.flip lib.concatMapStrings augmentedGeneratedManifestsList (attrs: with attrs; - appendNewline ( - if justEnsureEquivalence - then "# ${witnessEquivalent}" - else if origManifestExists - then "# ${witnessIdentical}" - else "cp ${normalizedManifest} ${origManifestPath}" - ) - ); - }; - -in rec { - inherit witness updateScript; - - # for debugging - - debug = { - byPath = lib.listToAttrs (lib.forEach augmentedGeneratedManifestsList (attrs: { - name = attrs.absolutePath; - value = attrs; - })); - byCrateName = lib.listToAttrs - (lib.forEach - (lib.filter (attrs: attrs.manifestValue.package.name or null != null) augmentedGeneratedManifestsList) - (attrs: { - name = attrs.manifestValue.package.name; - value = attrs; - })); - }; -}