From 6c62e4aa540aacd0a183e0f92629a02100f70054 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 17:15:17 -0400 Subject: [PATCH 01/11] ? --- .github/workflows/matrix.sh | 32 +++++++++++++++ ".github/workflows/\360\237\222\270.yml" | 51 ++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100755 .github/workflows/matrix.sh create mode 100644 ".github/workflows/\360\237\222\270.yml" diff --git a/.github/workflows/matrix.sh b/.github/workflows/matrix.sh new file mode 100755 index 000000000..8baa3ac63 --- /dev/null +++ b/.github/workflows/matrix.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +#nix flake show --json --all-systems \ +cat data.json \ + | nix run nixpkgs#gron -- -j \ + | nix run nixpkgs#jq -- -r \ + --slurp \ + --argjson map ' + { + "aarch64-darwin": "macos-latest", + "x86_64-darwin": "macos-latest", + "x86_64-linux": "UbuntuLatest32Cores128G", + "aarch64-linux": "UbuntuLatest32Cores128GArm", + "i686-linux": "UbuntuLatest32Cores128G" + } + ' \ + ' + map(select(.[0][-1] == "type" and .[1] == "derivation") + | .[0][0:-1] + | select(.[-1] != "all") + | { + attribute: . | join("."), + "nix-system": .[-2], + "runs-on": $map[.[-2]] + } + | if ."runs-on" == null then + ("No GitHub Actions Runner system known for the Nix system `" + ."nix-system" + "` on attribute `" + .attribute + "`.\n") | halt_error(1) + else + . + end) + | "matrix=" + tostring + ' >> "$GITHUB_OUTPUT" \ No newline at end of file diff --git "a/.github/workflows/\360\237\222\270.yml" "b/.github/workflows/\360\237\222\270.yml" new file mode 100644 index 000000000..9b327960d --- /dev/null +++ "b/.github/workflows/\360\237\222\270.yml" @@ -0,0 +1,51 @@ +name: 💸 + +on: + workflow_dispatch: + +jobs: + inventory: + name: Identify outputs to build + runs-on: UbuntuLatest32Cores128G + outputs: + matrix: ${{ steps.inventory.outputs.matrix }} + permissions: + id-token: "write" + contents: "read" + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + with: + flakehub: true + - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Enumerate Checks + id: inventory + run: | + ./.github/workflows/matrix.sh + + build: + runs-on: ${{ matrix.jobs.runs-on }} + needs: inventory + strategy: + matrix: + jobs: ${{ fromJSON(needs.inventory.outputs.matrix) }} + + permissions: + id-token: "write" + contents: "read" + + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + with: + flakehub: true + - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Build for ${{ matrix.jobs.nix-system }} + run: nix build .#${{ matrix.jobs.attribute }} + + success: + runs-on: ubuntu-latest + needs: build + + steps: + - run: true From 1dc65c4100eda4ceefd97232329e814561f11407 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 17:17:12 -0400 Subject: [PATCH 02/11] sigh, no fun --- .../workflows/money-with-wings.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ".github/workflows/\360\237\222\270.yml" => .github/workflows/money-with-wings.yml (100%) diff --git "a/.github/workflows/\360\237\222\270.yml" b/.github/workflows/money-with-wings.yml similarity index 100% rename from ".github/workflows/\360\237\222\270.yml" rename to .github/workflows/money-with-wings.yml From aee3bc4a6b3f2046fdf2c8bcc5eb59cb6def5d70 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 17:22:06 -0400 Subject: [PATCH 03/11] GHA continues to not enable fun --- .github/workflows/money-with-wings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/money-with-wings.yml b/.github/workflows/money-with-wings.yml index 9b327960d..c464b9c16 100644 --- a/.github/workflows/money-with-wings.yml +++ b/.github/workflows/money-with-wings.yml @@ -1,4 +1,4 @@ -name: 💸 +name: Extensive Release Checks on: workflow_dispatch: From 5203d2e3f493534e4db8303ad7de41bf932b1e81 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 17:29:51 -0400 Subject: [PATCH 04/11] Update money-with-wings.yml --- .github/workflows/money-with-wings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/money-with-wings.yml b/.github/workflows/money-with-wings.yml index c464b9c16..5cea2e118 100644 --- a/.github/workflows/money-with-wings.yml +++ b/.github/workflows/money-with-wings.yml @@ -48,4 +48,4 @@ jobs: needs: build steps: - - run: true + - run: "true" From 4beeeef754ecc83d55047f0aa45449a54f6eeb6c Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 17:34:50 -0400 Subject: [PATCH 05/11] ? --- .github/workflows/money-with-wings.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/money-with-wings.yml b/.github/workflows/money-with-wings.yml index 5cea2e118..627c71a1e 100644 --- a/.github/workflows/money-with-wings.yml +++ b/.github/workflows/money-with-wings.yml @@ -2,6 +2,9 @@ name: Extensive Release Checks on: workflow_dispatch: + push: + branches: + - money-with-wings jobs: inventory: From 89193b5807cd7fcd564dfa56142a17e3a77f44d7 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 17:36:19 -0400 Subject: [PATCH 06/11] derp --- .github/workflows/matrix.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/matrix.sh b/.github/workflows/matrix.sh index 8baa3ac63..246ef972e 100755 --- a/.github/workflows/matrix.sh +++ b/.github/workflows/matrix.sh @@ -1,7 +1,6 @@ #!/bin/sh -#nix flake show --json --all-systems \ -cat data.json \ +nix flake show --json --all-systems \ | nix run nixpkgs#gron -- -j \ | nix run nixpkgs#jq -- -r \ --slurp \ From 6c96c8a2a6f17b5b76b8da937de1060664c04210 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 18:30:50 -0400 Subject: [PATCH 07/11] fail-fast false --- .github/workflows/money-with-wings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/money-with-wings.yml b/.github/workflows/money-with-wings.yml index 627c71a1e..c4258c2ce 100644 --- a/.github/workflows/money-with-wings.yml +++ b/.github/workflows/money-with-wings.yml @@ -30,6 +30,7 @@ jobs: runs-on: ${{ matrix.jobs.runs-on }} needs: inventory strategy: + fail-fast: false matrix: jobs: ${{ fromJSON(needs.inventory.outputs.matrix) }} From 230a31ac41589ac2768094b1c3fe6310209b8d3f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 19:38:07 -0400 Subject: [PATCH 08/11] Only build hydraJobs, the rest are covered in CI --- .github/workflows/matrix.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/matrix.sh b/.github/workflows/matrix.sh index 246ef972e..7a2dd8aea 100755 --- a/.github/workflows/matrix.sh +++ b/.github/workflows/matrix.sh @@ -15,8 +15,9 @@ nix flake show --json --all-systems \ ' \ ' map(select(.[0][-1] == "type" and .[1] == "derivation") - | .[0][0:-1] - | select(.[-1] != "all") + | .[0][0:-1] # Take each attribute name and drop `type` + | select(.[-1] != "all") # Skip attributes which are `all` jobs, presumably combining other jobs + | select(.[0] == "hydraJobs") # Select the hydraJobs which are not typically run in CI | { attribute: . | join("."), "nix-system": .[-2], From 3d1f53ed2ee773b3a04e09551ca87926f9977308 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 20:00:20 -0400 Subject: [PATCH 09/11] Make the aggregate jobs end with -aggregate so we don't build those --- .github/workflows/matrix.sh | 1 + nix/tests/vm-test/default.nix | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/matrix.sh b/.github/workflows/matrix.sh index 7a2dd8aea..9262bd0fd 100755 --- a/.github/workflows/matrix.sh +++ b/.github/workflows/matrix.sh @@ -17,6 +17,7 @@ nix flake show --json --all-systems \ map(select(.[0][-1] == "type" and .[1] == "derivation") | .[0][0:-1] # Take each attribute name and drop `type` | select(.[-1] != "all") # Skip attributes which are `all` jobs, presumably combining other jobs + | select(.[-1] | endswith("-aggregate") != true) # Skip attributes which end in `-aggregate`, because those just depend on other jobs which build them | select(.[0] == "hydraJobs") # Select the hydraJobs which are not typically run in CI | { attribute: . | join("."), diff --git a/nix/tests/vm-test/default.nix b/nix/tests/vm-test/default.nix index 63eb2a543..613574044 100644 --- a/nix/tests/vm-test/default.nix +++ b/nix/tests/vm-test/default.nix @@ -600,7 +600,7 @@ let touch $out ''; - makeTests = name: tests: builtins.mapAttrs + makeTests = name: tests: imagePredicate: builtins.mapAttrs (imageName: image: rec { ${image.system} = (builtins.mapAttrs @@ -608,7 +608,7 @@ let makeTest imageName testName test ) tests) // { - "${name}" = (with (forSystem "x86_64-linux" ({ system, pkgs, ... }: pkgs)); pkgs.releaseTools.aggregate { + "${name}-aggregate" = (with (forSystem "x86_64-linux" ({ system, pkgs, ... }: pkgs)); pkgs.releaseTools.aggregate { name = name; constituents = ( pkgs.lib.mapAttrsToList @@ -621,17 +621,17 @@ let }; } ) - images; + (lib.filterAttrs imagePredicate images); allCases = lib.recursiveUpdate (lib.recursiveUpdate installCases (lib.recursiveUpdate cureSelfCases cureScriptCases)) uninstallCases; - install-tests = makeTests "install" installCases; + install-tests = makeTests "install" installCases (_: _: true); - cure-self-tests = makeTests "cure-self" cureSelfCases; + cure-self-tests = makeTests "cure-self" cureSelfCases (_: _: true); - cure-script-tests = makeTests "cure-script" cureScriptCases; + cure-script-tests = makeTests "cure-script" cureScriptCases (_name: { upstreamScriptsWork ? true, ... }: upstreamScriptsWork); - uninstall-tests = makeTests "uninstall" uninstallCases; + uninstall-tests = makeTests "uninstall" uninstallCases (_: _: true); all-tests = builtins.mapAttrs (imageName: image: { @@ -652,7 +652,7 @@ in lib.recursiveUpdate joined-tests { all."x86_64-linux" = (with (forSystem "x86_64-linux" ({ system, pkgs, ... }: pkgs)); pkgs.lib.mapAttrs (caseName: case: pkgs.releaseTools.aggregate { - name = caseName; + name = "${caseName}-aggregate"; constituents = pkgs.lib.mapAttrsToList (name: value: value."x86_64-linux"."${caseName}") joined-tests; } )) (allCases // { "cure-self" = { }; "cure-script" = { }; "install" = { }; "uninstall" = { }; "all" = { }; }); From e5f5f3f7d506dac007a19fbf2f14ffcbe9c549e7 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 20:15:56 -0400 Subject: [PATCH 10/11] fixup the aggregate job generation --- nix/tests/vm-test/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tests/vm-test/default.nix b/nix/tests/vm-test/default.nix index 613574044..89eac3868 100644 --- a/nix/tests/vm-test/default.nix +++ b/nix/tests/vm-test/default.nix @@ -653,7 +653,7 @@ lib.recursiveUpdate joined-tests { all."x86_64-linux" = (with (forSystem "x86_64-linux" ({ system, pkgs, ... }: pkgs)); pkgs.lib.mapAttrs (caseName: case: pkgs.releaseTools.aggregate { name = "${caseName}-aggregate"; - constituents = pkgs.lib.mapAttrsToList (name: value: value."x86_64-linux"."${caseName}") joined-tests; + constituents = pkgs.lib.mapAttrsToList (name: value: value."x86_64-linux"."${caseName}-aggregate") joined-tests; } )) (allCases // { "cure-self" = { }; "cure-script" = { }; "install" = { }; "uninstall" = { }; "all" = { }; }); } From c1c2792e5021ef5fb79c61df3bdbff57c756cf7c Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Jul 2024 20:21:52 -0400 Subject: [PATCH 11/11] Don't add vm-test.all and container-test.all jobs to the matrix --- .github/workflows/matrix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/matrix.sh b/.github/workflows/matrix.sh index 9262bd0fd..af0eb8e27 100755 --- a/.github/workflows/matrix.sh +++ b/.github/workflows/matrix.sh @@ -16,6 +16,8 @@ nix flake show --json --all-systems \ ' map(select(.[0][-1] == "type" and .[1] == "derivation") | .[0][0:-1] # Take each attribute name and drop `type` + | select(.[0:3] != ["hydraJobs", "vm-test", "all"]) # Skip the hydraJobs.vm-test.all jobs, which aggregate other jobs + | select(.[0:3] != ["hydraJobs", "container-test", "all"]) # Skip the hydraJobs.container-test.all jobs, which aggregate other jobs | select(.[-1] != "all") # Skip attributes which are `all` jobs, presumably combining other jobs | select(.[-1] | endswith("-aggregate") != true) # Skip attributes which end in `-aggregate`, because those just depend on other jobs which build them | select(.[0] == "hydraJobs") # Select the hydraJobs which are not typically run in CI