chore: package released binaries by architecture

Previously when we performed a release, we would do a Github Release for each binary in a particular
selection. Now, the approach is to have a single Github Release for all the binaries. The assets for
the single release will be a collection of zip files containing all nine binaries for each target
architecture.

The new release process stipulates that each package name should contain `YYYY.MM.X.Y`, where `X` is
the number of the current release cycle, and `Y` is a counter within the cycle, which will be
incremented for each RC build that is required.

To support these requirements, a `package-arch-artifacts` target is added. It produces packages in
the form, e.g., `2024.07.1.1.autonomi.aarch64-unknown-linux-musl.zip`. The release cycle numbers are
provided from a new file at the root of the repository. Those numbers should be updated or reset in
a release commit.

The new process retains the upload of individual binaries to S3, so those packaging and upload
targets are retained, but they are renamed to `package-bin` and `upload-packaged-bin-to-s3`, to
distinguish them from the architecture packages.

The `upload-github-release-assets` target is completely removed because it was uploading the
individual binaries to their respective Github Releases, which is no longer relevant. The release
workflow will be updated accordingly, but not in this commit.

For testing the new targets, this commit only updates the simpler 'build and package release
artifacts' workflow; a subsequent commit will update the full release workflow.

.github/workflows/build-release-artifacts.yml

@@ -1,18 +1,25 @@
-name: build release artifacts
+# This workflow builds and packages the release artifacts, without actually running a release.
+#
+# It can sometimes be useful to obtain these binaries built from other branches, or a tag, or when
+# the release process is being updated, this workflow can be used to test some of the changes.
+#
+# The built and packaged binaries will be attached to the workflow run as artifacts, available for
+# download.
+name: build and package release artifacts
 
 on:
   workflow_dispatch:
     inputs:
       branch:
-        description: The branch to build.
+        description: Set to build a particular branch
         type: string
       tag:
-        description: The tag to build.
+        description: Set to build a particular tag
         type: string
 
-# Copied from `release.yml`
-# During the build step, the env variable has to be manually sent to the containers for cross platform builds.
-# Update the Justfile as well.
+# Copied from `release.yml`.
+# During the build step, the key variables must be sent to the containers for cross platform builds.
+# If they are changed here, the Justfile also must be updated.
 env:
   JUST_BIN_URL: https://github.com/casey/just/releases/download/1.25.2/just-1.25.2-x86_64-unknown-linux-musl.tar.gz
   WORKFLOW_URL: https://github.com/maidsafe/safe_network/actions/runs
@@ -48,8 +55,6 @@ jobs:
         with:
           ref: ${{ inputs.tag || inputs.branch }}
       - uses: dtolnay/rust-toolchain@stable
-      # cargo-binstall will try and use pre-built binaries if they are available and also speeds up
-      # installing `cross`
       - uses: cargo-bins/cargo-binstall@main
       - shell: bash
         run: cargo binstall --no-confirm just
@@ -63,8 +68,6 @@ jobs:
             artifacts
             !artifacts/.cargo-lock
 
-  # This job isn't necessary, but it's useful for debugging the packaging process for the real release
-  # workflow, just in case any issues are ever encountered there.
   package:
     name: package artifacts
     runs-on: ubuntu-latest
@@ -104,19 +107,21 @@ jobs:
       - uses: cargo-bins/cargo-binstall@main
       - shell: bash
         run: cargo binstall --no-confirm just
-      - name: package artifacts
+
+      - name: package binaries
         shell: bash
         run: |
-          just package-release-assets "faucet"
-          just package-release-assets "nat-detection"
-          just package-release-assets "node-launchpad"
-          just package-release-assets "safe"
-          just package-release-assets "safenode"
-          just package-release-assets "safenode_rpc_client"
-          just package-release-assets "safenode-manager"
-          just package-release-assets "safenodemand"
-          just package-release-assets "sn_auditor"
+          just package-all-bins
       - uses: actions/upload-artifact@main
         with:
           name: packaged_binaries
-          path: deploy
+          path: packaged_bins
+
+      - name: package architectures
+        shell: bash
+        run: |
+          just package-all-architectures
+      - uses: actions/upload-artifact@main
+        with:
+          name: packaged_architectures
+          path: packaged_architectures

.gitignore

@@ -3,6 +3,8 @@
 /target/
 /artifacts/
 /deploy/
+/packaged_architectures/
+/packaged_bins/
 
 # These are backup files generated by rustfmt
 **/*.rs.bk

Justfile

@@ -166,7 +166,20 @@ make-artifacts-directory:
     rm safe_network-$arch.zip
   done
 
-package-release-assets bin version="":
+package-all-bins:
+  #!/usr/bin/env bash
+  set -e
+  just package-bin "faucet"
+  just package-bin "nat-detection"
+  just package-bin "node-launchpad"
+  just package-bin "safe"
+  just package-bin "safenode"
+  just package-bin "safenode_rpc_client"
+  just package-bin "safenode-manager"
+  just package-bin "safenodemand"
+  just package-bin "sn_auditor"
+
+package-bin bin version="":
   #!/usr/bin/env bash
   set -e
 
@@ -225,7 +238,6 @@ package-release-assets bin version="":
     sn_auditor)
       crate_dir_name="sn_auditor"
       ;;
-
     *)
       echo "The $bin binary is not supported"
       exit 1
@@ -244,7 +256,7 @@ package-release-assets bin version="":
     exit 1
   fi
 
-  rm -rf deploy/$bin
+  rm -rf packaged_bins/$bin
   find artifacts/ -name "$bin" -exec chmod +x '{}' \;
   for arch in "${architectures[@]}" ; do
     echo "Packaging for $arch..."
@@ -253,95 +265,30 @@ package-release-assets bin version="":
     tar -C artifacts/$arch/release -zcvf $bin-$version-$arch.tar.gz $bin_name
   done
 
-  mkdir -p deploy/$bin
-  mv *.tar.gz deploy/$bin
-  mv *.zip deploy/$bin
+  mkdir -p packaged_bins/$bin
+  mv *.tar.gz packaged_bins/$bin
+  mv *.zip packaged_bins/$bin
 
-upload-github-release-assets:
+upload-all-packaged-bins-to-s3 bin_name:
   #!/usr/bin/env bash
   set -e
 
-  binary_crates=(
-    "sn_faucet"
-    "node-launchpad"
-    "sn_cli"
-    "sn_node"
-    "sn-node-manager"
-    "sn_node_rpc_client"
-    "sn_auditor"
-    "nat-detection"
+  binaries=(
+    faucet
+    nat-detection
+    node-launchpad
+    safe
+    safenode
+    safenode-manager
+    safenode_rpc_client
+    safenodemand
+    sn_auditor
   )
-
-  commit_msg=$(git log -1 --pretty=%B)
-  commit_msg=${commit_msg#*: } # Remove 'chore(release): ' prefix
-
-  IFS='/' read -ra crates_with_versions <<< "$commit_msg"
-  declare -a crate_names
-  for crate_with_version in "${crates_with_versions[@]}"; do
-    crate=$(echo "$crate_with_version" | awk -F'-v' '{print $1}')
-    crates+=("$crate")
+  for binary in "${binaries[@]}"; do
+    just upload-packaged-bin-to-s3 "$binary"
   done
 
-  for crate in "${crates[@]}"; do
-    for binary_crate in "${binary_crates[@]}"; do
-      if [[ "$crate" == "$binary_crate" ]]; then
-        case "$crate" in
-          sn_faucet)
-            bin_name="faucet"
-            bucket="sn-faucet"
-            ;;
-          node-launchpad)
-            bin_name="node-launchpad"
-            bucket="node-launchpad"
-            ;;
-          sn_cli)
-            bin_name="safe"
-            bucket="sn-cli"
-            ;;
-          sn_node)
-            bin_name="safenode"
-            bucket="sn-node"
-            ;;
-          sn-node-manager)
-            bin_name="safenode-manager"
-            bucket="sn-node-manager"
-            ;;
-          sn_node_rpc_client)
-            bin_name="safenode_rpc_client"
-            bucket="sn-node-rpc-client"
-            ;;
-          sn_auditor)
-            bin_name="sn_auditor"
-            bucket="sn-auditor"
-            ;;
-          nat-detection)
-            bin_name="nat-detection"
-            bucket="nat-detection"
-            ;;
-          *)
-            echo "The $crate crate is not supported"
-            exit 1
-            ;;
-        esac
-        # The crate_with_version variable will correspond to the tag name of the release.
-        # However, only binary crates have releases, so we need to skip any tags that don't
-        # correspond to a binary.
-        for crate_with_version in "${crates_with_versions[@]}"; do
-          if [[ $crate_with_version == $crate-v* ]]; then
-            (
-              cd deploy/$bin_name
-              if [[ "$crate" == "node-launchpad" || "$crate" == "sn_cli" || "$crate" == "sn_node" || "$crate" == "sn-node-manager" || "$crate" == "sn_auditor" ]]; then
-                echo "Uploading $bin_name assets to $crate_with_version release..."
-                ls | xargs gh release upload $crate_with_version --repo {{release_repo}}
-              fi
-            )
-          fi
-        done
-      fi
-    done
-  done
-
-upload-release-assets-to-s3 bin_name:
+upload-packaged-bin-to-s3 bin_name:
   #!/usr/bin/env bash
   set -e
 
@@ -379,7 +326,7 @@ upload-release-assets-to-s3 bin_name:
       ;;
   esac
 
-  cd deploy/{{bin_name}}
+  cd packaged_bins/{{bin_name}}
   for file in *.zip *.tar.gz; do
     dest="s3://$bucket/$file"
     if [[ "$file" == *latest* ]]; then
@@ -397,6 +344,67 @@ upload-release-assets-to-s3 bin_name:
     fi
   done
 
+package-all-arch-artifacts:
+  #!/usr/bin/env bash
+  set -e
+
+  architectures=(
+    "x86_64-pc-windows-msvc"
+    "x86_64-apple-darwin"
+    "aarch64-apple-darwin"
+    "x86_64-unknown-linux-musl"
+    "arm-unknown-linux-musleabi"
+    "armv7-unknown-linux-musleabihf"
+    "aarch64-unknown-linux-musl"
+  )
+
+  rm -rf packaged_architectures
+  for arch in "${architectures[@]}" ; do
+    echo "Packaging artifacts for $arch..."
+    just package-arch-artifacts "$arch"
+  done
+
+package-arch-artifacts arch:
+  #!/usr/bin/env bash
+  set -e
+
+  if [[ -n $PACKAGE_VERSION ]]; then
+    version="$PACKAGE_VERSION"
+  else
+    current_date=$(date +%Y.%m)
+    release_cycle=$(grep 'release-cycle:' release-cycle-info | awk '{print $2}')
+    release_cycle_counter=$(grep 'release-cycle-counter:' release-cycle-info | awk '{print $2}')
+    version="$current_date.$release_cycle.$release_cycle_counter"
+  fi
+  architecture="{{arch}}"
+  zip_filename="${version}.autonomi.${architecture}.zip"
+
+  mkdir -p packaged_architectures
+  cd artifacts/$architecture/release
+
+  binaries=(
+    faucet
+    nat-detection
+    node-launchpad
+    safe
+    safenode
+    safenode-manager
+    safenode_rpc_client
+    safenodemand
+    sn_auditor
+  )
+
+  if [[ "$architecture" == *"windows"* ]]; then
+    for binary in "${binaries[@]}"; do
+      binaries_with_extension+=("$binary.exe")
+    done
+    zip "../../../packaged_architectures/$zip_filename" "${binaries_with_extension[@]}"
+  else
+    zip "../../../packaged_architectures/$zip_filename" "${binaries[@]}"
+  fi
+
+  cd ../../..
+
 node-man-integration-tests:
   #!/usr/bin/env bash
   set -e

release-cycle-info

@@ -0,0 +1,10 @@
+# The release-cycle is the cycle within the current month. It will be 1 or 2. It is set at the
+# beginning of the cycle.
+#
+# The release-cycle-counter is initially set to 1 at the beginning of each cycle, and during the
+# cycle, it will be incremented for each RC build.
+#
+# Both of these numbers are used in the packaged version number, which is a collective version
+# number for all the released binaries.
+release-cycle: 1
+release-cycle-counter: 1