release action: Create new release from list, include old release

.github/workflows/release.yaml

@@ -2,7 +2,7 @@ name: Build and release Systemd sysext images
 on:
   push:
     tags:
-      - '*'
+      - 'latest'
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -12,8 +12,8 @@ jobs:
     steps:
       # checkout the sources
       - uses: actions/checkout@v3
-      # build the images and generate a manifest
-      - name: build
+      # prepare build host
+      - name: install prerequisites
         run: |
           set -euxo pipefail
 
@@ -24,70 +24,19 @@ jobs:
             xz-utils \
             gawk
 
-          KBS_VERS=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20  https://raw.githubusercontent.com/kubernetes/website/main/data/releases/schedule.yaml | yq -r '.schedules[].previousPatches[0].release' | awk '{print "kubernetes-v"$1}')
-          [[ -z "${KBS_VERS}" ]] && echo "Failed fetching Kubernetes versions" && exit 1
+      - name: build release artifacts
+        run: release_build.sh
 
-          KBS_VERS_ARRAY=(${KBS_VERS})
+      - name: delete previous latest release
+  	    uses: actions/checkout@v3
+        run: gh release delete latest --cleanup-tag
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-          images=(
-              "docker-24.0.9"
-              "docker_compose-2.22.0"
-              "wasmtime-13.0.0"
-          )
-          images+=("${KBS_VERS_ARRAY[@]}")
-
-          streams=()
-
-          for image in "${images[@]}"; do
-              component="${image%-*}"
-              version="${image#*-}"
-              for arch in x86-64 arm64; do
-                ARCH="${arch}" "./create_${component}_sysext.sh" "${version}" "${component}"
-                mv "${component}.raw" "${image}-${arch}.raw"
-              done
-              streams+=("${component}:-@v")
-              if [ "${component}" = "kubernetes" ]; then
-                streams+=("kubernetes-${version%.*}:.@v")
-                # Should give, e.g., v1.28 for v1.28.2 (use ${version#*.*.} to get 2)
-              fi
-          done
-          for stream in "${streams[@]}"; do
-              component="${stream%:*}"
-              pattern="${stream#*:}"
-              cat << EOF > "${component}.conf"
-           [Transfer]
-           Verify=false
-           [Source]
-           Type=url-file
-           Path=https://github.com/flatcar/sysext-bakery/releases/latest/download/
-           MatchPattern=${component}${pattern}-%a.raw
-           [Target]
-           InstancesMax=3
-           Type=regular-file
-           Path=/opt/extensions/${component%-*}
-           CurrentSymlink=/etc/extensions/${component%-*}.raw
-          EOF
-          done
-
-          cat << EOF > "noop.conf"
-          [Source]
-          Type=regular-file
-          Path=/
-          [email protected]
-          [Target]
-          Type=regular-file
-          Path=/
-          EOF
-
-          # Fetch the current SHA256SUMS to append to it the new list of sha256 sums.
-          curl -fsSLO https://github.com/flatcar/sysext-bakery/releases/download/latest/SHA256SUMS
-          sha256sum *.raw >> SHA256SUMS
-          sort --unique --key 2 --output SHA256SUMS SHA256SUMS
-
-      # create a Github release with the generated artifacts
-      - name: release
+      - name: create a new latest release with all artifacts
         uses: softprops/action-gh-release@v1
         with:
+          body_path: Release.md
           files: |
             SHA256SUMS
             *.raw

README.md

@@ -91,9 +91,9 @@ storage:
     - path: /opt/extensions/wasmtime/wasmtime-13.0.0-x86-64.raw
       contents:
         source: https://github.com/flatcar/sysext-bakery/releases/download/latest/wasmtime-13.0.0-x86-64.raw
-    - path: /opt/extensions/docker/docker-24.0.5-x86-64.raw
+    - path: /opt/extensions/docker/docker-24.0.9-x86-64.raw
       contents:
-        source: https://github.com/flatcar/sysext-bakery/releases/download/latest/docker-24.0.5-x86-64.raw
+        source: https://github.com/flatcar/sysext-bakery/releases/download/latest/docker-24.0.9-x86-64.raw
     - path: /etc/systemd/system-generators/torcx-generator
     - path: /etc/sysupdate.d/noop.conf
       contents:
@@ -108,7 +108,7 @@ storage:
     - target: /opt/extensions/wasmtime/wasmtime-13.0.0-x86-64.raw
       path: /etc/extensions/wasmtime.raw
       hard: false
-    - target: /opt/extensions/docker/docker-24.0.5-x86-64.raw
+    - target: /opt/extensions/docker/docker-24.0.9-x86-64.raw
       path: /etc/extensions/docker.raw
       hard: false
     - path: /etc/extensions/docker-flatcar.raw
@@ -150,14 +150,14 @@ The Docker releases publish static binaries including containerd and the only mi
 To ease the process, the `create_docker_sysext.sh` helper script takes care of downloading the release binaries and adding the systemd unit files, and creates a combined Docker+containerd sysext image:
 
 ```
-./create_docker_sysext.sh 20.10.13 mydocker
+./create_docker_sysext.sh 24.0.9 mydocker
 [… writes mydocker.raw into current directory …]
 ```
 
 Pass the `OS` or `ARCH` environment variables to build for another target than Flatcar amd64, e.g., for any distro with arm64:
 
 ```
-OS=_any ARCH=arm64 ./create_docker_sysext.sh 20.10.13 mydocker
+OS=_any ARCH=arm64 ./create_docker_sysext.sh 24.0.9 mydocker
 [… writes mydocker.raw into current directory …]
 ```
 

release_build.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+#
+# Build a bakery release of all sysexts.
+#
+# The release will include all sysexts from the "latest" release
+# (these will be downloaded). Sysexts listed in release_build_versions.txt
+# and _not_ included in the "latest" release will be built.
+
+set -euo pipefail
+
+
+echo
+echo "Fetching list of latest Kubernetes minor releases"
+echo "================================================="
+KBS_VERS=$(curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused \
+                --retry-max-time 60 --connect-timeout 20  \
+                https://raw.githubusercontent.com/kubernetes/website/main/data/releases/schedule.yaml \
+                | yq -r '.schedules[].previousPatches[0].release' \
+                | awk '{print "kubernetes-v"$1}')
+if [[ -z "${KBS_VERS}" ]] ; then
+    echo "Failed fetching Kubernetes versions"
+    exit 1
+fi
+
+KBS_VERS_ARRAY=(${KBS_VERS})
+printf "%s\n" "${KBS_VERS_ARRAY[@]}"
+
+echo
+echo "Fetching previous 'latest' release sysexts"
+echo "=========================================="
+curl -fsSL --retry-delay 1 --retry 60 --retry-connrefused \
+         --retry-max-time 60 --connect-timeout 20  \
+         https://api.github.com/repos/flatcar/sysext-bakery/releases/latest \
+    | jq -r '.assets[].browser_download_url' | grep -E '\.raw$' | tee prev_release_sysexts.txt
+
+for asset in $(cat prev_release_sysexts.txt); do
+    echo
+    echo "  ## Fetching $(basename "${asset}") <-- ${asset}"
+    wget "${asset}"
+done
+
+streams=()
+
+echo
+echo "Building sysexts"
+echo "================"
+
+mapfile -t images < <( awk '{ content=sub("[[:space:]]*#.*", ""); if ($0) print $0; }' \
+                       release_build_versions.txt )
+images+=("${KBS_VERS_ARRAY[@]}")
+
+echo "# Release 2024-02-01 16:44:51" > Release.md
+echo "The release adds the following sysexts:" >> Release.md
+
+for image in "${images[@]}"; do
+  component="${image%-*}"
+  version="${image#*-}"
+  for arch in x86-64 arm64; do
+    target="${image}-${arch}.raw"
+    if [ -f "${target}" ] ; then
+        echo "  ## Skipping ${target} because it already exists (asset from previous release)"
+        continue
+    fi
+    echo "  ## Building ${target}."
+    ARCH="${arch}" "./create_${component}_sysext.sh" "${version}" "${component}"
+    mv "${component}.raw" "${target}"
+    echo "* ${target}" >> Release.md
+  done
+  streams+=("${component}:-@v")
+  if [ "${component}" = "kubernetes" ]; then
+    streams+=("kubernetes-${version%.*}:.@v")
+    # Should give, e.g., v1.28 for v1.28.2 (use ${version#*.*.} to get 2)
+  fi
+done
+
+echo "" >> Release.md
+echo "The release includes the following sysexts from previous releases:" >> Release.md
+sed 's/^/* /' prev_release_sysexts.txt >> Release.md
+
+echo
+echo "Generating systemd-sysupdate configurations and SHA256SUM."
+echo "=========================================================="
+
+for stream in "${streams[@]}"; do
+  component="${stream%:*}"
+  pattern="${stream#*:}"
+  cat << EOF > "${component}.conf"
+[Transfer]
+Verify=false
+[Source]
+Type=url-file
+Path=https://github.com/flatcar/sysext-bakery/releases/latest/download/
+MatchPattern=${component}${pattern}-%a.raw
+[Target]
+InstancesMax=3
+Type=regular-file
+Path=/opt/extensions/${component%-*}
+CurrentSymlink=/etc/extensions/${component%-*}.raw
+EOF
+done
+
+cat << EOF > "noop.conf"
+[Source]
+Type=regular-file
+Path=/
+[email protected]
+[Target]
+Type=regular-file
+Path=/
+EOF
+
+# Generate new SHA256SUMS from all assets
+sha256sum *.raw | tee SHA256SUMS

release_build_versions.txt

@@ -0,0 +1,17 @@
+# Versions to build.
+
+# For Kubernetes, we fetch the latest versions to be built.
+# The below lists _additional_ kubernetes versions to be built.
+
+kubernetes-v1.28.5 # required for CAPO CI
+
+
+
+docker-24.0.9 # Used in README.md. Update readme when version changes.
+docker-25.0.2
+
+docker_compose-2.22.0
+docker_compose-2.24.5
+
+wasmtime-12.0.0
+wasmtime-13.0.0 # Used in README.md. Update readme when version changes.