fix(ci): add a "build-id" input var to ci builds

.github/actions/clean-runner/action.yaml

@@ -0,0 +1,12 @@
+name: 'Clean runner'
+description: 'Remove unneeded tooling'
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        # To free up ~15 GB of disk space
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /usr/local/share/boost
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /usr/share/dotnet

.github/actions/show-disk-usage/action.yaml

@@ -0,0 +1,20 @@
+name: 'Show disk usage'
+description: 'Show information about disk usage'
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        cd $GITHUB_WORKSPACE
+        set -x
+        df -h
+        sudo ls -lRh /tmp/* || true
+        sudo du -sh /tmp || true
+        sudo du -sh /tmp/* || true
+        sudo find /tmp/ -size +5M | sudo xargs ls -lh
+        sudo du -sh ./* || true
+        sudo find ./ -size +5M   | xargs ls -lh
+        sudo du -sh /var/
+        sudo du -sh /var/lib/docker/
+        sudo du -sh /home/runner/work/
+        set +x

.github/workflows/build.yaml

@@ -2,34 +2,51 @@ name: Reusable stacker build
 on:
   workflow_call:
     inputs:
+      # note >-, args needs to be strings to be used as inputs
+      # for the reusable build.yaml workflow
       go-version:
-        required: true
+        required: false
         type: string
         description: 'Stringified JSON object listing go versions'
+        default: >-
+          ["1.21.x"]
       privilege-level:
-        required: true
+        required: false
         type: string
         description: 'Stringified JSON object listing stacker privilege-level'
+        default: >-
+          ["unpriv", "priv"]
       build-id:
-        required: true
+        required: false
         type: string
-        description: 'Stringified JSON object listing stacker build-id'
+        description: 'build-id'
+        default: "${{ github.sha }}"
+      slow-test:
+        required: false
+        type: boolean
+        description: 'Should slow tests be run?'
+        default: true
+    secrets:
+      codecov_token:
+        required: true
+
 jobs:
   build:
     runs-on: ubuntu-22.04
     services:
       registry:
-        image: registry:2
+        image: ghcr.io/project-stacker/registry:2
         ports:
           - 5000:5000
     strategy:
       matrix:
         go-version: ${{fromJson(inputs.go-version)}}
         privilege-level: ${{fromJson(inputs.privilege-level)}}
-        build-id: ${{fromJson(inputs.build-id)}}
     name: "golang ${{ matrix.go-version }} privilege ${{ matrix.privilege-level }}"
     steps:
       - uses: actions/checkout@v3
+      - name: Clean disk space
+        uses: ./.github/actions/clean-runner
       - uses: benjlevesque/[email protected]
         id: short-sha
       - name: Set up golang ${{ matrix.go-version }}
@@ -42,39 +59,61 @@ jobs:
           echo "GOPATH=$gopath" >> $GITHUB_ENV
           echo "GOCACHE=$gopath/gocache" >> $GITHUB_ENV
           echo "PATH=$gopath/bin:$PATH" >> $GITHUB_ENV
+          echo "SLOW_TEST=${{inputs.slow-test}}" >> $GITHUB_ENV
+          echo "STACKER_DOCKER_BASE=oci:$PWD/.build/oci-clone:" >> $GITHUB_ENV
 
           echo "PWD=$PWD"
           cat "$GITHUB_ENV"
       - name: install dependencies
         run: |
-          sudo add-apt-repository -y ppa:project-machine/squashfuse
-          sudo apt-get update
-          sudo apt-get install -yy lxc-utils lxc-dev libacl1-dev jq libcap-dev libseccomp-dev libpam-dev bats parallel libzstd-dev
-          GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin
-          sudo apt-get install -yy autoconf automake make autogen autoconf libtool binutils git squashfs-tools libcryptsetup-dev libdevmapper-dev cryptsetup-bin squashfuse
+          ./install-build-deps.sh
           echo "running kernel is: $(uname -a)"
+      - name: docker-clone
+        run: |
+          make docker-clone "STACKER_DOCKER_BASE=docker://" CLONE_D="$PWD/.build/oci-clone"
       - name: Go-download
         run: |
           make go-download
+      - name: Show disk usage before building the binaries
+        uses: ./.github/actions/show-disk-usage
       - name: Build-level1
         run: |
           make show-info
-          make stacker-dynamic VERSION_FULL=${{ matrix.build-id }}
+          make stacker-dynamic VERSION_FULL=${{ inputs.build-id }}
       - name: Build
         run: |
-          make stacker VERSION_FULL=${{ matrix.build-id }}
+          make stacker VERSION_FULL=${{ inputs.build-id }}
         env:
           REGISTRY_URL: localhost:5000
+          ZOT_HOST: localhost
+          ZOT_PORT: 8080
+      - name: Show disk usage before running the tests
+        if: always()
+        uses: ./.github/actions/show-disk-usage
       - name: Test
         run: |
-          make check VERSION_FULL=${{ matrix.build-id }} PRIVILEGE_LEVEL=${{ matrix.privilege-level }}
+          make check VERSION_FULL=${{ inputs.build-id }} PRIVILEGE_LEVEL=${{ matrix.privilege-level }}
         env:
           REGISTRY_URL: localhost:5000
+          ZOT_HOST: localhost
+          ZOT_PORT: 8080
+      - name: Show disk usage after running the tests
+        if: always()
+        uses: ./.github/actions/show-disk-usage
       - name: Upload code coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.codecov_token }}
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: ${{ matrix.privilege-level == 'priv' }}
+        with:
+          # if there is more than 1 go-version, we would need to account for that here.
+          name: binary
+          path: stacker
+          if-no-files-found: error
       - uses: actions/cache@v3
         id: restore-build
         with:
           path: stacker
-          key: ${{ matrix.build-id }}
+          key: ${{ inputs.build-id }}

.github/workflows/ci.yaml

@@ -12,11 +12,12 @@ jobs:
   build:
     uses: ./.github/workflows/build.yaml
     with:
-      # note >-, args needs to be strings to be used as inputs
-      # for the reusable build.yaml workflow
-      go-version: >-
-        ["1.20.x"]
-      privilege-level: >-
-        ["unpriv", "priv"]
-      build-id: >-
-        ["${{ github.sha }}"]
+      slow-test: false
+    secrets:
+      codecov_token: ${{ secrets.CODECOV_TOKEN }}
+  coverage:
+    uses: ./.github/workflows/coverage.yaml
+    with:
+      slow-test: false
+    secrets:
+      codecov_token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/cloc.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
       - name: Check out source code
         uses: actions/checkout@v3
       - name: Install dependencies

.github/workflows/codeql-analysis.yml

@@ -48,7 +48,7 @@ jobs:
     - name: Install go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/coverage.yaml

@@ -0,0 +1,117 @@
+name: Reusable stacker build for coverage
+on:
+  workflow_call:
+    inputs:
+      # note >-, args needs to be strings to be used as inputs
+      # for the reusable build.yaml workflow
+      go-version:
+        required: false
+        type: string
+        description: 'Stringified JSON object listing go versions'
+        default: >-
+          ["1.21.x"]
+      privilege-level:
+        required: false
+        type: string
+        description: 'Stringified JSON object listing stacker privilege-level'
+        default: >-
+          ["unpriv", "priv"]
+      build-id:
+        required: false
+        type: string
+        description: 'build-id'
+        default: "${{ github.sha }}"
+      slow-test:
+        required: false
+        type: boolean
+        description: 'Should slow tests be run?'
+        default: true
+    secrets:
+      codecov_token:
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    services:
+      registry:
+        image: ghcr.io/project-stacker/registry:2
+        ports:
+          - 5000:5000
+    strategy:
+      matrix:
+        go-version: ${{fromJson(inputs.go-version)}}
+        privilege-level: ${{fromJson(inputs.privilege-level)}}
+    name: "golang ${{ matrix.go-version }} privilege ${{ matrix.privilege-level }}"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Clean disk space
+        uses: ./.github/actions/clean-runner
+      - uses: benjlevesque/[email protected]
+        id: short-sha
+      - name: Set up golang ${{ matrix.go-version }}
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Setup Environment
+        run: |
+          gopath=$PWD/.build/gopath
+          echo "GOPATH=$gopath" >> $GITHUB_ENV
+          echo "GOCACHE=$gopath/gocache" >> $GITHUB_ENV
+          echo "PATH=$gopath/bin:$PATH" >> $GITHUB_ENV
+          echo "SLOW_TEST=${{inputs.slow-test}}" >> $GITHUB_ENV
+          echo "STACKER_DOCKER_BASE=oci:$PWD/.build/oci-clone:" >> $GITHUB_ENV
+          GOCOVERDIR=$(mktemp -d)
+          echo "GOCOVERDIR=$GOCOVERDIR" >> $GITHUB_ENV
+          echo "PWD=$PWD"
+          cat "$GITHUB_ENV"
+      - name: install dependencies
+        run: |
+          ./install-build-deps.sh
+          echo "running kernel is: $(uname -a)"
+      - name: docker-clone
+        run: |
+          make docker-clone "STACKER_DOCKER_BASE=docker://" CLONE_D="$PWD/.build/oci-clone"
+      - name: Go-download
+        run: |
+          make go-download
+      - name: Show disk usage before building the binaries
+        uses: ./.github/actions/show-disk-usage
+      - name: Build-level1
+        run: |
+          make show-info
+          make stacker-dynamic VERSION_FULL=${{ inputs.build-id }}
+      - name: Show disk usage before running the tests
+        if: always()
+        uses: ./.github/actions/show-disk-usage
+      - name: Build and test
+        run: |
+          make check-cov GOCOVERDIR=$GOCOVERDIR PRIVILEGE_LEVEL=${{ matrix.privilege-level }}
+          go tool covdata textfmt -i $GOCOVERDIR -o coverage-${{ matrix.privilege-level }}.txt
+          go tool covdata percent -i $GOCOVERDIR
+          ls -altR $GOCOVERDIR
+        env:
+          REGISTRY_URL: localhost:5000
+          ZOT_HOST: localhost
+          ZOT_PORT: 8080
+      - name: Show disk usage after running the tests
+        if: always()
+        uses: ./.github/actions/show-disk-usage
+      - name: Upload code coverage
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.codecov_token }}
+          files: coverage-${{ matrix.privilege-level}}.txt
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: ${{ matrix.privilege-level == 'priv' }}
+        with:
+          # if there is more than 1 go-version, we would need to account for that here.
+          name: binary-cov
+          path: stacker
+          if-no-files-found: error
+      - uses: actions/cache@v3
+        id: restore-build
+        with:
+          path: stacker
+          key: ${{ inputs.build-id }}

.github/workflows/nightly.yaml

@@ -1,34 +1,12 @@
-name: "Nightly"
+name: "nightly"
 
 on:
   workflow_dispatch:
   schedule:
     - cron: '0 0 * * *'
 
-env:
-  REGISTRY_URL: localhost:5000
-
 jobs:
-  slow-tests:
-    name: "slow tests"
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/cache@v3
-        id: restore-build
-        with:
-          path: stacker
-          key: ${{ github.sha }}
-      - name: zot registry
-        env:
-          ZOT_HOST: localhost
-          ZOT_PORT: 5000
-        run: |
-          # start a zot instance (minimal)
-          podman run -d -p ${ZOT_PORT}:${ZOT_PORT} ghcr.io/project-zot/zot-minimal-linux-amd64:latest
-          # check if reachable
-          while true; do x=0; curl -f http://${REGISTRY_URL}/v2/ || x=1; if [ $x -eq 0 ]; then break; fi; sleep 1; done
-      - name: test
-        env:
-          SLOW_TEST: true
-        run: |
-          make check
+  build:
+    uses: ./.github/workflows/build.yaml
+    secrets:
+      codecov_token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/release.yaml

@@ -23,11 +23,12 @@ jobs:
       # note >-, args needs to be strings to be used as inputs
       # for the reusable build.yaml workflow
       go-version: >-
-        ["1.20.x"]
+        ["1.21.x"]
       privilege-level: >-
         ["priv"]
-      build-id: >-
-        ["${{needs.build-id.outputs.build-id}}"]
+      build-id: "${{needs.build-id.outputs.build-id}}"
+    secrets:
+      codecov_token: ${{ secrets.CODECOV_TOKEN }}
   release:
     name: "Tagged Release"
     runs-on: ubuntu-20.04