✨Infra: Build for arm64 (#6984)

.github/workflows/ci-arm-build.yml

@@ -0,0 +1,67 @@
+name: CI ARM64 Build and Push
+
+on:
+  # push:
+  #   branches:
+  #     - "master"
+  #   tags-ignore:
+  #     - "*"
+  # pull_request:
+  #   branches-ignore:
+  #     - "*"
+  workflow_dispatch:
+
+jobs:
+  build-and-push-arm64:
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+        python: ["3.11"]
+    env:
+      # secrets can be set in settings/secrets on github
+      DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+      - name: expose github runtime for buildx
+        uses: crazy-max/ghaction-github-runtime@v3
+      - name: show system environs
+        run: ./ci/helpers/show_system_versions.bash
+      - name: login to Dockerhub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Set deployment variables
+        run: |
+          if [ "${GITHUB_REF}" == "refs/heads/master" ]; then
+            echo "TAG_PREFIX=master-github" >> $GITHUB_ENV
+          elif [[ "${GITHUB_REF}" == refs/heads/hotfix_v* ]]; then
+            echo "TAG_PREFIX=hotfix-github" >> $GITHUB_ENV
+          elif [[ "${GITHUB_REF}" == refs/heads/hotfix_staging_* ]]; then
+            echo "TAG_PREFIX=hotfix-staging-github" >> $GITHUB_ENV
+          fi
+      - name: build & push images for latest tag
+        run: |
+          export DOCKER_IMAGE_TAG="$TAG_PREFIX-latest-arm64"
+          export DOCKER_TARGET_PLATFORMS=linux/arm64
+          make build push=true
+      - name: build & push images for specific tag
+        run: |
+          export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)-arm64
+          export DOCKER_TARGET_PLATFORMS=linux/arm64
+          make build push=true
+      - name: fuse images in the registry for latest tag
+        run: |
+          export DOCKER_IMAGE_TAG="$TAG_PREFIX-latest"
+          make docker-image-fuse SUFFIX=arm64
+      - name: set git tag
+        run: echo "GIT_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV

Makefile

@@ -172,8 +172,16 @@ docker buildx bake --allow=fs.read=.. \
 			,--load\
 		)\
 	)\
+	$(if $(push),\
+		$(foreach service, $(SERVICES_NAMES_TO_BUILD),\
+				--set $(service).tags=$(DOCKER_REGISTRY)/$(service):$(DOCKER_IMAGE_TAG) \
+		) \
+		$(foreach service, $(SERVICES_NAMES_TO_BUILD),\
+			--set $(service).output="type=registry$(comma)push=true" \
+		)\
+	,) \
 	$(if $(push),--push,) \
-	$(if $(push),--file docker-bake.hcl,) --file docker-compose-build.yml $(if $(target),$(target),$(INCLUDED_SERVICES)) \
+	--file docker-compose-build.yml $(if $(target),$(target),$(INCLUDED_SERVICES)) \
 	$(if $(findstring -nc,$@),--no-cache,\
 		$(foreach service, $(SERVICES_NAMES_TO_BUILD),\
 			--set $(service).cache-to=type=gha$(comma)mode=max$(comma)scope=$(service) \
@@ -859,3 +867,8 @@ release-staging release-prod: .check-on-master-branch  ## Helper to create a sta
 .PHONY: release-hotfix release-staging-hotfix
 release-hotfix release-staging-hotfix: ## Helper to create a hotfix release in Github (usage: make release-hotfix version=1.2.4 git_sha=optional or make release-staging-hotfix name=Sprint version=2)
 	$(create_github_release_url)
+
+.PHONY: docker-image-fuse
+docker-image-fuse:
+	$(foreach service, $(SERVICES_NAMES_TO_BUILD),\
+		docker buildx imagetools create --tag $(DOCKER_REGISTRY)/$(service):$(DOCKER_IMAGE_TAG) $(DOCKER_REGISTRY)/$(service):$(DOCKER_IMAGE_TAG)-$(SUFFIX) $(DOCKER_REGISTRY)/$(service):$(DOCKER_IMAGE_TAG);)

scripts/install_rclone.bash

@@ -10,7 +10,11 @@ set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 R_CLONE_VERSION="1.63.1"
-curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
-dpkg --install "rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
-rm "rclone-v${R_CLONE_VERSION}-linux-amd64.deb"
+TARGETARCH="${TARGETARCH:-amd64}"
+
+echo "platform ${TARGETARCH}"
+
+curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-${TARGETARCH}.deb"
+dpkg --install "rclone-v${R_CLONE_VERSION}-linux-${TARGETARCH}.deb"
+rm "rclone-v${R_CLONE_VERSION}-linux-${TARGETARCH}.deb"
 rclone --version

scripts/shellcheck.bash

@@ -5,4 +5,4 @@
 # - VS extension: https://github.com/timonwong/vscode-shellcheck
 #
 
-exec docker run --rm --interactive --volume "$PWD:/mnt:ro" koalaman/shellcheck:v0.9.0 "$@"
+exec docker run --rm --interactive --volume "$PWD:/mnt:ro" koalaman/shellcheck:v0.10.0 "$@"

services/agent/Dockerfile

@@ -3,10 +3,12 @@
 # Define arguments in the global scope
 ARG PYTHON_VERSION="3.11.9"
 ARG UV_VERSION="0.4"
+
 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build
 # we docker image is built based on debian
 FROM python:${PYTHON_VERSION}-slim-bookworm AS base
 
+
 #
 #  USAGE:
 #     cd sercices/agent
@@ -57,6 +59,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 
 # rclone installation
+ARG TARGETARCH
+ENV TARGETARCH=${TARGETARCH}
 RUN \
   --mount=type=bind,source=scripts/install_rclone.bash,target=/tmp/install_rclone.bash \
   ./tmp/install_rclone.bash

services/dynamic-sidecar/Dockerfile

@@ -53,6 +53,8 @@ RUN \
   && gosu nobody true
 
 # install RClone, we do it in a separate layer such that the cache is not locked forever, as this seems to take a long time
+ARG TARGETARCH
+ENV TARGETARCH=${TARGETARCH}
 RUN \
   --mount=type=bind,source=scripts/install_rclone.bash,target=install_rclone.bash \
   ./install_rclone.bash
@@ -64,7 +66,12 @@ RUN \
   ./install_7zip.bash
 
 RUN AWS_CLI_VERSION="2.11.11" \
-  && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \
+  && case "${TARGETARCH}" in \
+  "amd64") ARCH="x86_64" ;; \
+  "arm64") ARCH="aarch64" ;; \
+  *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \
+  esac \
+  && curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \
   && apt-get update && apt-get install -y unzip \
   && unzip awscliv2.zip \
   && ./aws/install \