From 324c0531d25c80405c6407b6d8526571075bcf90 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Wed, 8 Jan 2025 08:22:38 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8Infra:=20Build=20for=20arm64=20(#6984)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci-arm-build.yml | 67 +++++++++++++++++++++++++++++ Makefile | 15 ++++++- scripts/install_rclone.bash | 10 +++-- scripts/shellcheck.bash | 2 +- services/agent/Dockerfile | 4 ++ services/docker-bake.hcl | 12 ------ services/dynamic-sidecar/Dockerfile | 9 +++- 7 files changed, 101 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/ci-arm-build.yml delete mode 100644 services/docker-bake.hcl diff --git a/.github/workflows/ci-arm-build.yml b/.github/workflows/ci-arm-build.yml new file mode 100644 index 00000000000..0dca219a690 --- /dev/null +++ b/.github/workflows/ci-arm-build.yml @@ -0,0 +1,67 @@ +name: CI ARM64 Build and Push + +on: + # push: + # branches: + # - "master" + # tags-ignore: + # - "*" + # pull_request: + # branches-ignore: + # - "*" + workflow_dispatch: + +jobs: + build-and-push-arm64: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-22.04] + python: ["3.11"] + env: + # secrets can be set in settings/secrets on github + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + steps: + - uses: actions/checkout@v4 + - name: setup QEMU + uses: docker/setup-qemu-action@v3 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: expose github runtime for buildx + uses: crazy-max/ghaction-github-runtime@v3 + - name: show system environs + run: ./ci/helpers/show_system_versions.bash + - name: login to Dockerhub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set deployment variables + run: | + if [ "${GITHUB_REF}" == "refs/heads/master" ]; then + echo "TAG_PREFIX=master-github" >> $GITHUB_ENV + elif [[ "${GITHUB_REF}" == refs/heads/hotfix_v* ]]; then + echo "TAG_PREFIX=hotfix-github" >> $GITHUB_ENV + elif [[ "${GITHUB_REF}" == refs/heads/hotfix_staging_* ]]; then + echo "TAG_PREFIX=hotfix-staging-github" >> $GITHUB_ENV + fi + - name: build & push images for latest tag + run: | + export DOCKER_IMAGE_TAG="$TAG_PREFIX-latest-arm64" + export DOCKER_TARGET_PLATFORMS=linux/arm64 + make build push=true + - name: build & push images for specific tag + run: | + export DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)-arm64 + export DOCKER_TARGET_PLATFORMS=linux/arm64 + make build push=true + - name: fuse images in the registry for latest tag + run: | + export DOCKER_IMAGE_TAG="$TAG_PREFIX-latest" + make docker-image-fuse SUFFIX=arm64 + - name: set git tag + run: echo "GIT_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV diff --git a/Makefile b/Makefile index b640fe626b8..d8f0ccf252d 100644 --- a/Makefile +++ b/Makefile @@ -172,8 +172,16 @@ docker buildx bake --allow=fs.read=.. \ ,--load\ )\ )\ + $(if $(push),\ + $(foreach service, $(SERVICES_NAMES_TO_BUILD),\ + --set $(service).tags=$(DOCKER_REGISTRY)/$(service):$(DOCKER_IMAGE_TAG) \ + ) \ + $(foreach service, $(SERVICES_NAMES_TO_BUILD),\ + --set $(service).output="type=registry$(comma)push=true" \ + )\ + ,) \ $(if $(push),--push,) \ - $(if $(push),--file docker-bake.hcl,) --file docker-compose-build.yml $(if $(target),$(target),$(INCLUDED_SERVICES)) \ + --file docker-compose-build.yml $(if $(target),$(target),$(INCLUDED_SERVICES)) \ $(if $(findstring -nc,$@),--no-cache,\ $(foreach service, $(SERVICES_NAMES_TO_BUILD),\ --set $(service).cache-to=type=gha$(comma)mode=max$(comma)scope=$(service) \ @@ -859,3 +867,8 @@ release-staging release-prod: .check-on-master-branch ## Helper to create a sta .PHONY: release-hotfix release-staging-hotfix release-hotfix release-staging-hotfix: ## Helper to create a hotfix release in Github (usage: make release-hotfix version=1.2.4 git_sha=optional or make release-staging-hotfix name=Sprint version=2) $(create_github_release_url) + +.PHONY: docker-image-fuse +docker-image-fuse: + $(foreach service, $(SERVICES_NAMES_TO_BUILD),\ + docker buildx imagetools create --tag $(DOCKER_REGISTRY)/$(service):$(DOCKER_IMAGE_TAG) $(DOCKER_REGISTRY)/$(service):$(DOCKER_IMAGE_TAG)-$(SUFFIX) $(DOCKER_REGISTRY)/$(service):$(DOCKER_IMAGE_TAG);) diff --git a/scripts/install_rclone.bash b/scripts/install_rclone.bash index 4b67522b731..e6378cdd9b3 100755 --- a/scripts/install_rclone.bash +++ b/scripts/install_rclone.bash @@ -10,7 +10,11 @@ set -o pipefail # don't hide errors within pipes IFS=$'\n\t' R_CLONE_VERSION="1.63.1" -curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-amd64.deb" -dpkg --install "rclone-v${R_CLONE_VERSION}-linux-amd64.deb" -rm "rclone-v${R_CLONE_VERSION}-linux-amd64.deb" +TARGETARCH="${TARGETARCH:-amd64}" + +echo "platform ${TARGETARCH}" + +curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-${TARGETARCH}.deb" +dpkg --install "rclone-v${R_CLONE_VERSION}-linux-${TARGETARCH}.deb" +rm "rclone-v${R_CLONE_VERSION}-linux-${TARGETARCH}.deb" rclone --version diff --git a/scripts/shellcheck.bash b/scripts/shellcheck.bash index e90777d7fd4..2279a9a86fb 100755 --- a/scripts/shellcheck.bash +++ b/scripts/shellcheck.bash @@ -5,4 +5,4 @@ # - VS extension: https://github.com/timonwong/vscode-shellcheck # -exec docker run --rm --interactive --volume "$PWD:/mnt:ro" koalaman/shellcheck:v0.9.0 "$@" +exec docker run --rm --interactive --volume "$PWD:/mnt:ro" koalaman/shellcheck:v0.10.0 "$@" diff --git a/services/agent/Dockerfile b/services/agent/Dockerfile index 3c7469ac92b..e2920791520 100644 --- a/services/agent/Dockerfile +++ b/services/agent/Dockerfile @@ -3,10 +3,12 @@ # Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" ARG UV_VERSION="0.4" + FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build # we docker image is built based on debian FROM python:${PYTHON_VERSION}-slim-bookworm AS base + # # USAGE: # cd sercices/agent @@ -57,6 +59,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # rclone installation +ARG TARGETARCH +ENV TARGETARCH=${TARGETARCH} RUN \ --mount=type=bind,source=scripts/install_rclone.bash,target=/tmp/install_rclone.bash \ ./tmp/install_rclone.bash diff --git a/services/docker-bake.hcl b/services/docker-bake.hcl deleted file mode 100644 index 7cc0470f177..00000000000 --- a/services/docker-bake.hcl +++ /dev/null @@ -1,12 +0,0 @@ -variable "DOCKER_REGISTRY" { - default = "itisfoundation" -} - -variable "DASK_SIDECAR_VERSION" { - default = "latest" -} - -target "dask-sidecar" { - tags = ["${DOCKER_REGISTRY}/dask-sidecar:latest","${DOCKER_REGISTRY}/dask-sidecar:${DASK_SIDECAR_VERSION}"] - output = ["type=registry"] -} diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile index 6b96c64da25..5ccdceb43ad 100644 --- a/services/dynamic-sidecar/Dockerfile +++ b/services/dynamic-sidecar/Dockerfile @@ -53,6 +53,8 @@ RUN \ && gosu nobody true # install RClone, we do it in a separate layer such that the cache is not locked forever, as this seems to take a long time +ARG TARGETARCH +ENV TARGETARCH=${TARGETARCH} RUN \ --mount=type=bind,source=scripts/install_rclone.bash,target=install_rclone.bash \ ./install_rclone.bash @@ -64,7 +66,12 @@ RUN \ ./install_7zip.bash RUN AWS_CLI_VERSION="2.11.11" \ - && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \ + && case "${TARGETARCH}" in \ + "amd64") ARCH="x86_64" ;; \ + "arm64") ARCH="aarch64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac \ + && curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}-${AWS_CLI_VERSION}.zip" -o "awscliv2.zip" \ && apt-get update && apt-get install -y unzip \ && unzip awscliv2.zip \ && ./aws/install \