From ce6b4163f5b600a5cdb94270a0eb7e9a54239795 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 20 Sep 2021 03:25:29 -0500 Subject: [PATCH 1/3] Upgrade image stack with new features and add placeholder for ARM builds. --- .github/workflows/development.yaml | 97 +++++++++++++++++++++++++++--- config/.env | 5 +- dist/alpine/Dockerfile | 13 ++-- dist/alpine/docker-compose.yaml | 2 +- dist/debian/Dockerfile | 14 ++--- dist/debian/docker-compose.yaml | 2 +- docs/index.rst | 2 +- tests/main.sh | 21 ++++--- 8 files changed, 117 insertions(+), 39 deletions(-) diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml index 356fd96..9dc38fe 100644 --- a/.github/workflows/development.yaml +++ b/.github/workflows/development.yaml @@ -11,48 +11,78 @@ jobs: - distro: alpine conda_ver: 4.10.3 py_ver: 3.9 + platform: linux/amd64 - distro: alpine conda_ver: 4.10.3 py_ver: 3.8 + platform: linux/amd64 - distro: alpine conda_ver: 4.10.3 py_ver: 3.7 + platform: linux/amd64 - distro: alpine conda_ver: 4.5.4 py_ver: 3.6 + platform: linux/amd64 - distro: debian conda_ver: 4.10.3 py_ver: 3.9 + platform: linux/amd64 - distro: debian conda_ver: 4.10.3 py_ver: 3.8 + platform: linux/amd64 - distro: debian conda_ver: 4.10.3 py_ver: 3.7 + platform: linux/amd64 - distro: debian conda_ver: 4.5.4 py_ver: 3.6 + platform: linux/amd64 + # - distro: debian + # conda_ver: 4.10.3 + # py_ver: 3.9 + # platform: linux/arm64 + # - distro: debian + # conda_ver: 4.10.3 + # py_ver: 3.8 + # platform: linux/arm64 + # - distro: debian + # conda_ver: 4.10.3 + # py_ver: 3.7 + # platform: linux/arm64 env: DISTRO: ${{matrix.distro}} CONDA_VER: ${{matrix.conda_ver}} PY_VER: ${{matrix.py_ver}} + PLATFORM: ${{matrix.platform}} DOCKER_CLIENT_TIMEOUT: "120" COMPOSE_HTTP_TIMEOUT: "120" - BASE_IMAGE_HASH: "8281c8f" + BASE_IMAGE_HASH: "344d9f2" steps: - uses: actions/checkout@v2 + - name: Determine platform tag + run: | + PLATFORM_TAG=$(echo "$PLATFORM" | tr '/' '_') + echo "PLATFORM_TAG=${PLATFORM_TAG}" >> $GITHUB_ENV + - uses: dbhi/qus/action@main + with: + targets: aarch64 - name: Compile image run: | - docker-compose -f dist/${DISTRO}/docker-compose.yaml build + docker buildx bake -f dist/${DISTRO}/docker-compose.yaml \ + --set *.platform=${PLATFORM} --set *.context=. --load REF=$(eval "echo $(cat dist/${DISTRO}/docker-compose.yaml | grep 'image:' | \ awk '{print $2}')") - docker save "${REF}" | gzip > "py${PY_VER}-${DISTRO}.tar.gz" + docker save "${REF}" | gzip > "py${PY_VER}-${DISTRO}-${PLATFORM_TAG}.tar.gz" - name: Add image artifact uses: actions/upload-artifact@v2 with: - name: ${{format('py{0}-{1}', matrix.py_ver, matrix.distro)}} + name: ${{format('py{0}-{1}-{2}', matrix.py_ver, matrix.distro, env.PLATFORM_TAG)}} path: > - ${{format('py{0}-{1}.tar.gz', matrix.py_ver, matrix.distro)}} + ${{format('py{0}-{1}-{2}.tar.gz', matrix.py_ver, matrix.distro, + env.PLATFORM_TAG)}} retention-days: 1 test: needs: build @@ -62,32 +92,57 @@ jobs: include: - distro: alpine py_ver: 3.9 + platform: linux/amd64 - distro: alpine py_ver: 3.8 + platform: linux/amd64 - distro: alpine py_ver: 3.7 + platform: linux/amd64 - distro: alpine py_ver: 3.6 + platform: linux/amd64 - distro: debian py_ver: 3.9 + platform: linux/amd64 - distro: debian py_ver: 3.8 + platform: linux/amd64 - distro: debian py_ver: 3.7 + platform: linux/amd64 - distro: debian py_ver: 3.6 + platform: linux/amd64 + # - distro: debian + # py_ver: 3.9 + # platform: linux/arm64 + # - distro: debian + # py_ver: 3.8 + # platform: linux/arm64 + # - distro: debian + # py_ver: 3.7 + # platform: linux/arm64 env: DISTRO: ${{matrix.distro}} PY_VER: ${{matrix.py_ver}} + PLATFORM: ${{matrix.platform}} steps: - uses: actions/checkout@v2 + - name: Determine platform tag + run: | + PLATFORM_TAG=$(echo "$PLATFORM" | tr '/' '_') + echo "PLATFORM_TAG=${PLATFORM_TAG}" >> $GITHUB_ENV - name: Fetch image artifact uses: actions/download-artifact@v2 with: - name: ${{format('py{0}-{1}', matrix.py_ver, matrix.distro)}} + name: ${{format('py{0}-{1}-{2}', matrix.py_ver, matrix.distro, env.PLATFORM_TAG)}} + - uses: dbhi/qus/action@main + with: + targets: aarch64 - name: Run integration tests run: | - docker load < "py${PY_VER}-${DISTRO}.tar.gz" + docker load < "py${PY_VER}-${DISTRO}-${PLATFORM_TAG}.tar.gz" tests/main.sh publish: if: github.event_name == 'push' && github.ref == 'refs/heads/master' @@ -98,35 +153,57 @@ jobs: include: - distro: alpine py_ver: 3.9 + platform: linux/amd64 - distro: alpine py_ver: 3.8 + platform: linux/amd64 - distro: alpine py_ver: 3.7 + platform: linux/amd64 - distro: alpine py_ver: 3.6 + platform: linux/amd64 - distro: debian py_ver: 3.9 + platform: linux/amd64 - distro: debian py_ver: 3.8 + platform: linux/amd64 - distro: debian py_ver: 3.7 + platform: linux/amd64 - distro: debian py_ver: 3.6 + platform: linux/amd64 + # - distro: debian + # py_ver: 3.9 + # platform: linux/arm64 + # - distro: debian + # py_ver: 3.8 + # platform: linux/arm64 + # - distro: debian + # py_ver: 3.7 + # platform: linux/arm64 env: DISTRO: ${{matrix.distro}} PY_VER: ${{matrix.py_ver}} + PLATFORM: ${{matrix.platform}} DOCKER_USERNAME: ${{secrets.docker_username}} DOCKER_PASSWORD: ${{secrets.docker_password}} steps: - uses: actions/checkout@v2 + - name: Determine platform tag + run: | + PLATFORM_TAG=$(echo "$PLATFORM" | tr '/' '_') + echo "PLATFORM_TAG=${PLATFORM_TAG}" >> $GITHUB_ENV - name: Fetch image artifact uses: actions/download-artifact@v2 with: - name: ${{format('py{0}-{1}', matrix.py_ver, matrix.distro)}} + name: ${{format('py{0}-{1}-{2}', matrix.py_ver, matrix.distro, env.PLATFORM_TAG)}} - name: Publish image run: | echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker load < "py${PY_VER}-${DISTRO}.tar.gz" + docker load < "py${PY_VER}-${DISTRO}-${PLATFORM_TAG}.tar.gz" REF=$(eval "echo $(cat dist/${DISTRO}/docker-compose.yaml | grep 'image:' | \ awk '{print $2}')") TAG=$(echo $REF | awk -F':' '{print $2}') @@ -142,7 +219,7 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/master' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Build docs uses: ammaraskar/sphinx-action@master with: diff --git a/config/.env b/config/.env index 9da3385..9e7b19e 100644 --- a/config/.env +++ b/config/.env @@ -12,13 +12,16 @@ CONDA_VER=4.10.3 # PY_VER=3.6 # CONDA_VER=4.5.4 -BASE_IMAGE_HASH=8281c8f +BASE_IMAGE_HASH=344d9f2 ## test +PLATFORM=linux/amd64 +# PLATFORM=linux/arm64 + HOST_UID=1000 DISTRO=alpine diff --git a/dist/alpine/Dockerfile b/dist/alpine/Dockerfile index 7f564f7..41270fa 100644 --- a/dist/alpine/Dockerfile +++ b/dist/alpine/Dockerfile @@ -11,8 +11,8 @@ RUN \ umask u+rwx,g+rwx,o-rwx && \ # https://gist.github.com/yuriburger/f13208717b1878fe835346b9871d0ead echo "graphviz fontconfig msttcorefonts-installer" | \ - sed 's|\s|\n|g' > /tmp/apk_requirements.txt && \ - /entrypoint.sh "anaconda" "/home/anaconda" echo done && \ + sed 's|\s|\n|g' > $APK_REQUIREMENTS && \ + /entrypoint.sh echo done && \ export PY_VER=$(python --version 2>&1 | grep -o "\d.\d") && \ if echo "$PY_VER" | grep -o "3\.9\|3\.8"; then \ pip install --platform=manylinux1_x86_64 --only-binary=:all: \ @@ -21,7 +21,7 @@ RUN \ fi && \ pip install --no-cache-dir datajoint && \ pip uninstall datajoint -y && \ - rm /tmp/apk_requirements.txt + rm $APK_REQUIREMENTS ENV APK_REQUIREMENTS /tmp/apk_requirements.txt ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt @@ -34,16 +34,15 @@ RUN \ chmod +x /entrypoint.sh && \ chmod 4755 /startup USER anaconda:anaconda -ENTRYPOINT ["/entrypoint.sh", "anaconda", "/home/anaconda"] +ENTRYPOINT ["/entrypoint.sh"] WORKDIR /main VOLUME /tmp/.X11-unix CMD ["sh"] -# #Squashed Final Image +# Squashed Final Image FROM scratch COPY --from=py_build / / -# RUN chmod 4755 /startup && /startup -user=anaconda LABEL maintainerName="Raphael Guzman" \ maintainerEmail="raphael@datajoint.com" \ maintainerCompany="DataJoint" @@ -55,7 +54,7 @@ ENV LANG C.UTF-8 ENV APK_REQUIREMENTS /tmp/apk_requirements.txt ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt ENV CONDA_REQUIREMENTS /tmp/conda_requirements.txt -ENTRYPOINT ["/entrypoint.sh", "anaconda", "/home/anaconda"] +ENTRYPOINT ["/entrypoint.sh"] WORKDIR /main VOLUME /tmp/.X11-unix CMD ["sh"] \ No newline at end of file diff --git a/dist/alpine/docker-compose.yaml b/dist/alpine/docker-compose.yaml index e0197c6..89c729a 100644 --- a/dist/alpine/docker-compose.yaml +++ b/dist/alpine/docker-compose.yaml @@ -1,4 +1,4 @@ -version: '2.4' +version: '3.7' services: app: build: # Uncomment to enable local build diff --git a/dist/debian/Dockerfile b/dist/debian/Dockerfile index c2c3c8c..92cfb54 100644 --- a/dist/debian/Dockerfile +++ b/dist/debian/Dockerfile @@ -9,8 +9,8 @@ LABEL maintainerName="Raphael Guzman" \ RUN \ umask u+rwx,g+rwx,o-rwx && \ - echo 'graphviz' > /tmp/apt_requirements.txt && \ - /entrypoint.sh "anaconda" "/home/anaconda" echo done && \ + echo 'graphviz' > $APT_REQUIREMENTS && \ + /entrypoint.sh echo done && \ export PY_VER=$(python --version 2>&1 | grep -o "\d.\d") && \ if echo "$PY_VER" | grep -oP "3\.9\|3\.8"; then \ pip install --platform=manylinux1_x86_64 --only-binary=:all: \ @@ -22,7 +22,7 @@ RUN \ # decorator package had a conflict so installing again to force resolve pip install datajoint --no-cache-dir && \ pip uninstall datajoint -y && \ - rm /tmp/apt_requirements.txt + rm $APT_REQUIREMENTS ENV APT_REQUIREMENTS /tmp/apt_requirements.txt ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt @@ -32,28 +32,26 @@ RUN \ chmod +x /entrypoint.sh && \ chmod 4755 /startup USER anaconda:anaconda -ENTRYPOINT ["/entrypoint.sh", "anaconda", "/home/anaconda"] +ENTRYPOINT ["/entrypoint.sh"] WORKDIR /main VOLUME /tmp/.X11-unix CMD ["bash"] -# #Squashed Final Image +# Squashed Final Image FROM scratch COPY --from=py_build / / -# RUN chmod 4755 /startup && /startup -user=anaconda LABEL maintainerName="Raphael Guzman" \ maintainerEmail="raphael@datajoint.com" \ maintainerCompany="DataJoint" USER anaconda:anaconda ENV PATH /opt/conda/bin:$PATH -RUN conda clean -ya ENV HOME /home/anaconda ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV APT_REQUIREMENTS /tmp/apt_requirements.txt ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt ENV CONDA_REQUIREMENTS /tmp/conda_requirements.txt -ENTRYPOINT ["/entrypoint.sh", "anaconda", "/home/anaconda"] +ENTRYPOINT ["/entrypoint.sh"] WORKDIR /main VOLUME /tmp/.X11-unix CMD ["bash"] \ No newline at end of file diff --git a/dist/debian/docker-compose.yaml b/dist/debian/docker-compose.yaml index 101960d..af8128d 100644 --- a/dist/debian/docker-compose.yaml +++ b/dist/debian/docker-compose.yaml @@ -1,4 +1,4 @@ -version: '2.4' +version: '3.7' services: app: build: # Uncomment to enable local build diff --git a/docs/index.rst b/docs/index.rst index 9cf21db..f60d6d0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,7 +38,7 @@ To rebuild and run tests locally, execute the following statements: set -a # automatically export sourced variables . config/.env # source config for build and tests - docker-compose -f dist/${DISTRO}/docker-compose.yaml build # build image + docker buildx bake -f dist/${DISTRO}/docker-compose.yaml --set *.platform=${PLATFORM} --set *.context=. --load # build image tests/main.sh # run tests set +a # disable auto-export behavior for sourced variables diff --git a/tests/main.sh b/tests/main.sh index 1445fc3..2f4a768 100755 --- a/tests/main.sh +++ b/tests/main.sh @@ -11,7 +11,7 @@ assert () lineno=$3 if ! eval "$2"; then - echo "Assertion failed: \"$2\"" + echo "Assertion ($1) failed: \"$2\"" echo "File \"$0\", line $lineno" exit $E_ASSERT_FAILED else @@ -38,27 +38,28 @@ validate () { )" $LINENO } # set image context -REF=$(eval "echo $(cat dist/${DISTRO}/docker-compose.yaml | grep 'image:' | awk '{print $2}')") +REF=$(eval "echo $(cat dist/${DISTRO}/docker-compose.yaml | grep 'image:' | \ + awk '{print $2}')") TAG=$(echo $REF | awk -F':' '{print $2}') IMAGE=$(echo $REF | awk -F':' '{print $1}') SHELL_CMD_TEMPLATE="docker run --rm -i \$SHELL_CMD_FLAGS $REF \ $(docker inspect "$REF" --format '{{join .Config.Cmd " "}}') -c" # determine reference size -if [ $DISTRO == alpine ] && [ $PY_VER == '3.9' ]; then +if [ $DISTRO == alpine ] && [ $PY_VER == '3.9' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=484 -elif [ $DISTRO == alpine ] && [ $PY_VER == '3.8' ]; then +elif [ $DISTRO == alpine ] && [ $PY_VER == '3.8' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=442 # 599 -elif [ $DISTRO == alpine ] && [ $PY_VER == '3.7' ]; then +elif [ $DISTRO == alpine ] && [ $PY_VER == '3.7' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=452 # 629 -elif [ $DISTRO == alpine ] && [ $PY_VER == '3.6' ]; then +elif [ $DISTRO == alpine ] && [ $PY_VER == '3.6' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=385 # 689 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.9' ]; then +elif [ $DISTRO == debian ] && [ $PY_VER == '3.9' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=604 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.8' ]; then +elif [ $DISTRO == debian ] && [ $PY_VER == '3.8' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=558 # 833 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.7' ]; then +elif [ $DISTRO == debian ] && [ $PY_VER == '3.7' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=563 # 863 -elif [ $DISTRO == debian ] && [ $PY_VER == '3.6' ]; then +elif [ $DISTRO == debian ] && [ $PY_VER == '3.6' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=496 # 923 fi SIZE_LIMIT=$(echo "scale=4; $SIZE_LIMIT * 1.03" | bc) From b6c271450ef3ed27918e0d6aaa99397447ea0721 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 20 Sep 2021 03:30:58 -0500 Subject: [PATCH 2/3] Check sizes of the failed images. --- tests/main.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/main.sh b/tests/main.sh index 2f4a768..1dff88d 100755 --- a/tests/main.sh +++ b/tests/main.sh @@ -65,6 +65,7 @@ fi SIZE_LIMIT=$(echo "scale=4; $SIZE_LIMIT * 1.03" | bc) # verify size minimal SIZE=$(docker images --filter "reference=$REF" --format "{{.Size}}" | awk -F'MB' '{print $1}') +echo $SIZE assert "minimal footprint" "(( $(echo "$SIZE <= $SIZE_LIMIT" | bc -l) ))" $LINENO # run tests SHELL_CMD=$(eval "echo \"$SHELL_CMD_TEMPLATE\"") From 23711f6582f67ede2aed03612567e06205113de3 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 20 Sep 2021 03:37:32 -0500 Subject: [PATCH 3/3] Update size limit. --- tests/main.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/main.sh b/tests/main.sh index 1dff88d..c1d400c 100755 --- a/tests/main.sh +++ b/tests/main.sh @@ -62,10 +62,9 @@ elif [ $DISTRO == debian ] && [ $PY_VER == '3.7' ] && [ $PLATFORM == 'linux/amd6 elif [ $DISTRO == debian ] && [ $PY_VER == '3.6' ] && [ $PLATFORM == 'linux/amd64' ]; then SIZE_LIMIT=496 # 923 fi -SIZE_LIMIT=$(echo "scale=4; $SIZE_LIMIT * 1.03" | bc) +SIZE_LIMIT=$(echo "scale=4; $SIZE_LIMIT * 1.06" | bc) # verify size minimal SIZE=$(docker images --filter "reference=$REF" --format "{{.Size}}" | awk -F'MB' '{print $1}') -echo $SIZE assert "minimal footprint" "(( $(echo "$SIZE <= $SIZE_LIMIT" | bc -l) ))" $LINENO # run tests SHELL_CMD=$(eval "echo \"$SHELL_CMD_TEMPLATE\"")