Skip to content

Commit

Permalink
Merge pull request #26 from guzman-raphael/upgrade
Browse files Browse the repository at this point in the history
Upgrade image stack with new features and add placeholder for ARM builds
  • Loading branch information
guzman-raphael authored Sep 20, 2021
2 parents 534f4bc + 23711f6 commit 5054080
Show file tree
Hide file tree
Showing 8 changed files with 118 additions and 40 deletions.
97 changes: 87 additions & 10 deletions .github/workflows/development.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,48 +11,78 @@ jobs:
- distro: alpine
conda_ver: 4.10.3
py_ver: 3.9
platform: linux/amd64
- distro: alpine
conda_ver: 4.10.3
py_ver: 3.8
platform: linux/amd64
- distro: alpine
conda_ver: 4.10.3
py_ver: 3.7
platform: linux/amd64
- distro: alpine
conda_ver: 4.5.4
py_ver: 3.6
platform: linux/amd64
- distro: debian
conda_ver: 4.10.3
py_ver: 3.9
platform: linux/amd64
- distro: debian
conda_ver: 4.10.3
py_ver: 3.8
platform: linux/amd64
- distro: debian
conda_ver: 4.10.3
py_ver: 3.7
platform: linux/amd64
- distro: debian
conda_ver: 4.5.4
py_ver: 3.6
platform: linux/amd64
# - distro: debian
# conda_ver: 4.10.3
# py_ver: 3.9
# platform: linux/arm64
# - distro: debian
# conda_ver: 4.10.3
# py_ver: 3.8
# platform: linux/arm64
# - distro: debian
# conda_ver: 4.10.3
# py_ver: 3.7
# platform: linux/arm64
env:
DISTRO: ${{matrix.distro}}
CONDA_VER: ${{matrix.conda_ver}}
PY_VER: ${{matrix.py_ver}}
PLATFORM: ${{matrix.platform}}
DOCKER_CLIENT_TIMEOUT: "120"
COMPOSE_HTTP_TIMEOUT: "120"
BASE_IMAGE_HASH: "8281c8f"
BASE_IMAGE_HASH: "344d9f2"
steps:
- uses: actions/checkout@v2
- name: Determine platform tag
run: |
PLATFORM_TAG=$(echo "$PLATFORM" | tr '/' '_')
echo "PLATFORM_TAG=${PLATFORM_TAG}" >> $GITHUB_ENV
- uses: dbhi/qus/action@main
with:
targets: aarch64
- name: Compile image
run: |
docker-compose -f dist/${DISTRO}/docker-compose.yaml build
docker buildx bake -f dist/${DISTRO}/docker-compose.yaml \
--set *.platform=${PLATFORM} --set *.context=. --load
REF=$(eval "echo $(cat dist/${DISTRO}/docker-compose.yaml | grep 'image:' | \
awk '{print $2}')")
docker save "${REF}" | gzip > "py${PY_VER}-${DISTRO}.tar.gz"
docker save "${REF}" | gzip > "py${PY_VER}-${DISTRO}-${PLATFORM_TAG}.tar.gz"
- name: Add image artifact
uses: actions/upload-artifact@v2
with:
name: ${{format('py{0}-{1}', matrix.py_ver, matrix.distro)}}
name: ${{format('py{0}-{1}-{2}', matrix.py_ver, matrix.distro, env.PLATFORM_TAG)}}
path: >
${{format('py{0}-{1}.tar.gz', matrix.py_ver, matrix.distro)}}
${{format('py{0}-{1}-{2}.tar.gz', matrix.py_ver, matrix.distro,
env.PLATFORM_TAG)}}
retention-days: 1
test:
needs: build
Expand All @@ -62,32 +92,57 @@ jobs:
include:
- distro: alpine
py_ver: 3.9
platform: linux/amd64
- distro: alpine
py_ver: 3.8
platform: linux/amd64
- distro: alpine
py_ver: 3.7
platform: linux/amd64
- distro: alpine
py_ver: 3.6
platform: linux/amd64
- distro: debian
py_ver: 3.9
platform: linux/amd64
- distro: debian
py_ver: 3.8
platform: linux/amd64
- distro: debian
py_ver: 3.7
platform: linux/amd64
- distro: debian
py_ver: 3.6
platform: linux/amd64
# - distro: debian
# py_ver: 3.9
# platform: linux/arm64
# - distro: debian
# py_ver: 3.8
# platform: linux/arm64
# - distro: debian
# py_ver: 3.7
# platform: linux/arm64
env:
DISTRO: ${{matrix.distro}}
PY_VER: ${{matrix.py_ver}}
PLATFORM: ${{matrix.platform}}
steps:
- uses: actions/checkout@v2
- name: Determine platform tag
run: |
PLATFORM_TAG=$(echo "$PLATFORM" | tr '/' '_')
echo "PLATFORM_TAG=${PLATFORM_TAG}" >> $GITHUB_ENV
- name: Fetch image artifact
uses: actions/download-artifact@v2
with:
name: ${{format('py{0}-{1}', matrix.py_ver, matrix.distro)}}
name: ${{format('py{0}-{1}-{2}', matrix.py_ver, matrix.distro, env.PLATFORM_TAG)}}
- uses: dbhi/qus/action@main
with:
targets: aarch64
- name: Run integration tests
run: |
docker load < "py${PY_VER}-${DISTRO}.tar.gz"
docker load < "py${PY_VER}-${DISTRO}-${PLATFORM_TAG}.tar.gz"
tests/main.sh
publish:
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
Expand All @@ -98,35 +153,57 @@ jobs:
include:
- distro: alpine
py_ver: 3.9
platform: linux/amd64
- distro: alpine
py_ver: 3.8
platform: linux/amd64
- distro: alpine
py_ver: 3.7
platform: linux/amd64
- distro: alpine
py_ver: 3.6
platform: linux/amd64
- distro: debian
py_ver: 3.9
platform: linux/amd64
- distro: debian
py_ver: 3.8
platform: linux/amd64
- distro: debian
py_ver: 3.7
platform: linux/amd64
- distro: debian
py_ver: 3.6
platform: linux/amd64
# - distro: debian
# py_ver: 3.9
# platform: linux/arm64
# - distro: debian
# py_ver: 3.8
# platform: linux/arm64
# - distro: debian
# py_ver: 3.7
# platform: linux/arm64
env:
DISTRO: ${{matrix.distro}}
PY_VER: ${{matrix.py_ver}}
PLATFORM: ${{matrix.platform}}
DOCKER_USERNAME: ${{secrets.docker_username}}
DOCKER_PASSWORD: ${{secrets.docker_password}}
steps:
- uses: actions/checkout@v2
- name: Determine platform tag
run: |
PLATFORM_TAG=$(echo "$PLATFORM" | tr '/' '_')
echo "PLATFORM_TAG=${PLATFORM_TAG}" >> $GITHUB_ENV
- name: Fetch image artifact
uses: actions/download-artifact@v2
with:
name: ${{format('py{0}-{1}', matrix.py_ver, matrix.distro)}}
name: ${{format('py{0}-{1}-{2}', matrix.py_ver, matrix.distro, env.PLATFORM_TAG)}}
- name: Publish image
run: |
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
docker load < "py${PY_VER}-${DISTRO}.tar.gz"
docker load < "py${PY_VER}-${DISTRO}-${PLATFORM_TAG}.tar.gz"
REF=$(eval "echo $(cat dist/${DISTRO}/docker-compose.yaml | grep 'image:' | \
awk '{print $2}')")
TAG=$(echo $REF | awk -F':' '{print $2}')
Expand All @@ -142,7 +219,7 @@ jobs:
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/checkout@v2
- name: Build docs
uses: ammaraskar/sphinx-action@master
with:
Expand Down
5 changes: 4 additions & 1 deletion config/.env
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,16 @@ CONDA_VER=4.10.3
# PY_VER=3.6
# CONDA_VER=4.5.4

BASE_IMAGE_HASH=8281c8f
BASE_IMAGE_HASH=344d9f2




## test

PLATFORM=linux/amd64
# PLATFORM=linux/arm64

HOST_UID=1000

DISTRO=alpine
Expand Down
13 changes: 6 additions & 7 deletions dist/alpine/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ RUN \
umask u+rwx,g+rwx,o-rwx && \
# https://gist.github.com/yuriburger/f13208717b1878fe835346b9871d0ead
echo "graphviz fontconfig msttcorefonts-installer" | \
sed 's|\s|\n|g' > /tmp/apk_requirements.txt && \
/entrypoint.sh "anaconda" "/home/anaconda" echo done && \
sed 's|\s|\n|g' > $APK_REQUIREMENTS && \
/entrypoint.sh echo done && \
export PY_VER=$(python --version 2>&1 | grep -o "\d.\d") && \
if echo "$PY_VER" | grep -o "3\.9\|3\.8"; then \
pip install --platform=manylinux1_x86_64 --only-binary=:all: \
Expand All @@ -21,7 +21,7 @@ RUN \
fi && \
pip install --no-cache-dir datajoint && \
pip uninstall datajoint -y && \
rm /tmp/apk_requirements.txt
rm $APK_REQUIREMENTS

ENV APK_REQUIREMENTS /tmp/apk_requirements.txt
ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt
Expand All @@ -34,16 +34,15 @@ RUN \
chmod +x /entrypoint.sh && \
chmod 4755 /startup
USER anaconda:anaconda
ENTRYPOINT ["/entrypoint.sh", "anaconda", "/home/anaconda"]
ENTRYPOINT ["/entrypoint.sh"]
WORKDIR /main
VOLUME /tmp/.X11-unix
CMD ["sh"]


# #Squashed Final Image
# Squashed Final Image
FROM scratch
COPY --from=py_build / /
# RUN chmod 4755 /startup && /startup -user=anaconda
LABEL maintainerName="Raphael Guzman" \
maintainerEmail="[email protected]" \
maintainerCompany="DataJoint"
Expand All @@ -55,7 +54,7 @@ ENV LANG C.UTF-8
ENV APK_REQUIREMENTS /tmp/apk_requirements.txt
ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt
ENV CONDA_REQUIREMENTS /tmp/conda_requirements.txt
ENTRYPOINT ["/entrypoint.sh", "anaconda", "/home/anaconda"]
ENTRYPOINT ["/entrypoint.sh"]
WORKDIR /main
VOLUME /tmp/.X11-unix
CMD ["sh"]
2 changes: 1 addition & 1 deletion dist/alpine/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version: '2.4'
version: '3.7'
services:
app:
build: # Uncomment to enable local build
Expand Down
14 changes: 6 additions & 8 deletions dist/debian/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ LABEL maintainerName="Raphael Guzman" \

RUN \
umask u+rwx,g+rwx,o-rwx && \
echo 'graphviz' > /tmp/apt_requirements.txt && \
/entrypoint.sh "anaconda" "/home/anaconda" echo done && \
echo 'graphviz' > $APT_REQUIREMENTS && \
/entrypoint.sh echo done && \
export PY_VER=$(python --version 2>&1 | grep -o "\d.\d") && \
if echo "$PY_VER" | grep -oP "3\.9\|3\.8"; then \
pip install --platform=manylinux1_x86_64 --only-binary=:all: \
Expand All @@ -22,7 +22,7 @@ RUN \
# decorator package had a conflict so installing again to force resolve
pip install datajoint --no-cache-dir && \
pip uninstall datajoint -y && \
rm /tmp/apt_requirements.txt
rm $APT_REQUIREMENTS

ENV APT_REQUIREMENTS /tmp/apt_requirements.txt
ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt
Expand All @@ -32,28 +32,26 @@ RUN \
chmod +x /entrypoint.sh && \
chmod 4755 /startup
USER anaconda:anaconda
ENTRYPOINT ["/entrypoint.sh", "anaconda", "/home/anaconda"]
ENTRYPOINT ["/entrypoint.sh"]
WORKDIR /main
VOLUME /tmp/.X11-unix
CMD ["bash"]


# #Squashed Final Image
# Squashed Final Image
FROM scratch
COPY --from=py_build / /
# RUN chmod 4755 /startup && /startup -user=anaconda
LABEL maintainerName="Raphael Guzman" \
maintainerEmail="[email protected]" \
maintainerCompany="DataJoint"
USER anaconda:anaconda
ENV PATH /opt/conda/bin:$PATH
RUN conda clean -ya
ENV HOME /home/anaconda
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ENV APT_REQUIREMENTS /tmp/apt_requirements.txt
ENV PIP_REQUIREMENTS /tmp/pip_requirements.txt
ENV CONDA_REQUIREMENTS /tmp/conda_requirements.txt
ENTRYPOINT ["/entrypoint.sh", "anaconda", "/home/anaconda"]
ENTRYPOINT ["/entrypoint.sh"]
WORKDIR /main
VOLUME /tmp/.X11-unix
CMD ["bash"]
2 changes: 1 addition & 1 deletion dist/debian/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version: '2.4'
version: '3.7'
services:
app:
build: # Uncomment to enable local build
Expand Down
2 changes: 1 addition & 1 deletion docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ To rebuild and run tests locally, execute the following statements:
set -a # automatically export sourced variables
. config/.env # source config for build and tests
docker-compose -f dist/${DISTRO}/docker-compose.yaml build # build image
docker buildx bake -f dist/${DISTRO}/docker-compose.yaml --set *.platform=${PLATFORM} --set *.context=. --load # build image
tests/main.sh # run tests
set +a # disable auto-export behavior for sourced variables
Expand Down
Loading

0 comments on commit 5054080

Please sign in to comment.