From 0208cf2e0ece42e6445aa27917f446aa1935bd60 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 4 Dec 2024 11:05:22 +0100 Subject: [PATCH 01/13] Add CI action for uploading Docker image for wheels --- .github/workflows/wheel_image.yml | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/wheel_image.yml diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml new file mode 100644 index 0000000000..a5403918cb --- /dev/null +++ b/.github/workflows/wheel_image.yml @@ -0,0 +1,46 @@ +name: Build custom Docker image for manylinux wheels + +on: + workflow_dispatch: + inputs: + base_image: + description: 'The base Docker image to use' + default: 'manylinux2014' + type: choice + options: + - 'manylinux2014' + - 'manylinux_2_28' + - 'manylinux_2_34' + +jobs: + docker: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - platform: 'linux/amd64' + tag: 'x86_64' + - platform: 'linux/arm64' + tag: 'aarch64' + + steps: + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + platforms: ${{ matrix.platform }} + push: true + tags: neuronsimulator/neuron_wheel:latest + build-args: | + MANYLINUX_IMAGE: ${{ github.event.inputs.base_image }}_${{ matrix.tag }} From 0b6ceb535f6830998389f86e6738bf6c0aebe14e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 4 Dec 2024 11:14:26 +0100 Subject: [PATCH 02/13] Minor safety improvements --- .github/workflows/wheel_image.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index a5403918cb..3ca5fa89c4 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -12,6 +12,12 @@ on: - 'manylinux_2_28' - 'manylinux_2_34' + upload: + description: 'Whether to upload (push) the image to DockerHub' + default: 'false' + required: true + type: 'boolean' + jobs: docker: runs-on: ubuntu-latest @@ -19,9 +25,9 @@ jobs: matrix: include: - platform: 'linux/amd64' - tag: 'x86_64' + arch: 'x86_64' - platform: 'linux/arm64' - tag: 'aarch64' + arch: 'aarch64' steps: - name: Login to Docker Hub @@ -40,7 +46,7 @@ jobs: uses: docker/build-push-action@v6 with: platforms: ${{ matrix.platform }} - push: true + push: ${{ github.event.inputs.upload }} tags: neuronsimulator/neuron_wheel:latest build-args: | - MANYLINUX_IMAGE: ${{ github.event.inputs.base_image }}_${{ matrix.tag }} + MANYLINUX_IMAGE: ${{ github.event.inputs.base_image }}_${{ matrix.arch }} From 0e6433da6c30ebb68cc3c2bff2459b1c733b6c8a Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 4 Dec 2024 11:47:46 +0100 Subject: [PATCH 03/13] Make image tag customizable --- .github/workflows/wheel_image.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index 3ca5fa89c4..8a12726145 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -18,6 +18,11 @@ on: required: true type: 'boolean' + tag: + description: 'The tag for the final Docker image' + default: 'latest' + required: true + jobs: docker: runs-on: ubuntu-latest @@ -47,6 +52,6 @@ jobs: with: platforms: ${{ matrix.platform }} push: ${{ github.event.inputs.upload }} - tags: neuronsimulator/neuron_wheel:latest + tags: neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }} build-args: | MANYLINUX_IMAGE: ${{ github.event.inputs.base_image }}_${{ matrix.arch }} From 1904ec2636d6f2acc21029a46887ec6d1ffbebdc Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 4 Dec 2024 13:48:06 +0100 Subject: [PATCH 04/13] Use arch in tag because BuildKit is complicated --- .github/workflows/wheel_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index 8a12726145..d826830c71 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -52,6 +52,6 @@ jobs: with: platforms: ${{ matrix.platform }} push: ${{ github.event.inputs.upload }} - tags: neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }} + tags: neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }}-${{ matrix.arch }} build-args: | MANYLINUX_IMAGE: ${{ github.event.inputs.base_image }}_${{ matrix.arch }} From dcc62f8245b885eda9100da4dee0fd37d390046a Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 9 Dec 2024 14:20:35 +0100 Subject: [PATCH 05/13] Allow uploading to GHCR --- .github/workflows/wheel_image.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index d826830c71..0644392164 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -13,11 +13,19 @@ on: - 'manylinux_2_34' upload: - description: 'Whether to upload (push) the image to DockerHub' + description: 'Whether to upload (push) the image to the container registry' default: 'false' required: true type: 'boolean' + container_registry: + description: 'The name of the container registry to upload the image to (only useful if used with upload=true)' + default: 'ghcr.io' + required: true + options: + - 'ghcr.io' + - 'docker.io' + tag: description: 'The tag for the final Docker image' default: 'latest' @@ -36,11 +44,20 @@ jobs: steps: - name: Login to Docker Hub + if: ${{ github.event.inputs.upload }} == 'true' && ${{ github.event.inputs.container_registry }} == 'docker.io' uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub container registry + if: ${{ github.event.inputs.upload }} == 'true' && ${{ github.event.inputs.container_registry }} == 'ghcr.io' + uses: docker/login-action@v3 + with: + registry: 'ghcr.io' + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -52,6 +69,6 @@ jobs: with: platforms: ${{ matrix.platform }} push: ${{ github.event.inputs.upload }} - tags: neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }}-${{ matrix.arch }} + tags: ${{ github.event.inputs.container_registry }}/neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }}-${{ matrix.arch }} build-args: | MANYLINUX_IMAGE: ${{ github.event.inputs.base_image }}_${{ matrix.arch }} From 20fa088a4246a5ce9a2fed7d3fe0b792c554c977 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 9 Dec 2024 15:21:14 +0100 Subject: [PATCH 06/13] Trigger CI for first time --- .github/workflows/wheel_image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index 0644392164..c024477f00 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -1,6 +1,7 @@ name: Build custom Docker image for manylinux wheels on: + push: workflow_dispatch: inputs: base_image: From 005313d84f644b52e0de98f48849802d85508988 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 9 Dec 2024 15:21:53 +0100 Subject: [PATCH 07/13] Revert "Trigger CI for first time" This reverts commit 20fa088a4246a5ce9a2fed7d3fe0b792c554c977. --- .github/workflows/wheel_image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index c024477f00..0644392164 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -1,7 +1,6 @@ name: Build custom Docker image for manylinux wheels on: - push: workflow_dispatch: inputs: base_image: From cc5328653e8eb87c2c6ea14c37f4fcbfc537c070 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 9 Dec 2024 15:24:42 +0100 Subject: [PATCH 08/13] Fixes --- .github/workflows/wheel_image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index 0644392164..063d028e91 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -44,14 +44,14 @@ jobs: steps: - name: Login to Docker Hub - if: ${{ github.event.inputs.upload }} == 'true' && ${{ github.event.inputs.container_registry }} == 'docker.io' + if: github.event.inputs.upload == 'true' && github.event.inputs.container_registry == 'docker.io' uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub container registry - if: ${{ github.event.inputs.upload }} == 'true' && ${{ github.event.inputs.container_registry }} == 'ghcr.io' + if: github.event.inputs.upload == 'true' && github.event.inputs.container_registry == 'ghcr.io' uses: docker/login-action@v3 with: registry: 'ghcr.io' From 1b441dfeffbb1f4b1174188e95486dcb9eb438ba Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 9 Dec 2024 15:33:12 +0100 Subject: [PATCH 09/13] Fix path --- .github/workflows/wheel_image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index 063d028e91..e4f7faee10 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -33,6 +33,7 @@ on: jobs: docker: + name: 'Create Docker image for manylinux wheels' runs-on: ubuntu-latest strategy: matrix: @@ -67,6 +68,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v6 with: + file: packaging/python/Dockerfile platforms: ${{ matrix.platform }} push: ${{ github.event.inputs.upload }} tags: ${{ github.event.inputs.container_registry }}/neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }}-${{ matrix.arch }} From 0e3c6a727f0b1a3ce7bf11e2fbdbbfc3768ef41e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 9 Dec 2024 15:49:17 +0100 Subject: [PATCH 10/13] Do not copy dockerfile --- packaging/python/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/packaging/python/Dockerfile b/packaging/python/Dockerfile index 36c7d0ad4f..07bb55d3fb 100644 --- a/packaging/python/Dockerfile +++ b/packaging/python/Dockerfile @@ -87,9 +87,6 @@ ENV PATH /nrnwheel/openmpi/bin:$PATH RUN yum -y install epel-release libX11-devel libXcomposite-devel vim-enhanced && yum -y clean all && rm -rf /var/cache RUN yum -y remove ncurses-devel -# Copy Dockerfile for reference -COPY Dockerfile . - # build wheels from there WORKDIR /root From de6cb905757fcf23f8779a57b84ebb07c9e665a1 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 9 Dec 2024 16:22:53 +0100 Subject: [PATCH 11/13] Add context dir --- .github/workflows/wheel_image.yml | 4 ++-- packaging/python/Dockerfile | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index e4f7faee10..b53b884b63 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -68,9 +68,9 @@ jobs: - name: Build and push uses: docker/build-push-action@v6 with: - file: packaging/python/Dockerfile + context: "{{defaultContext}}:packaging/python" platforms: ${{ matrix.platform }} push: ${{ github.event.inputs.upload }} tags: ${{ github.event.inputs.container_registry }}/neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }}-${{ matrix.arch }} build-args: | - MANYLINUX_IMAGE: ${{ github.event.inputs.base_image }}_${{ matrix.arch }} + MANYLINUX_IMAGE=${{ github.event.inputs.base_image }}_${{ matrix.arch }} diff --git a/packaging/python/Dockerfile b/packaging/python/Dockerfile index 07bb55d3fb..36c7d0ad4f 100644 --- a/packaging/python/Dockerfile +++ b/packaging/python/Dockerfile @@ -87,6 +87,9 @@ ENV PATH /nrnwheel/openmpi/bin:$PATH RUN yum -y install epel-release libX11-devel libXcomposite-devel vim-enhanced && yum -y clean all && rm -rf /var/cache RUN yum -y remove ncurses-devel +# Copy Dockerfile for reference +COPY Dockerfile . + # build wheels from there WORKDIR /root From c5429d6e5b01119e0bf0e63f3100dd50704785b4 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 10 Dec 2024 09:26:08 +0100 Subject: [PATCH 12/13] Update Dockerfile Remove Python 3.13t (the free-threaded build) as we do not support it yet. --- packaging/python/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/python/Dockerfile b/packaging/python/Dockerfile index 36c7d0ad4f..95931335cc 100644 --- a/packaging/python/Dockerfile +++ b/packaging/python/Dockerfile @@ -93,4 +93,7 @@ COPY Dockerfile . # build wheels from there WORKDIR /root +# remove Python 3.13t since we do not support the free-threaded build yet +RUN rm -fr /opt/python/cp313-cp313t + ENV NMODL_PYLIB=/nrnwheel/python/lib/libpython3.10.so.1.0 From 7d5e83a75f709ddc16bb1b278e10bf61b85f892e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 20 Jan 2025 15:17:51 +0100 Subject: [PATCH 13/13] Try out ARM runners See https://github.blog/changelog/2025-01-16-linux-arm64-hosted-runners-now-available-for-free-in-public-repositories-public-preview/ --- .github/workflows/wheel_image.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/wheel_image.yml b/.github/workflows/wheel_image.yml index b53b884b63..3d2a097dd2 100644 --- a/.github/workflows/wheel_image.yml +++ b/.github/workflows/wheel_image.yml @@ -28,20 +28,22 @@ on: tag: description: 'The tag for the final Docker image' - default: 'latest' + default: 'latest-' required: true jobs: docker: name: 'Create Docker image for manylinux wheels' - runs-on: ubuntu-latest + runs-on: ${{ matrix.base_image }} strategy: matrix: include: - platform: 'linux/amd64' arch: 'x86_64' + base_image: 'ubuntu-22.04' - platform: 'linux/arm64' arch: 'aarch64' + base_image: 'ubuntu-22.04-arm' steps: - name: Login to Docker Hub @@ -59,18 +61,12 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build and push uses: docker/build-push-action@v6 with: context: "{{defaultContext}}:packaging/python" platforms: ${{ matrix.platform }} push: ${{ github.event.inputs.upload }} - tags: ${{ github.event.inputs.container_registry }}/neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }}-${{ matrix.arch }} + tags: ${{ github.event.inputs.container_registry }}/neuronsimulator/neuron_wheel:${{ github.event.inputs.tag }}${{ github.event.inputs.base_image }}_${{ matrix.arch }} build-args: | MANYLINUX_IMAGE=${{ github.event.inputs.base_image }}_${{ matrix.arch }}