U24 element reusable CICD (#9)

* feat(u24-element): ✨ Add reusable CI for u24 elements * fix(u24-element): 🐛 remove registry creds * fix(u24-element): 🐛 fix CI name * fix(u24-element): 🐛 rm registry creds from inputs * fix(u24-element): 🐛 fix publish if block * test(u24-element): 🧪 * fix(u24-element): 🐛 fix test changelog action * fix(u24-element): 🐛 fix changelog action * fix(u24-element): 🐛 skip test-changelog for now * fix(u24-element): 🐛 skip changelog dependency * fix(u24-element): 🐛 parse version by double quote * Revert "fix(u24-element): 🐛 parse version by double quote" This reverts commit dfdfc27. * fix(u24-element): 🐛 version double quote * fix(u24-element): 🐛 version double quote * fix(u24-element): 🐛 version double quote * feat(u24-element): ✨ common build Dockerfile and docker-compose-build.yaml * fix(u24-element): 🐛 build path * fix(u24-element): 🐛 include dependency: git * fix(u24-element): 🐛 Dockerfile permission issue * fix(u24-element): 🐛 correct alpine apk * fix(u24-element): 🐛 debian build * feat(u24-element): ✨ enable test-changelog * fix(u24-element): 🐛 artifact path * fix(u24-element): 🐛 apply changes * feat(u24-element): 🚀 * fix: 🐛 update PR suggestions
dj-sciops · Jul 1, 2022 · bf8986d · bf8986d
1 parent 2b15a70
commit bf8986d
Show file tree

Hide file tree

Showing 9 changed files with 444 additions and 3 deletions.
diff --git a/.github/.staging_workflows/anchored_sciops_docker_images.yaml b/.github/.staging_workflows/anchored_sciops_docker_images.yaml
@@ -101,7 +101,7 @@ jobs:
         name: Build codebook env image
         run: |
           ## Get docker template
-          cp -r ./cicd/docker-template/* ./main/docker/
+          cp -r ./cicd/docker-template/codebook_env ./main/docker/
 
           ## Get build info
           cd ./main/docker/codebook_env

diff --git a/.github/.staging_workflows/anchored_u24_element_build_release.yaml b/.github/.staging_workflows/anchored_u24_element_build_release.yaml
@@ -0,0 +1,184 @@
+name: u24_elements
+
+on:
+  workflow_call:
+    inputs:
+      py_ver:
+        required: true
+        type: string
+      dist:
+        required: true
+        type: string
+      image:
+        required: true
+        type: string
+
+anchor-dev-push: &dev-push
+  if: |
+    github.event_name == 'push' &&
+    startsWith(github.ref, 'refs/tags') &&
+    github.repository_owner == 'yambottle'
+anchor-dev-cicd-repo: &dev-cicd-repo
+  repository: 'yambottle/djsciops-cicd'
+anchor-test-push: &test-push
+  if: |
+    github.event_name == 'push' &&
+    startsWith(github.ref, 'refs/tags') &&
+    github.repository_owner == 'yambottle'
+anchor-test-cicd-repo: &test-cicd-repo
+  repository: 'yambottle/djsciops-cicd'
+anchor-prod-push: &prod-push
+  if: |
+    github.event_name == 'push' &&
+    startsWith(github.ref, 'refs/tags') &&
+    github.repository_owner == 'dj-sciops'
+anchor-prod-cicd-repo: &prod-cicd-repo
+  repository: 'dj-sciops/djsciops-cicd'
+
+jobs:
+  test-changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get changelog entry
+        id: changelog_reader
+        uses: guzman-raphael/changelog-reader-action@v5
+        with:
+          path: ./CHANGELOG.md
+      - name: Verify changelog parsing
+        env:
+          TAG_NAME: ${{steps.changelog_reader.outputs.version}}
+          RELEASE_NAME: Release ${{steps.changelog_reader.outputs.version}}
+          BODY: ${{steps.changelog_reader.outputs.changes}}
+          PRERELEASE: ${{steps.changelog_reader.outputs.status == 'prereleased'}}
+          DRAFT: ${{steps.changelog_reader.outputs.status == 'unreleased'}}
+        run: |
+          echo "TAG_NAME=${TAG_NAME}"
+          echo "RELEASE_NAME=${RELEASE_NAME}"
+          echo "BODY=${BODY}"
+          echo "PRERELEASE=${PRERELEASE}"
+          echo "DRAFT=${DRAFT}"
+  build:
+    needs: test-changelog
+    runs-on: ubuntu-latest
+    env:
+      PY_VER: ${{inputs.py_ver}}
+      DIST: ${{inputs.dist}}
+      IMAGE: ${{inputs.image}}
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
+    steps:
+      - name: Checkout element repo
+        uses: actions/checkout@v2
+        with:
+          path: 'main'
+      - name: Checkout cicd repo
+        uses: actions/checkout@v2
+        with:
+          !!merge <<: *$STAGE-cicd-repo
+          path: 'cicd'
+      - name: Compile image
+        run: |
+          ## Get docker template
+          cp -r ./cicd/docker-template/elements/dist/${DIST}/* ./main/
+          cd ./main
+
+          export PKG_NAME=$(python3 -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])")
+          export PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+')
+          export HOST_UID=$(id -u)
+          docker-compose -f docker-compose-build.yaml up --exit-code-from element --build
+          IMAGE=$(docker images --filter "reference=datajoint/${PKG_NAME}*" \
+            --format "{{.Repository}}")
+          TAG=$(docker images --filter "reference=datajoint/${PKG_NAME}*" --format "{{.Tag}}")
+          docker save "${IMAGE}:${TAG}" | \
+            gzip > "image-${PKG_NAME}-${PKG_VERSION}-py${PY_VER}-${DIST}.tar.gz"
+          echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_ENV
+          echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_ENV
+      - name: Add image artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{env.PY_VER}}-${{env.DIST}}
+          path: "./main/image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{env.PY_VER}}-\
+            ${{env.DIST}}.tar.gz"
+          retention-days: 1
+      - if: env.PY_VER == '3.9' && env.DIST == 'alpine'
+        name: Add pip artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}}
+          path: ./main/dist
+          retention-days: 1
+  publish-release:
+    !!merge <<: *$STAGE-push
+    needs: build
+    runs-on: ubuntu-latest
+    env:
+      TWINE_USERNAME: ${{secrets.twine_username}}
+      TWINE_PASSWORD: ${{secrets.twine_password}}
+    outputs:
+      release_upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Determine package version
+        run: |
+          PKG_NAME=$(python3 -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])")
+          SDIST_PKG_NAME=$(echo ${PKG_NAME} | sed 's|_|-|g')
+          PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+')
+          echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_ENV
+          echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_ENV
+          echo "SDIST_PKG_NAME=${SDIST_PKG_NAME}" >> $GITHUB_ENV
+      - name: Get changelog entry
+        id: changelog_reader
+        uses: guzman-raphael/changelog-reader-action@v5
+        with:
+          path: ./CHANGELOG.md
+          version: ${{env.PKG_VERSION}}
+      - name: Create GH release
+        id: create_gh_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          tag_name: ${{steps.changelog_reader.outputs.version}}
+          release_name: Release ${{steps.changelog_reader.outputs.version}}
+          body: ${{steps.changelog_reader.outputs.changes}}
+          prerelease: ${{steps.changelog_reader.outputs.status == 'prereleased'}}
+          draft: ${{steps.changelog_reader.outputs.status == 'unreleased'}}
+      - name: Fetch image artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{inputs.py_ver}}-${{inputs.dist}}
+      - name: Fetch pip artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}}
+          path: dist
+      - name: Publish pip release
+        run: |
+          export HOST_UID=$(id -u)
+          docker load < "image-${{env.PKG_NAME}}-${PKG_VERSION}-py${{inputs.py_ver}}-${{inputs.dist}}.tar.gz"
+          docker-compose -f docker-compose-build.yaml run \
+            -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} element \
+            sh -lc "pip install twine && python -m twine upload dist/*"
+      - name: Determine pip artifact paths
+        run: |
+          echo "PKG_WHEEL_PATH=$(ls dist/${PKG_NAME}-*.whl)" >> $GITHUB_ENV
+          echo "PKG_SDIST_PATH=$(ls dist/${SDIST_PKG_NAME}-*.tar.gz)" >> $GITHUB_ENV
+      - name: Upload pip wheel asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.PKG_WHEEL_PATH}}
+          asset_name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}}.whl
+          asset_content_type: application/zip
+      - name: Upload pip sdist asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.PKG_SDIST_PATH}}
+          asset_name: pip-${{env.SDIST_PKG_NAME}}-${{env.PKG_VERSION}}.tar.gz
+          asset_content_type: application/gzip
diff --git a/.github/workflows/sciops_docker_images.yaml b/.github/workflows/sciops_docker_images.yaml
@@ -57,7 +57,7 @@ jobs:
         name: Build codebook env image
         run: |
           ## Get docker template
-          cp -r ./cicd/docker-template/* ./main/docker/
+          cp -r ./cicd/docker-template/codebook_env ./main/docker/
 
           ## Get build info
           cd ./main/docker/codebook_env
@@ -167,7 +167,7 @@ jobs:
           github.event_name == 'push' &&
           startsWith(github.ref, 'refs/tags') &&
           github.repository_owner == 'dj-sciops'
-        run: |-
+        run: |
           cd ./docker
 
           find . -maxdepth 1 -type d \( -iname "*" ! -iname "*local*" \) | awk "NR>1" | cut -d/ -f2 | while read DIR;

diff --git a/.github/workflows/u24_element_build_release.yaml b/.github/workflows/u24_element_build_release.yaml
@@ -0,0 +1,162 @@
+name: u24_elements
+on:
+  workflow_call:
+    inputs:
+      py_ver:
+        required: true
+        type: string
+      dist:
+        required: true
+        type: string
+      image:
+        required: true
+        type: string
+jobs:
+  test-changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get changelog entry
+        id: changelog_reader
+        uses: guzman-raphael/changelog-reader-action@v5
+        with:
+          path: ./CHANGELOG.md
+      - name: Verify changelog parsing
+        env:
+          TAG_NAME: ${{steps.changelog_reader.outputs.version}}
+          RELEASE_NAME: Release ${{steps.changelog_reader.outputs.version}}
+          BODY: ${{steps.changelog_reader.outputs.changes}}
+          PRERELEASE: ${{steps.changelog_reader.outputs.status == 'prereleased'}}
+          DRAFT: ${{steps.changelog_reader.outputs.status == 'unreleased'}}
+        run: |
+          echo "TAG_NAME=${TAG_NAME}"
+          echo "RELEASE_NAME=${RELEASE_NAME}"
+          echo "BODY=${BODY}"
+          echo "PRERELEASE=${PRERELEASE}"
+          echo "DRAFT=${DRAFT}"
+  build:
+    needs: test-changelog
+    runs-on: ubuntu-latest
+    env:
+      PY_VER: ${{inputs.py_ver}}
+      DIST: ${{inputs.dist}}
+      IMAGE: ${{inputs.image}}
+      DOCKER_CLIENT_TIMEOUT: "120"
+      COMPOSE_HTTP_TIMEOUT: "120"
+    steps:
+      - name: Checkout element repo
+        uses: actions/checkout@v2
+        with:
+          path: 'main'
+      - name: Checkout cicd repo
+        uses: actions/checkout@v2
+        with:
+          repository: 'dj-sciops/djsciops-cicd'
+          path: 'cicd'
+      - name: Compile image
+        run: |
+          ## Get docker template
+          cp -r ./cicd/docker-template/elements/dist/${DIST}/* ./main/
+          cd ./main
+
+          export PKG_NAME=$(python3 -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])")
+          export PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+')
+          export HOST_UID=$(id -u)
+          docker-compose -f docker-compose-build.yaml up --exit-code-from element --build
+          IMAGE=$(docker images --filter "reference=datajoint/${PKG_NAME}*" \
+            --format "{{.Repository}}")
+          TAG=$(docker images --filter "reference=datajoint/${PKG_NAME}*" --format "{{.Tag}}")
+          docker save "${IMAGE}:${TAG}" | \
+            gzip > "image-${PKG_NAME}-${PKG_VERSION}-py${PY_VER}-${DIST}.tar.gz"
+          echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_ENV
+          echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_ENV
+      - name: Add image artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{env.PY_VER}}-${{env.DIST}}
+          path: "./main/image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{env.PY_VER}}-${{env.DIST}}.tar.gz"
+          retention-days: 1
+      - if: env.PY_VER == '3.9' && env.DIST == 'alpine'
+        name: Add pip artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}}
+          path: ./main/dist
+          retention-days: 1
+  publish-release:
+    if: |
+      github.event_name == 'push' &&
+      startsWith(github.ref, 'refs/tags') &&
+      github.repository_owner == 'dj-sciops'
+    needs: build
+    runs-on: ubuntu-latest
+    env:
+      TWINE_USERNAME: ${{secrets.twine_username}}
+      TWINE_PASSWORD: ${{secrets.twine_password}}
+    outputs:
+      release_upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Determine package version
+        run: |
+          PKG_NAME=$(python3 -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])")
+          SDIST_PKG_NAME=$(echo ${PKG_NAME} | sed 's|_|-|g')
+          PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+')
+          echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_ENV
+          echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_ENV
+          echo "SDIST_PKG_NAME=${SDIST_PKG_NAME}" >> $GITHUB_ENV
+      - name: Get changelog entry
+        id: changelog_reader
+        uses: guzman-raphael/changelog-reader-action@v5
+        with:
+          path: ./CHANGELOG.md
+          version: ${{env.PKG_VERSION}}
+      - name: Create GH release
+        id: create_gh_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          tag_name: ${{steps.changelog_reader.outputs.version}}
+          release_name: Release ${{steps.changelog_reader.outputs.version}}
+          body: ${{steps.changelog_reader.outputs.changes}}
+          prerelease: ${{steps.changelog_reader.outputs.status == 'prereleased'}}
+          draft: ${{steps.changelog_reader.outputs.status == 'unreleased'}}
+      - name: Fetch image artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{inputs.py_ver}}-${{inputs.dist}}
+      - name: Fetch pip artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}}
+          path: dist
+      - name: Publish pip release
+        run: |
+          export HOST_UID=$(id -u)
+          docker load < "image-${{env.PKG_NAME}}-${PKG_VERSION}-py${{inputs.py_ver}}-${{inputs.dist}}.tar.gz"
+          docker-compose -f docker-compose-build.yaml run \
+            -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} element \
+            sh -lc "pip install twine && python -m twine upload dist/*"
+      - name: Determine pip artifact paths
+        run: |
+          echo "PKG_WHEEL_PATH=$(ls dist/${PKG_NAME}-*.whl)" >> $GITHUB_ENV
+          echo "PKG_SDIST_PATH=$(ls dist/${SDIST_PKG_NAME}-*.tar.gz)" >> $GITHUB_ENV
+      - name: Upload pip wheel asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.PKG_WHEEL_PATH}}
+          asset_name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}}.whl
+          asset_content_type: application/zip
+      - name: Upload pip sdist asset to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        with:
+          upload_url: ${{steps.create_gh_release.outputs.upload_url}}
+          asset_path: ${{env.PKG_SDIST_PATH}}
+          asset_name: pip-${{env.SDIST_PKG_NAME}}-${{env.PKG_VERSION}}.tar.gz
+          asset_content_type: application/gzip
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+.vscode
 ## Github Actions
 .github/.test