diff --git a/.github/.staging_workflows/anchored_sciops_docker_images.yaml b/.github/.staging_workflows/anchored_sciops_docker_images.yaml index 5df5a20..92106ee 100644 --- a/.github/.staging_workflows/anchored_sciops_docker_images.yaml +++ b/.github/.staging_workflows/anchored_sciops_docker_images.yaml @@ -101,7 +101,7 @@ jobs: name: Build codebook env image run: | ## Get docker template - cp -r ./cicd/docker-template/* ./main/docker/ + cp -r ./cicd/docker-template/codebook_env ./main/docker/ ## Get build info cd ./main/docker/codebook_env diff --git a/.github/.staging_workflows/anchored_u24_element_build_release.yaml b/.github/.staging_workflows/anchored_u24_element_build_release.yaml new file mode 100644 index 0000000..36c1186 --- /dev/null +++ b/.github/.staging_workflows/anchored_u24_element_build_release.yaml @@ -0,0 +1,184 @@ +name: u24_elements + +on: + workflow_call: + inputs: + py_ver: + required: true + type: string + dist: + required: true + type: string + image: + required: true + type: string + +anchor-dev-push: &dev-push + if: | + github.event_name == 'push' && + startsWith(github.ref, 'refs/tags') && + github.repository_owner == 'yambottle' +anchor-dev-cicd-repo: &dev-cicd-repo + repository: 'yambottle/djsciops-cicd' +anchor-test-push: &test-push + if: | + github.event_name == 'push' && + startsWith(github.ref, 'refs/tags') && + github.repository_owner == 'yambottle' +anchor-test-cicd-repo: &test-cicd-repo + repository: 'yambottle/djsciops-cicd' +anchor-prod-push: &prod-push + if: | + github.event_name == 'push' && + startsWith(github.ref, 'refs/tags') && + github.repository_owner == 'dj-sciops' +anchor-prod-cicd-repo: &prod-cicd-repo + repository: 'dj-sciops/djsciops-cicd' + +jobs: + test-changelog: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get changelog entry + id: changelog_reader + uses: guzman-raphael/changelog-reader-action@v5 + with: + path: ./CHANGELOG.md + - name: Verify changelog parsing + env: + TAG_NAME: ${{steps.changelog_reader.outputs.version}} + RELEASE_NAME: Release ${{steps.changelog_reader.outputs.version}} + BODY: ${{steps.changelog_reader.outputs.changes}} + PRERELEASE: ${{steps.changelog_reader.outputs.status == 'prereleased'}} + DRAFT: ${{steps.changelog_reader.outputs.status == 'unreleased'}} + run: | + echo "TAG_NAME=${TAG_NAME}" + echo "RELEASE_NAME=${RELEASE_NAME}" + echo "BODY=${BODY}" + echo "PRERELEASE=${PRERELEASE}" + echo "DRAFT=${DRAFT}" + build: + needs: test-changelog + runs-on: ubuntu-latest + env: + PY_VER: ${{inputs.py_ver}} + DIST: ${{inputs.dist}} + IMAGE: ${{inputs.image}} + DOCKER_CLIENT_TIMEOUT: "120" + COMPOSE_HTTP_TIMEOUT: "120" + steps: + - name: Checkout element repo + uses: actions/checkout@v2 + with: + path: 'main' + - name: Checkout cicd repo + uses: actions/checkout@v2 + with: + !!merge <<: *$STAGE-cicd-repo + path: 'cicd' + - name: Compile image + run: | + ## Get docker template + cp -r ./cicd/docker-template/elements/dist/${DIST}/* ./main/ + cd ./main + + export PKG_NAME=$(python3 -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])") + export PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+') + export HOST_UID=$(id -u) + docker-compose -f docker-compose-build.yaml up --exit-code-from element --build + IMAGE=$(docker images --filter "reference=datajoint/${PKG_NAME}*" \ + --format "{{.Repository}}") + TAG=$(docker images --filter "reference=datajoint/${PKG_NAME}*" --format "{{.Tag}}") + docker save "${IMAGE}:${TAG}" | \ + gzip > "image-${PKG_NAME}-${PKG_VERSION}-py${PY_VER}-${DIST}.tar.gz" + echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_ENV + echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_ENV + - name: Add image artifact + uses: actions/upload-artifact@v2 + with: + name: image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{env.PY_VER}}-${{env.DIST}} + path: "./main/image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{env.PY_VER}}-\ + ${{env.DIST}}.tar.gz" + retention-days: 1 + - if: env.PY_VER == '3.9' && env.DIST == 'alpine' + name: Add pip artifacts + uses: actions/upload-artifact@v2 + with: + name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}} + path: ./main/dist + retention-days: 1 + publish-release: + !!merge <<: *$STAGE-push + needs: build + runs-on: ubuntu-latest + env: + TWINE_USERNAME: ${{secrets.twine_username}} + TWINE_PASSWORD: ${{secrets.twine_password}} + outputs: + release_upload_url: ${{steps.create_gh_release.outputs.upload_url}} + steps: + - uses: actions/checkout@v2 + - name: Determine package version + run: | + PKG_NAME=$(python3 -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])") + SDIST_PKG_NAME=$(echo ${PKG_NAME} | sed 's|_|-|g') + PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+') + echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_ENV + echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_ENV + echo "SDIST_PKG_NAME=${SDIST_PKG_NAME}" >> $GITHUB_ENV + - name: Get changelog entry + id: changelog_reader + uses: guzman-raphael/changelog-reader-action@v5 + with: + path: ./CHANGELOG.md + version: ${{env.PKG_VERSION}} + - name: Create GH release + id: create_gh_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + tag_name: ${{steps.changelog_reader.outputs.version}} + release_name: Release ${{steps.changelog_reader.outputs.version}} + body: ${{steps.changelog_reader.outputs.changes}} + prerelease: ${{steps.changelog_reader.outputs.status == 'prereleased'}} + draft: ${{steps.changelog_reader.outputs.status == 'unreleased'}} + - name: Fetch image artifact + uses: actions/download-artifact@v2 + with: + name: image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{inputs.py_ver}}-${{inputs.dist}} + - name: Fetch pip artifacts + uses: actions/download-artifact@v2 + with: + name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}} + path: dist + - name: Publish pip release + run: | + export HOST_UID=$(id -u) + docker load < "image-${{env.PKG_NAME}}-${PKG_VERSION}-py${{inputs.py_ver}}-${{inputs.dist}}.tar.gz" + docker-compose -f docker-compose-build.yaml run \ + -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} element \ + sh -lc "pip install twine && python -m twine upload dist/*" + - name: Determine pip artifact paths + run: | + echo "PKG_WHEEL_PATH=$(ls dist/${PKG_NAME}-*.whl)" >> $GITHUB_ENV + echo "PKG_SDIST_PATH=$(ls dist/${SDIST_PKG_NAME}-*.tar.gz)" >> $GITHUB_ENV + - name: Upload pip wheel asset to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + upload_url: ${{steps.create_gh_release.outputs.upload_url}} + asset_path: ${{env.PKG_WHEEL_PATH}} + asset_name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}}.whl + asset_content_type: application/zip + - name: Upload pip sdist asset to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + upload_url: ${{steps.create_gh_release.outputs.upload_url}} + asset_path: ${{env.PKG_SDIST_PATH}} + asset_name: pip-${{env.SDIST_PKG_NAME}}-${{env.PKG_VERSION}}.tar.gz + asset_content_type: application/gzip \ No newline at end of file diff --git a/.github/workflows/sciops_docker_images.yaml b/.github/workflows/sciops_docker_images.yaml index 7d11b7a..bfe982f 100644 --- a/.github/workflows/sciops_docker_images.yaml +++ b/.github/workflows/sciops_docker_images.yaml @@ -57,7 +57,7 @@ jobs: name: Build codebook env image run: | ## Get docker template - cp -r ./cicd/docker-template/* ./main/docker/ + cp -r ./cicd/docker-template/codebook_env ./main/docker/ ## Get build info cd ./main/docker/codebook_env @@ -167,7 +167,7 @@ jobs: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') && github.repository_owner == 'dj-sciops' - run: |- + run: | cd ./docker find . -maxdepth 1 -type d \( -iname "*" ! -iname "*local*" \) | awk "NR>1" | cut -d/ -f2 | while read DIR; diff --git a/.github/workflows/u24_element_build_release.yaml b/.github/workflows/u24_element_build_release.yaml new file mode 100644 index 0000000..273dffa --- /dev/null +++ b/.github/workflows/u24_element_build_release.yaml @@ -0,0 +1,162 @@ +name: u24_elements +on: + workflow_call: + inputs: + py_ver: + required: true + type: string + dist: + required: true + type: string + image: + required: true + type: string +jobs: + test-changelog: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get changelog entry + id: changelog_reader + uses: guzman-raphael/changelog-reader-action@v5 + with: + path: ./CHANGELOG.md + - name: Verify changelog parsing + env: + TAG_NAME: ${{steps.changelog_reader.outputs.version}} + RELEASE_NAME: Release ${{steps.changelog_reader.outputs.version}} + BODY: ${{steps.changelog_reader.outputs.changes}} + PRERELEASE: ${{steps.changelog_reader.outputs.status == 'prereleased'}} + DRAFT: ${{steps.changelog_reader.outputs.status == 'unreleased'}} + run: | + echo "TAG_NAME=${TAG_NAME}" + echo "RELEASE_NAME=${RELEASE_NAME}" + echo "BODY=${BODY}" + echo "PRERELEASE=${PRERELEASE}" + echo "DRAFT=${DRAFT}" + build: + needs: test-changelog + runs-on: ubuntu-latest + env: + PY_VER: ${{inputs.py_ver}} + DIST: ${{inputs.dist}} + IMAGE: ${{inputs.image}} + DOCKER_CLIENT_TIMEOUT: "120" + COMPOSE_HTTP_TIMEOUT: "120" + steps: + - name: Checkout element repo + uses: actions/checkout@v2 + with: + path: 'main' + - name: Checkout cicd repo + uses: actions/checkout@v2 + with: + repository: 'dj-sciops/djsciops-cicd' + path: 'cicd' + - name: Compile image + run: | + ## Get docker template + cp -r ./cicd/docker-template/elements/dist/${DIST}/* ./main/ + cd ./main + + export PKG_NAME=$(python3 -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])") + export PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+') + export HOST_UID=$(id -u) + docker-compose -f docker-compose-build.yaml up --exit-code-from element --build + IMAGE=$(docker images --filter "reference=datajoint/${PKG_NAME}*" \ + --format "{{.Repository}}") + TAG=$(docker images --filter "reference=datajoint/${PKG_NAME}*" --format "{{.Tag}}") + docker save "${IMAGE}:${TAG}" | \ + gzip > "image-${PKG_NAME}-${PKG_VERSION}-py${PY_VER}-${DIST}.tar.gz" + echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_ENV + echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_ENV + - name: Add image artifact + uses: actions/upload-artifact@v2 + with: + name: image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{env.PY_VER}}-${{env.DIST}} + path: "./main/image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{env.PY_VER}}-${{env.DIST}}.tar.gz" + retention-days: 1 + - if: env.PY_VER == '3.9' && env.DIST == 'alpine' + name: Add pip artifacts + uses: actions/upload-artifact@v2 + with: + name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}} + path: ./main/dist + retention-days: 1 + publish-release: + if: | + github.event_name == 'push' && + startsWith(github.ref, 'refs/tags') && + github.repository_owner == 'dj-sciops' + needs: build + runs-on: ubuntu-latest + env: + TWINE_USERNAME: ${{secrets.twine_username}} + TWINE_PASSWORD: ${{secrets.twine_password}} + outputs: + release_upload_url: ${{steps.create_gh_release.outputs.upload_url}} + steps: + - uses: actions/checkout@v2 + - name: Determine package version + run: | + PKG_NAME=$(python3 -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])") + SDIST_PKG_NAME=$(echo ${PKG_NAME} | sed 's|_|-|g') + PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+') + echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_ENV + echo "PKG_VERSION=${PKG_VERSION}" >> $GITHUB_ENV + echo "SDIST_PKG_NAME=${SDIST_PKG_NAME}" >> $GITHUB_ENV + - name: Get changelog entry + id: changelog_reader + uses: guzman-raphael/changelog-reader-action@v5 + with: + path: ./CHANGELOG.md + version: ${{env.PKG_VERSION}} + - name: Create GH release + id: create_gh_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + tag_name: ${{steps.changelog_reader.outputs.version}} + release_name: Release ${{steps.changelog_reader.outputs.version}} + body: ${{steps.changelog_reader.outputs.changes}} + prerelease: ${{steps.changelog_reader.outputs.status == 'prereleased'}} + draft: ${{steps.changelog_reader.outputs.status == 'unreleased'}} + - name: Fetch image artifact + uses: actions/download-artifact@v2 + with: + name: image-${{env.PKG_NAME}}-${{env.PKG_VERSION}}-py${{inputs.py_ver}}-${{inputs.dist}} + - name: Fetch pip artifacts + uses: actions/download-artifact@v2 + with: + name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}} + path: dist + - name: Publish pip release + run: | + export HOST_UID=$(id -u) + docker load < "image-${{env.PKG_NAME}}-${PKG_VERSION}-py${{inputs.py_ver}}-${{inputs.dist}}.tar.gz" + docker-compose -f docker-compose-build.yaml run \ + -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} element \ + sh -lc "pip install twine && python -m twine upload dist/*" + - name: Determine pip artifact paths + run: | + echo "PKG_WHEEL_PATH=$(ls dist/${PKG_NAME}-*.whl)" >> $GITHUB_ENV + echo "PKG_SDIST_PATH=$(ls dist/${SDIST_PKG_NAME}-*.tar.gz)" >> $GITHUB_ENV + - name: Upload pip wheel asset to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + upload_url: ${{steps.create_gh_release.outputs.upload_url}} + asset_path: ${{env.PKG_WHEEL_PATH}} + asset_name: pip-${{env.PKG_NAME}}-${{env.PKG_VERSION}}.whl + asset_content_type: application/zip + - name: Upload pip sdist asset to release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + with: + upload_url: ${{steps.create_gh_release.outputs.upload_url}} + asset_path: ${{env.PKG_SDIST_PATH}} + asset_name: pip-${{env.SDIST_PKG_NAME}}-${{env.PKG_VERSION}}.tar.gz + asset_content_type: application/gzip diff --git a/.gitignore b/.gitignore index ea727af..36f9707 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.vscode ## Github Actions .github/.test diff --git a/docker-template/elements/dist/alpine/Dockerfile b/docker-template/elements/dist/alpine/Dockerfile new file mode 100644 index 0000000..91f8e5c --- /dev/null +++ b/docker-template/elements/dist/alpine/Dockerfile @@ -0,0 +1,21 @@ +ARG PY_VER +ARG DIST +ARG IMAGE +ARG PKG_NAME +ARG PKG_VERSION + +FROM datajoint/${IMAGE}:py${PY_VER}-${DIST} +USER root +RUN \ + apk update && \ + apk add git +USER anaconda + +COPY --chown=anaconda:anaconda ./requirements.txt ./setup.py \ + /main/ +COPY --chown=anaconda:anaconda ./${PKG_NAME} /main/${PKG_NAME} +RUN \ + cd /main && \ + pip install . && \ + rm -R /main/* +WORKDIR /main \ No newline at end of file diff --git a/docker-template/elements/dist/alpine/docker-compose-build.yaml b/docker-template/elements/dist/alpine/docker-compose-build.yaml new file mode 100644 index 0000000..cf2f3fe --- /dev/null +++ b/docker-template/elements/dist/alpine/docker-compose-build.yaml @@ -0,0 +1,26 @@ +# PY_VER=3.8 IMAGE=djbase DISTRO=alpine PKG_NAME=$(python -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])") PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+') HOST_UID=$(id -u) docker-compose -f docker-compose-build.yaml up --exit-code-from element --build +# +# Intended for updating dependencies and docker image. +# Used to build release artifacts. +version: "2.4" +services: + element: + build: + context: . + args: + - PY_VER + - DIST + - IMAGE + - PKG_NAME + - PKG_VERSION + image: datajoint/${PKG_NAME}:${PKG_VERSION} + user: ${HOST_UID}:anaconda + volumes: + - .:/main + command: + - sh + - -lc + - | + set -e + rm -R build dist *.egg-info || echo "No prev build" + python setup.py bdist_wheel sdist diff --git a/docker-template/elements/dist/debian/Dockerfile b/docker-template/elements/dist/debian/Dockerfile new file mode 100644 index 0000000..0f3b0a9 --- /dev/null +++ b/docker-template/elements/dist/debian/Dockerfile @@ -0,0 +1,21 @@ +ARG PY_VER +ARG DIST +ARG IMAGE +ARG PKG_NAME +ARG PKG_VERSION + +FROM datajoint/${IMAGE}:py${PY_VER}-${DIST} +USER root +RUN \ + apt update && \ + apt-get install -y git +USER anaconda + +COPY --chown=anaconda:anaconda ./requirements.txt ./setup.py \ + /main/ +COPY --chown=anaconda:anaconda ./${PKG_NAME} /main/${PKG_NAME} +RUN \ + cd /main && \ + pip install . && \ + rm -R /main/* +WORKDIR /main \ No newline at end of file diff --git a/docker-template/elements/dist/debian/docker-compose-build.yaml b/docker-template/elements/dist/debian/docker-compose-build.yaml new file mode 100644 index 0000000..b88c227 --- /dev/null +++ b/docker-template/elements/dist/debian/docker-compose-build.yaml @@ -0,0 +1,26 @@ +# PY_VER=3.8 IMAGE=djbase DISTRO=alpine PKG_NAME=$(python -c "print([p for p in __import__('setuptools').find_packages() if '.' not in p][0])") PKG_VERSION=$(cat ${PKG_NAME}/version.py | grep -oP '\d+\.\d+\.[a-z0-9]+') HOST_UID=$(id -u) docker-compose -f docker-compose-build.yaml up --exit-code-from element --build +# +# Intended for updating dependencies and docker image. +# Used to build release artifacts. +version: "2.4" +services: + element: + build: + context: . + args: + - PY_VER + - DIST + - IMAGE + - PKG_NAME + - PKG_VERSION + image: datajoint/${PKG_NAME}:${PKG_VERSION} + user: ${HOST_UID}:anaconda + volumes: + - .:/main + command: + - bash + - -c + - | + set -e + rm -R build dist *.egg-info || echo "No prev build" + python setup.py bdist_wheel sdist