WIP: Versioned images and release on pypi

.bumpversion.cfg

@@ -1,16 +1,21 @@
 [bumpversion]
-current_version = 0.0.1
+current_version = 0.0.15.dev3
 commit = True
 tag = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.dev(?P<dev>\d+))?
+serialize = 
+	{major}.{minor}.{patch}.dev{dev}
+	{major}.{minor}.{patch}
+commit-args = --no-verify
 
 [bumpversion:file:setup.py]
-search = version='{current_version}'
-replace = version='{new_version}'
-
-[bumpversion:file:docs/conf.py]
-search = version = release = '{current_version}'
-replace = version = release = '{new_version}'
+search = version\="{current_version}"
+replace = {new_version}
 
 [bumpversion:file:derex/runner/__init__.py]
-search = __version__ = '{current_version}'
-replace = __version__ = '{new_version}'
+search = __version__ \= "{current_version}"
+replace = {new_version}
+
+[bumpversion:file:docker-definition/Dockerfile]
+search = ENV DEREX_VERSION\='{current_version}'
+replace = {new_version}

.envrc

@@ -1,3 +1,5 @@
 layout_python python3.7
 
 export DEREX_ADMIN_SERVICES=true
+export DOCKER_BUILDKIT=1
+export DOCKER_CLI_EXPERIMENTAL=enabled

.pre-commit-config.yaml

@@ -3,9 +3,11 @@ repos:
   rev: v2.4.0
   hooks:
   - id: trailing-whitespace
+    exclude: .bumpversion.cfg
   - id: debug-statements
   - id: check-added-large-files
   - id: end-of-file-fixer
+    exclude: .bumpversion.cfg
   - id: check-yaml
   - id: flake8
 

CONTRIBUTING.rst

@@ -18,22 +18,9 @@ Report Bugs
 Report bugs at
 https://github.com/Abstract-Tech/derex.runner/issues.
 
-If possible, provide a minimal derex project to trigger the bug.
+Please provide a minimal derex project and instructions to trigger the bug.
 
 
-Submit Feedback
-~~~~~~~~~~~~~~~
-
-The best way to send feedback is to file an issue at
-https://github.com/Abstract-Tech/derex.runner/issues.
-
-If you are proposing a feature:
-
-* Explain in detail how it would work.
-* Keep the scope as narrow as possible, to make it easier to implement.
-* Remember that this is a volunteer-driven project, and that contributions
-  are welcome :)
-
 Get Started!
 ------------
 
@@ -73,7 +60,9 @@ local development.
     $ git commit -m "Your detailed description of your changes."
     $ git push origin name-of-your-bugfix-or-feature
 
-7. Submit a pull request through the GitHub website.
+8. Create an azure pipeline to run your tests and make sure you keep it green.
+
+9. When everything is green submit a pull request through the GitHub website.
 
 Pull Request Guidelines
 -----------------------
@@ -93,6 +82,6 @@ A reminder for the maintainers on how to deploy.
 Make sure all your changes are committed (including an entry in HISTORY.rst).
 Then run::
 
-    $ bumpversion patch # possible: major / minor / patch
+    $ bump2version patch # possible: major / minor / patch
     $ git push
     $ git push --tags

MANIFEST.in

@@ -9,6 +9,7 @@ recursive-include derex/runner/templates *
 recursive-include derex/runner/compose_files *
 recursive-include derex/runner/fixtures *
 recursive-include derex/runner/settings * *.py
+recursive-include docker-definition *
 recursive-include tests *
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]

README.rst

@@ -3,11 +3,11 @@ Manage Open edX projects
 
 
 .. image:: https://img.shields.io/azure-devops/tests/abstract-technology/derex.runner/5/master?compact_message&style=for-the-badge
-   :target: https://dev.azure.com/abstract-technology/derex.runner/_build
+   :target: https://dev.azure.com/abstract-technology/derex/_build?definitionId=12&_a=summary&repositoryFilter=12&branchFilter=198
    :alt: Test results
 
 .. image:: https://img.shields.io/azure-devops/coverage/abstract-technology/derex.runner/5/master?style=for-the-badge
-   :target: https://dev.azure.com/abstract-technology/derex.runner/_build
+   :target: https://dev.azure.com/abstract-technology/derex/_build?definitionId=12&_a=summary&repositoryFilter=12&branchFilter=198
    :alt: Coverage results
 
 Introduction

azure-pipelines.yml

@@ -6,243 +6,29 @@ schedules:
       - master
   always: true
 
-jobs:
-- job: RunPytests
-  timeoutInMinutes: 40
-  pool:
-    vmImage: 'ubuntu-latest'
-  steps:
-    - template: azure-pipelines/prepare.yml
-
-    - script: |
-        set -ex
-        set -o pipefail
-        cd tests
-        pytest -m "not slowtest" --ignore-docstrings |grep -v codecoveragetool=Cobertura
-      displayName: 'Run python tests'
-
-    - publish: $(System.DefaultWorkingDirectory)/tests/.coverage
-      artifact: fasttests_coverage
-
-
-- job: RunSlowPytests
-  timeoutInMinutes: 40
-  pool:
-    vmImage: 'ubuntu-latest'
-  steps:
-    - template: azure-pipelines/prepare.yml
-
-    - script: ddc-services pull
-      displayName: 'Pull edX services images'
-
-    - script: |
-        set -ex
-        export DEREX_ADMIN_SERVICES=False
-        ddc-services up -d
-        derex reset-mailslurper
-      displayName: 'Start service daemons'
-
-    - script: |
-        # Slow tests do not currently contribute to coverage
-        set -ex
-        set -o pipefail
-        cd tests
-        pytest -m "slowtest" --ignore-docstrings |grep -v codecoveragetool=Cobertura
-      displayName: 'Run python tests'
-
-    - publish: $(System.DefaultWorkingDirectory)/tests/.coverage
-      artifact: slowtests_coverage
-
-- job: CombineCoverage
-  dependsOn:
-    - RunPytests
-    - RunSlowPytests
-  pool:
-    vmImage: 'ubuntu-latest'
-  steps:
-    - template: azure-pipelines/prepare.yml
-
-    - download: current
-
-    - task: Cache@2
-      inputs:
-        key: npmcache2
-        path: $(Pipeline.Workspace)/.cache/npm
-      displayName: Cache npm packages
-
-    - script: |
-        set -ex
-        cp $(Pipeline.Workspace)/slowtests_coverage/.coverage tests/.coverage.slow
-        cp $(Pipeline.Workspace)/fasttests_coverage/.coverage tests/.coverage.fast
-        cd tests
-        coverage combine
-        coverage html
-        coverage xml
-        cd ..
-        # We installed the package with pip, and coverage reports the full absolute path.
-        # We cut to derex/runner/etc/etc
-        DEREX_RUNNER_PATH=`cd tests;python -c "from pathlib import Path; import derex.runner;print(Path(derex.runner.__file__).parent.parent.parent)"`
-        echo Replacing ${DEREX_RUNNER_PATH} in tests/htmlcov/*.html
-        sudo npm config set cache $(Pipeline.Workspace)/.cache/npm --global
-        npm install juice
-        # Azure pipelines strips style sheets but leaves styles in place.
-        # juice can embed the styles in the HTML for us and present a much better
-        # view in the coverage results tab.
-        for filename in tests/htmlcov/*.html; do $(npm bin)/juice $filename $filename; done
-        echo "##vso[codecoverage.publish codecoveragetool=Cobertura;summaryfile=${PWD}/tests/coverage.xml;reportdirectory=${PWD}/tests/htmlcov;]"
-      displayName: Fix coverage result and publish it
-      condition: succeededOrFailed()
-
-
-- job: TestComplete
-  timeoutInMinutes: 40
-  pool:
-    vmImage: 'ubuntu-latest'
-  steps:
-    - template: azure-pipelines/prepare.yml
-
-    - script: ddc-services pull
-      displayName: 'Pull edX services images'
-
-    - script: |
-        set -ex
-        export DEREX_ADMIN_SERVICES=False
-        ddc-services up -d
-        derex reset-mailslurper
-      displayName: 'Start service daemons'
-
-    - script: |
-        set -ex
-        cd tests/fixtures/complete
-        derex reset-mysql
-        derex reset-rabbitmq
-        ddc-services exec -T mysql mysql -h localhost --protocol tcp -u root -psecret complete_myedx -e "SELECT * from auth_user WHERE username='derex.runner'"|grep derex.runner
-      displayName: 'Test complete project fixtures'
-
-    - script: |
-        set -ex
-        cd tests/fixtures/complete
-        derex build-final
-      displayName: 'Build image for project "complete"'
-
-    - script: cd tests/fixtures/complete; ddc-project up -d
-      displayName: 'Start edX daemons'
-
-    - script: docker run --network derex -v $(pwd)/nginx.conf:/etc/nginx/conf.d/default.conf -p 80:80 -d --rm nginx:1.17.0-alpine
-      displayName: 'Start nginx to expose LMS/CMS on the same domain/port'
-
-    - script: |
-        echo 127.0.0.1 localhost studio.localhost | sudo tee -a /etc/hosts
-        sleep 3  # Give the services a little time to start up
-      displayName: Add studio.localhost to /etc/hosts
-
-    - script: |
-        cd tests/fixtures/complete; ddc-project logs cms lms
-      condition: always()
-      displayName: 'Show CMS/LMS logs'
-
-    - script: |
-        $CMD="curl --connect-timeout 30 --retry 5 --retry-delay 5 -f http://localhost:4700/"
-        $CMD || (sleep 10; $CMD )
-      displayName: 'Curl the LMS'
-
-    - script: |
-        $CMD="curl --connect-timeout 30 --retry 5 --retry-delay 5 -f http://localhost:4800/"
-        $CMD || (sleep 10; $CMD )
-      displayName: 'Curl the CMS'
-
-    - script: |
-        set -ex
-        cd tests/fixtures/complete
-        docker images
-        # This should not be needed, but for some reason Azure does not tag the image
-        docker tag $(docker images -q|head -n1) complete/openedx-themes
-        docker images
-
-        echo Installing dive
-        wget -q https://github.com/wagoodman/dive/releases/download/v0.8.0/dive_0.8.0_linux_amd64.deb
-        DEBIAN_FRONTEND=noninteractive sudo -E apt-get install -y ./dive_0.8.0_linux_amd64.deb
-        echo Analyzing image
-        dive --ci complete/openedx-themes
-      condition: always()
-      displayName: 'Test the complete image with dive'
-      continueOnError: true  # Do not make the build fail because of this step, for now
-      timeoutInMinutes: 40
-
-    - script: |
-        derex runmode production
-        cd tests/fixtures/complete; derex build-themes; ddc-project up -d lms cms
-        sleep 5  # Give it time to start up
-      displayName: 'Start production image lms/cms'
-
-    - script: |
-        cd tests/fixtures/complete
-        ddc-project config
-        ddc-project logs cms lms
-      condition: always()
-      displayName: 'Show CMS/LMS final logs'
-
-    - script: |
-        $CMD="curl --connect-timeout 30 --retry 5 --retry-delay 5 -f http://localhost:4710/"
-        $CMD || (sleep 10; $CMD )
-      continueOnError: true  # Temporarily disable this test: it's failing with this line in the logs:
-      # pymongo.errors.OperationFailure: command SON([('saslStart', 1), ('mechanism', 'SCRAM-SHA-1'), ('autoAuthorize', 1), ('payload', Binary(b'n,,n=edxapp,r=MDg3MjIwMjQyMDc4MTg3MjQ=', 0))]) on namespace complete_mongoedx.$cmd failed: Authentication failed.
-      displayName: 'Curl the final LMS'
-
-    - script: |
-        $CMD="curl --connect-timeout 30 --retry 5 --retry-delay 5 -f http://localhost:4810/"
-        $CMD || (sleep 10; $CMD )
-      continueOnError: true  # Temporarily disable this test: it's failing with the line mentioned above
-      displayName: 'Curl the final CMS'
-
-    - script: |
-        ddc-services logs
-        cd tests/fixtures/complete; ddc-project logs
-      condition: always()
-      displayName: 'Show logs'
-
-    - script: |
-        cd tests/fixtures/complete; ddc-project config
-      condition: always()
-      displayName: 'Show ddc-project config'
-
-
-- job: TestMinimal
-  timeoutInMinutes: 40
-  pool:
-    vmImage: 'ubuntu-latest'
-  steps:
-    - template: azure-pipelines/prepare.yml
-
-    - script: ddc-services pull
-      displayName: 'Pull edX services images'
-
-    - script: |
-        set -ex
-        export DEREX_ADMIN_SERVICES=False
-        ddc-services up -d
-        derex reset-mailslurper
-      displayName: 'Start service daemons'
-
-    - script: |
-        set -ex
-        cd tests/fixtures/minimal
-        ddc-project pull
-      displayName: 'Pull edx image'
-
-    - script: |
-        set -ex
-        cd tests/fixtures/minimal
-        derex reset-mysql
-        derex reset-rabbitmq
-      displayName: 'Prime Mysql DB'
-
-
-- job: CheckDocs
-  timeoutInMinutes: 40
-  pool:
-    vmImage: 'ubuntu-latest'
-  steps:
-    - template: azure-pipelines/prepare.yml
-    - script: make docs
-      displayName: 'Compile docs'
+stages:
+  - template: azure-pipelines/build.yml
+  - template: azure-pipelines/test.yml
+  - template: azure-pipelines/push.yml
+
+variables:
+  - name: DOCKER_BUILDKIT
+    value: '1'
+  - name: DOCKER_CLI_EXPERIMENTAL
+    value: enabled
+  - name: DOCKER_COMMAND
+    value: docker buildx build docker-definition
+  - name: CACHE_KEY_BASE  # Increment this value to reset the images cache
+    value: 21
+  - name: CACHE_KEY_FILES
+    value: $(System.DefaultWorkingDirectory)/docker-definition/*
+  - name: CACHE_KEY_IMAGES
+    value: BASE_SEED=$(CACHE_KEY_BASE) | $(CACHE_KEY_FILES)
+  - name: CACHE_KEY_IMAGES_FALLBACK
+    value: BASE_SEED=$(CACHE_KEY_BASE)
+  - name: SENTINEL_CACHE_PATH
+    value: $(Pipeline.Workspace)/sentinel_cache
+  - name: IMAGE_CACHE_PATH
+    value: $(Pipeline.Workspace)/image_cache
+  - name: needs_build
+    value: "false"