From 7853d626548d75e2d8d258cf4452473c1c4c5688 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Thu, 12 Oct 2023 15:24:56 +0000 Subject: [PATCH] wip: cache db --- .github/actions/build-push/action.yml | 41 +++++++++++++++--------- .github/actions/unittests/action.yml | 18 ++++++++--- .github/workflows/single-test-simple.yml | 2 +- tasks.py | 15 ++------- 4 files changed, 43 insertions(+), 33 deletions(-) diff --git a/.github/actions/build-push/action.yml b/.github/actions/build-push/action.yml index fcf8839b..d536bb78 100644 --- a/.github/actions/build-push/action.yml +++ b/.github/actions/build-push/action.yml @@ -21,11 +21,15 @@ inputs: description: "GitHub Username" required: true push: - description: "Push Docker Image" + description: "Build and Push Docker Image" required: false default: "" load: - description: "Load Docker Image" + description: "Build and Load Docker Image" + required: false + default: "" + pull: + description: "Pull Docker Image" required: false default: "" runs: @@ -67,25 +71,30 @@ runs: build-args: | NAUTOBOT_VER=${{ inputs.nautobot-version }} PYTHON_VER=${{ inputs.python-version }} - # - name: "Load" - # if: | - # inputs.load == 'true' - # uses: "docker/build-push-action@v5" - # with: - # load: true - # context: "./" - # file: "./development/Dockerfile" - # tags: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}" - # cache-from: "type=gha,scope=${{ inputs.image-tag }}" - # build-args: | - # NAUTOBOT_VER=${{ inputs.nautobot-version }} - # PYTHON_VER=${{ inputs.python-version }} + - name: "Load" + if: | + inputs.load == 'true' + uses: "docker/build-push-action@v5" + with: + load: true + context: "./" + file: "./development/Dockerfile" + tags: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}" + cache-from: "type=gha,scope=${{ inputs.image-tag }}" + build-args: | + NAUTOBOT_VER=${{ inputs.nautobot-version }} + PYTHON_VER=${{ inputs.python-version }} - name: "Pull" shell: "bash" if: | - inputs.load == 'true' + inputs.pull == 'true' run: | docker pull '${{ inputs.image-prefix }}:${{ inputs.image-tag }}' + - name: "Tag" + shell: "bash" + if: | + inputs.pull == 'true' || inputs.load == 'true' + run: | docker tag '${{ inputs.image-prefix }}:${{ inputs.image-tag }}' '${{ steps.config.outputs.image }}' outputs: ghcr-image: diff --git a/.github/actions/unittests/action.yml b/.github/actions/unittests/action.yml index 5858be7a..4dc92af6 100644 --- a/.github/actions/unittests/action.yml +++ b/.github/actions/unittests/action.yml @@ -18,6 +18,8 @@ runs: id: "config" env: COMPOSE_FILE: "docker-compose.base.yml:docker-compose.${{ inputs.db-backend }}.yml:docker-compose.redis.yml:docker-compose.dev.yml" + NAUTOBOT_VER: "${{ inputs.nautobot-version }}" + PYTHON_VER: "${{ inputs.python-version }}" shell: "bash" run: | if [[ "${{ inputs.db-backend }}" == "mysql" ]]; then @@ -25,9 +27,10 @@ runs: fi cd development - docker compose up -- db + docker compose up --detach -- db redis - CACHE-KEY=$(docker compose run --rm --entrypoint='' -- nautobot invoke calc-dbdump-cache-key) + DB_IMAGE="$(docker compose convert --format json | jq -r .services.db.image)" + CACHE-KEY=$(docker compose run --rm --entrypoint='' -- nautobot invoke calc-dbdump-cache-key --salt='$DB_IMAGE') echo "cache-key=$CACHE-KEY" | tee -a "$GITHUB_OUTPUT" - name: "Restore Cache Database Dump" @@ -45,14 +48,21 @@ runs: run: | cd development if [[ -f ../dump.sql ]]; then - docker-compose run --rm --entrypoint='' -- nautobot invoke import-db --import-file ../dump.sql + docker-compose run --rm --entrypoint='' -- nautobot \ + invoke import-db --input-file=../dump.sql + else + docker compose exec -- db createdb --user nautobot test_nautobot + docker-compose run --rm --entrypoint='' --env=NAUTOBOT_DB_NAME=test_nautobot -- nautobot \ + nautobot-server migrate + docker-compose run --rm --entrypoint='' --env=NAUTOBOT_DB_NAME=test_nautobot -- nautobot \ + invoke backup-db --output-file=../dump.sql fi docker compose run \ --rm \ --entrypoint='' \ -- \ nautobot \ - invoke unittest --failfast + invoke unittest --failfast --keepdb - name: "Store Cache Database Dump" uses: actions/cache@v3 with: diff --git a/.github/workflows/single-test-simple.yml b/.github/workflows/single-test-simple.yml index 5e67b0a6..ee72cf01 100644 --- a/.github/workflows/single-test-simple.yml +++ b/.github/workflows/single-test-simple.yml @@ -24,7 +24,7 @@ jobs: with: image-prefix: "ghcr.io/${{ github.repository }}/nautobot-dev" image-tag: "pr-${{ github.event.pull_request.number }}-${{ env.NAUTOBOT_VER }}-py${{ env.PYTHON_VER }}" - load: true + pull: true nautobot-version: "${{ env.NAUTOBOT_VER }}" password: ${{ secrets.GH_NAUTOBOT_BOT_TOKEN }} # push: true diff --git a/tasks.py b/tasks.py index 48b09bc0..15bc79c3 100644 --- a/tasks.py +++ b/tasks.py @@ -693,15 +693,14 @@ def tests(context, failfast=False, keepdb=False, lint_only=False, test_docs=True print("All tests have passed!") -@task -def calc_dbdump_cache_key(context): +@task(help={"salt": "Salt to use when generating cache key."}) +def calc_dbdump_cache_key(context, salt=""): """Calculate database dump cache key. Calculate cache key as: - `migrations` folder file content. - Nautobot version. - - Database server Docker image reference. """ migrations_dir = Path("nautobot_firewall_modesl/migrations") hasher = sha256() @@ -714,14 +713,6 @@ def calc_dbdump_cache_key(context): hasher.update(context.nautobot_firewall_models.nautobot_ver.encode()) - with context.cd(context.nautobot_firewall_models.compose_dir): - db_image_ref = context.run( - "docker compose convert --format json | jq -r .services.db.image", - hide=True, - env={ - "COMPOSE_FILE": ":".join(context.nautobot_firewall_models.compose_files), - }, - ) - hasher.update(db_image_ref.stdout.strip().encode()) + hasher.update(salt.encode()) print(hasher.hexdigest())