Skip to content

Commit

Permalink
wip: cache db
Browse files Browse the repository at this point in the history
  • Loading branch information
snaselj committed Oct 12, 2023
1 parent 288dd8d commit 7853d62
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 33 deletions.
41 changes: 25 additions & 16 deletions .github/actions/build-push/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,15 @@ inputs:
description: "GitHub Username"
required: true
push:
description: "Push Docker Image"
description: "Build and Push Docker Image"
required: false
default: ""
load:
description: "Load Docker Image"
description: "Build and Load Docker Image"
required: false
default: ""
pull:
description: "Pull Docker Image"
required: false
default: ""
runs:
Expand Down Expand Up @@ -67,25 +71,30 @@ runs:
build-args: |
NAUTOBOT_VER=${{ inputs.nautobot-version }}
PYTHON_VER=${{ inputs.python-version }}
# - name: "Load"
# if: |
# inputs.load == 'true'
# uses: "docker/build-push-action@v5"
# with:
# load: true
# context: "./"
# file: "./development/Dockerfile"
# tags: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}"
# cache-from: "type=gha,scope=${{ inputs.image-tag }}"
# build-args: |
# NAUTOBOT_VER=${{ inputs.nautobot-version }}
# PYTHON_VER=${{ inputs.python-version }}
- name: "Load"
if: |
inputs.load == 'true'
uses: "docker/build-push-action@v5"
with:
load: true
context: "./"
file: "./development/Dockerfile"
tags: "${{ inputs.image-prefix }}:${{ inputs.image-tag }}"
cache-from: "type=gha,scope=${{ inputs.image-tag }}"
build-args: |
NAUTOBOT_VER=${{ inputs.nautobot-version }}
PYTHON_VER=${{ inputs.python-version }}
- name: "Pull"
shell: "bash"
if: |
inputs.load == 'true'
inputs.pull == 'true'
run: |
docker pull '${{ inputs.image-prefix }}:${{ inputs.image-tag }}'
- name: "Tag"
shell: "bash"
if: |
inputs.pull == 'true' || inputs.load == 'true'
run: |
docker tag '${{ inputs.image-prefix }}:${{ inputs.image-tag }}' '${{ steps.config.outputs.image }}'
outputs:
ghcr-image:
Expand Down
18 changes: 14 additions & 4 deletions .github/actions/unittests/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,19 @@ runs:
id: "config"
env:
COMPOSE_FILE: "docker-compose.base.yml:docker-compose.${{ inputs.db-backend }}.yml:docker-compose.redis.yml:docker-compose.dev.yml"
NAUTOBOT_VER: "${{ inputs.nautobot-version }}"
PYTHON_VER: "${{ inputs.python-version }}"
shell: "bash"
run: |
if [[ "${{ inputs.db-backend }}" == "mysql" ]]; then
cp invoke.mysql.yml invoke.yml
fi
cd development
docker compose up -- db
docker compose up --detach -- db redis
CACHE-KEY=$(docker compose run --rm --entrypoint='' -- nautobot invoke calc-dbdump-cache-key)
DB_IMAGE="$(docker compose convert --format json | jq -r .services.db.image)"
CACHE-KEY=$(docker compose run --rm --entrypoint='' -- nautobot invoke calc-dbdump-cache-key --salt='$DB_IMAGE')
echo "cache-key=$CACHE-KEY" | tee -a "$GITHUB_OUTPUT"
- name: "Restore Cache Database Dump"
Expand All @@ -45,14 +48,21 @@ runs:
run: |
cd development
if [[ -f ../dump.sql ]]; then
docker-compose run --rm --entrypoint='' -- nautobot invoke import-db --import-file ../dump.sql
docker-compose run --rm --entrypoint='' -- nautobot \
invoke import-db --input-file=../dump.sql
else
docker compose exec -- db createdb --user nautobot test_nautobot
docker-compose run --rm --entrypoint='' --env=NAUTOBOT_DB_NAME=test_nautobot -- nautobot \
nautobot-server migrate
docker-compose run --rm --entrypoint='' --env=NAUTOBOT_DB_NAME=test_nautobot -- nautobot \
invoke backup-db --output-file=../dump.sql
fi
docker compose run \
--rm \
--entrypoint='' \
-- \
nautobot \
invoke unittest --failfast
invoke unittest --failfast --keepdb
- name: "Store Cache Database Dump"
uses: actions/cache@v3
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/single-test-simple.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
with:
image-prefix: "ghcr.io/${{ github.repository }}/nautobot-dev"
image-tag: "pr-${{ github.event.pull_request.number }}-${{ env.NAUTOBOT_VER }}-py${{ env.PYTHON_VER }}"
load: true
pull: true
nautobot-version: "${{ env.NAUTOBOT_VER }}"
password: ${{ secrets.GH_NAUTOBOT_BOT_TOKEN }}
# push: true
Expand Down
15 changes: 3 additions & 12 deletions tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -693,15 +693,14 @@ def tests(context, failfast=False, keepdb=False, lint_only=False, test_docs=True
print("All tests have passed!")


@task
def calc_dbdump_cache_key(context):
@task(help={"salt": "Salt to use when generating cache key."})
def calc_dbdump_cache_key(context, salt=""):
"""Calculate database dump cache key.
Calculate cache key as:
- `migrations` folder file content.
- Nautobot version.
- Database server Docker image reference.
"""
migrations_dir = Path("nautobot_firewall_modesl/migrations")
hasher = sha256()
Expand All @@ -714,14 +713,6 @@ def calc_dbdump_cache_key(context):

hasher.update(context.nautobot_firewall_models.nautobot_ver.encode())

with context.cd(context.nautobot_firewall_models.compose_dir):
db_image_ref = context.run(
"docker compose convert --format json | jq -r .services.db.image",
hide=True,
env={
"COMPOSE_FILE": ":".join(context.nautobot_firewall_models.compose_files),
},
)
hasher.update(db_image_ref.stdout.strip().encode())
hasher.update(salt.encode())

print(hasher.hexdigest())

0 comments on commit 7853d62

Please sign in to comment.