From f21b16d0ec6fb2d956dbd4d6b06fba0fdefbbc79 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 14 Jan 2025 22:09:11 +0000 Subject: [PATCH] ci: Use generic Docker volume cache action (#3524) See https://github.com/BYK/docker-volume-cache-action --- .github/workflows/test.yml | 47 +++++++++++++--------------------- action.yaml | 48 ++++++++++++----------------------- install/upgrade-clickhouse.sh | 2 +- 3 files changed, 35 insertions(+), 62 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed1547909a..3e471c84ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,12 +56,9 @@ jobs: sudo curl -L https://github.com/docker/compose/releases/download/v2.26.0/docker-compose-`uname -s`-`uname -m` -o "/usr/local/lib/docker/cli-plugins/docker-compose" sudo chmod +x "/usr/local/lib/docker/cli-plugins/docker-compose" - - name: Prepare Docker Volume Caching + - name: Compute Docker Volume Cache Key id: cache_key run: | - # Set permissions for docker volumes so we can cache and restore - sudo chmod o+x /var/lib/docker - sudo chmod -R o+rwx /var/lib/docker/volumes source .env SENTRY_MIGRATIONS_MD5=$(docker run --rm --entrypoint bash $SENTRY_IMAGE -c 'ls -Rv1rpq src/sentry/migrations/' | md5sum | cut -d ' ' -f 1) echo "SENTRY_MIGRATIONS_MD5=$SENTRY_MIGRATIONS_MD5" >> $GITHUB_OUTPUT @@ -70,44 +67,36 @@ jobs: - name: Restore DB Volumes Cache id: restore_cache - uses: actions/cache/restore@v4 + uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd with: - key: db-volumes-v5-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }} + key: db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }} restore-keys: | - db-volumes-v5-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} - db-volumes-v5- - path: | - /var/lib/docker/volumes/sentry-postgres/_data - /var/lib/docker/volumes/sentry-clickhouse/_data - /var/lib/docker/volumes/sentry-kafka/_data + db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} + db-volumes-v6- + volumes: | + sentry-postgres + sentry-clickhouse + sentry-kafka - name: Install ${{ env.LATEST_TAG }} env: SKIP_DB_MIGRATIONS: ${{ steps.restore_cache.outputs.cache-hit == 'true' && '1' || '' }} run: | - # This is for the cache restore on Kafka to work in older releases - docker run --rm -v "sentry-kafka:/data" busybox chown -R 1000:1000 /data + # This is to compensate for a bug in upgrade-clickhouse where + # if we have sentry-clickhouse volume without the rest, it fails + # We may get sentry-clickhouse from the cache step above + source install/create-docker-volumes.sh ./install.sh - - name: Prepare Docker Volume Caching - run: | - # Set permissions for docker volumes so we can cache and restore - # We need these for the backup/restore test snapshotting too - sudo chmod o+x /var/lib/docker - sudo chmod -R o+rx /var/lib/docker/volumes - # Set tar ownership for it to be able to read - # From: https://github.com/actions/toolkit/issues/946#issuecomment-1726311681 - sudo chown root /usr/bin/tar && sudo chmod u+s /usr/bin/tar - - name: Save DB Volumes Cache if: steps.restore_cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 + uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd with: key: ${{ steps.restore_cache.outputs.cache-primary-key }} - path: | - /var/lib/docker/volumes/sentry-postgres/_data - /var/lib/docker/volumes/sentry-clickhouse/_data - /var/lib/docker/volumes/sentry-kafka/_data + volumes: | + sentry-postgres + sentry-clickhouse + sentry-kafka - name: Checkout current ref uses: actions/checkout@v4 diff --git a/action.yaml b/action.yaml index fe2cc073ec..ec5897f0a2 100644 --- a/action.yaml +++ b/action.yaml @@ -18,7 +18,7 @@ runs: shell: bash run: | image_var=$(echo ${{ inputs.project_name }}_IMAGE | tr '[:lower:]' '[:upper:]') - echo "${image_var}=${{ inputs.image_url }}" >> ${{ github.action_path }}/.env + echo "${image_var}=${{ inputs.image_url }}" >> ${{ github.action_path }}.env - name: Setup dev environment shell: bash @@ -54,13 +54,10 @@ runs: sudo curl -L https://github.com/docker/compose/releases/download/${{ env.COMPOSE_VERSION }}/docker-compose-`uname -s`-`uname -m` -o "${{ env.COMPOSE_PATH }}/docker-compose" sudo chmod +x "${{ env.COMPOSE_PATH }}/docker-compose" - - name: Prepare Docker Volume Caching + - name: Compute Docker Volume Cache Key id: cache_key shell: bash run: | - # Set permissions for docker volumes so we can cache and restore - sudo chmod o+x /var/lib/docker - sudo chmod -R o+rwx /var/lib/docker/volumes source ${{ github.action_path }}/.env # See https://explainshell.com/explain?cmd=ls%20-Rv1rpq # for that long `ls` command @@ -71,16 +68,16 @@ runs: - name: Restore DB Volumes Cache id: restore_cache - uses: actions/cache/restore@v4 + uses: BYK/docker-volume-cache-action/restore@be89365902126f508dcae387a32ec3712df6b1cd with: - key: db-volumes-v5-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }} + key: db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }}-${{ steps.cache_key.outputs.SENTRY_MIGRATIONS_MD5 }} restore-keys: | - db-volumes-v5-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} - db-volumes-v5- - path: | - /var/lib/docker/volumes/sentry-postgres/_data - /var/lib/docker/volumes/sentry-clickhouse/_data - /var/lib/docker/volumes/sentry-kafka/_data + db-volumes-v6-${{ steps.cache_key.outputs.SNUBA_MIGRATIONS_MD5 }} + db-volumes-v6- + volumes: | + sentry-postgres + sentry-clickhouse + sentry-kafka - name: Install self-hosted env: @@ -88,8 +85,6 @@ runs: shell: bash run: | cd ${{ github.action_path }} - # This is for the cache restore on Kafka to work in older releases - docker run --rm -v "sentry-kafka:/data" busybox chown -R 1000:1000 /data # Add some customizations to test that path cat <> sentry/enhance-image.sh #!/bin/bash @@ -102,31 +97,20 @@ runs: ./install.sh --no-report-self-hosted-issues --skip-commit-check - - name: Prepare Docker Volume Caching - shell: bash - run: | - # Set permissions for docker volumes so we can cache and restore - # We need these for the backup/restore test snapshotting too - sudo chmod o+x /var/lib/docker - sudo chmod -R o+rx /var/lib/docker/volumes - # Set tar ownership for it to be able to read - # From: https://github.com/actions/toolkit/issues/946#issuecomment-1726311681 - sudo chown root /usr/bin/tar && sudo chmod u+s /usr/bin/tar - sudo chown root /usr/bin/rsync && sudo chmod u+s /usr/bin/rsync - - name: Save DB Volumes Cache if: steps.restore_cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 + uses: BYK/docker-volume-cache-action/save@be89365902126f508dcae387a32ec3712df6b1cd with: key: ${{ steps.restore_cache.outputs.cache-primary-key }} - path: | - /var/lib/docker/volumes/sentry-postgres/_data - /var/lib/docker/volumes/sentry-clickhouse/_data - /var/lib/docker/volumes/sentry-kafka/_data + volumes: | + sentry-postgres + sentry-clickhouse + sentry-kafka - name: Integration Test shell: bash run: | + sudo chown root /usr/bin/rsync && sudo chmod u+s /usr/bin/rsync rsync -aW --no-compress --mkpath \ /var/lib/docker/volumes/sentry-postgres \ /var/lib/docker/volumes/sentry-clickhouse \ diff --git a/install/upgrade-clickhouse.sh b/install/upgrade-clickhouse.sh index 05e74bb00b..bd69d7bf74 100644 --- a/install/upgrade-clickhouse.sh +++ b/install/upgrade-clickhouse.sh @@ -1,7 +1,7 @@ echo "${_group}Upgrading Clickhouse ..." # First check to see if user is upgrading by checking for existing clickhouse volume -if [[ -n "$(docker volume ls -q --filter name=sentry-clickhouse)" ]]; then +if docker compose ps -a | grep -q clickhouse; then # Start clickhouse if it is not already running $dc up --wait clickhouse