diff --git a/.github/environment/README.md b/.github/environment/README.md new file mode 100644 index 0000000..25658a3 --- /dev/null +++ b/.github/environment/README.md @@ -0,0 +1,13 @@ +## Github Deployment Environments +Github deployment environments are used to define unique settings for each environment i.e. staging and production + +The build and push workflows need to know which AWS account to push updated docker images to. + +### DotEnv Files +The .env files in this directory are here as a record of the "variables" and their values. + +The variables can be updated from these files using the following command: +```bash +gh variable set -R aodn/ -e -f .env + +``` diff --git a/.github/environment/production.env b/.github/environment/production.env new file mode 100644 index 0000000..cf20cb0 --- /dev/null +++ b/.github/environment/production.env @@ -0,0 +1,3 @@ +AWS_REGION=ap-southeast-2 +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 diff --git a/.github/environment/staging.env b/.github/environment/staging.env new file mode 100644 index 0000000..cf20cb0 --- /dev/null +++ b/.github/environment/staging.env @@ -0,0 +1,3 @@ +AWS_REGION=ap-southeast-2 +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 diff --git a/.github/workflows/build-production.yml b/.github/workflows/build-production.yml new file mode 100644 index 0000000..fc56bff --- /dev/null +++ b/.github/workflows/build-production.yml @@ -0,0 +1,102 @@ +name: Build Production + +on: + push: + tags: + - v*.*.* + +permissions: + id-token: write + contents: read + +env: + environment_name: production + +jobs: + build_push: + runs-on: ubuntu-latest + environment: production + outputs: + image_digest: ${{ steps.build_and_push.outputs.digest }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build and Push Docker Image + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . + # Only building for AMD64 for now + # platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.ref_name }} + + - name: Push Image Digest to SSM + run: | + aws ssm put-parameter \ + --name "/apps/sample-django-app/${{ env.environment_name }}/image_digest" \ + --type "String" \ + --value "$digest" \ + --overwrite + env: + digest: ${{ steps.build_and_push.outputs.digest }} + + trigger_deploy: + runs-on: ubuntu-latest + needs: [build_push] + steps: + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" + + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: '${{ env.environment_name }}' + } + }) diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml new file mode 100644 index 0000000..19ccb5c --- /dev/null +++ b/.github/workflows/build-staging.yml @@ -0,0 +1,113 @@ +name: Build Staging + +on: + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - '.github/environment/**' + # This is only here for testing purposes while in a PR + #TODO remove before merge to main + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - '.github/environment/**' + +permissions: + id-token: write + contents: read + +env: + environment_name: staging + +jobs: + build_push: + runs-on: ubuntu-latest + environment: staging + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build and Push Docker Image + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . + # Only building for AMD64 for now + # platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.sha }} + + - name: Push Image Digest to SSM + run: | + aws ssm put-parameter \ + --name "/apps/geonetwork4/${{ env.environment_name }}/image_digest" \ + --type "String" \ + --value "$digest" \ + --overwrite + env: + digest: ${{ steps.build_and_push.outputs.digest }} + + trigger_deploy: + runs-on: ubuntu-latest + needs: [build_push] + steps: + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" + + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: '${{ env.environment_name }}' + } + }) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..64730fc --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,20 @@ +name: Run Pre-commit Checks + +on: + pull_request: + branches: + - master + +permissions: + id-token: write + contents: read + +jobs: + pre_commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..17144b0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,69 @@ +name: Test + +on: + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - '.github/environment/**' + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + build_test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Setup Docker Structure Test + run: > + curl -LO + https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 + && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 + /usr/local/bin/container-structure-test + + - name: Set Image Tag + id: set_image_tag + run: | + branch_name=${{ github.head_ref || github.ref_name }} + tag=${{ env.TAG_PREFIX}}-${branch_name//\//-} + echo "$tag" + echo "image_tag=$tag" >> $GITHUB_OUTPUT + env: + TAG_PREFIX: test + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Build Docker Image + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: image:${{ github.sha }} + + - name: Test Docker Image + run: | + container-structure-test test --image image:${{ github.sha }} --config tests/config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f670082 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + # Security + - id: detect-private-key + - repo: https://github.com/gitleaks/gitleaks + rev: v8.18.0 + hooks: + - id: gitleaks diff --git a/README.md b/README.md index 7f732dd..a2a31db 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repo is used to customize a GeoNetwork4 to be used by AODN. The key customi build a jar file with the code we want to add and insert it use COPY function to copy it to the lib folder during the Dockerfile build. -The jar file contains a hack where we name a @Configuration class with the same package as +The jar file contains a hack where we name a @Configuration class with the same package as geonetwork4 base package (org.fao.geonet), so that the initial component-scan will pick up this class. From there we add additional component-scan to our custom classes. This avoided the need to alter the xml like what we did before plus we are using a Docker base image of GeoNetwork4. @@ -29,6 +29,6 @@ Once you have the json, you can generate code like the one here in Java to acces ## Use of S3 -You can see a config file related to S3, however we do not use it because after experiment it, it +You can see a config file related to S3, however we do not use it because after experiment it, it didn't support well as the GN4 will issue warning on file not found with relative folder name. The -code is just keep as a record. +code is just keep as a record. diff --git a/pom.xml b/pom.xml index 448b1d3..b2ec6a1 100644 --- a/pom.xml +++ b/pom.xml @@ -179,4 +179,4 @@ - \ No newline at end of file + diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 00a396b..7cb1cc4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,4 +2,4 @@ aodn.geonetwork4.esIndexer.protocol=http aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:localhost} aodn.geonetwork4.esIndexer.port=${INDEXER_PORT:80} aodn.geonetwork4.esIndexer.apikey=${INDEXER_APIKEY} -aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.protocol}://${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid} \ No newline at end of file +aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.protocol}://${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid} diff --git a/src/main/resources/schema/gn4-api.json b/src/main/resources/schema/gn4-api.json index a176751..c15e17b 100644 --- a/src/main/resources/schema/gn4-api.json +++ b/src/main/resources/schema/gn4-api.json @@ -22473,4 +22473,4 @@ } } } -} \ No newline at end of file +} diff --git a/startEsLocal.sh b/startEsLocal.sh index 92b505d..99a8326 100755 --- a/startEsLocal.sh +++ b/startEsLocal.sh @@ -31,4 +31,3 @@ fi # If you run in EC2, you need to change the docker-es-compose.yml localhost in cert to the hostname of EC2 and install docker # docker-compose and sudo chmod 666 /run/docker.sock docker-compose -f docker-es-compose.yml up --detach - diff --git a/tests/README-tests.md b/tests/README-tests.md new file mode 100644 index 0000000..c850d8e --- /dev/null +++ b/tests/README-tests.md @@ -0,0 +1,17 @@ +# Docker Image Structural Testing +The configuration file `test/config.yaml` defines various "structural" checks to perform on the docker image created by `../Dockerfile`. + +The test require the use of the [container-structure-test](https://github.com/GoogleContainerTools/container-structure-test) from Google. + +Follow their documentation to install. + +## Usage +Generate your docker image using your preferred method e.g. +```shell +docker build -t myimage . +``` + +To run the tests against the image, run the following: +```shell +container-structure-test test --image myimage --config tests/config.yaml +``` diff --git a/tests/config.yaml b/tests/config.yaml new file mode 100644 index 0000000..7981f83 --- /dev/null +++ b/tests/config.yaml @@ -0,0 +1,10 @@ +schemaVersion: 2.0.0 + +commandTests: + - name: "say hello world" + command: "bash" + args: + - -c + - | + echo hello && + echo world