diff --git a/.github/environment/README.md b/.github/environment/README.md new file mode 100644 index 0000000..25658a3 --- /dev/null +++ b/.github/environment/README.md @@ -0,0 +1,13 @@ +## Github Deployment Environments +Github deployment environments are used to define unique settings for each environment i.e. staging and production + +The build and push workflows need to know which AWS account to push updated docker images to. + +### DotEnv Files +The .env files in this directory are here as a record of the "variables" and their values. + +The variables can be updated from these files using the following command: +```bash +gh variable set -R aodn/ -e -f .env + +``` diff --git a/.github/environment/production.env b/.github/environment/production.env new file mode 100644 index 0000000..dd518e8 --- /dev/null +++ b/.github/environment/production.env @@ -0,0 +1,5 @@ +AWS_REGION=ap-southeast-2 +CODEARTIFACT_DOMAIN=gamma-aodn-org-au +CODEARTIFACT_REPO=maven-aodn-store +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 diff --git a/.github/environment/staging.env b/.github/environment/staging.env new file mode 100644 index 0000000..dd518e8 --- /dev/null +++ b/.github/environment/staging.env @@ -0,0 +1,5 @@ +AWS_REGION=ap-southeast-2 +CODEARTIFACT_DOMAIN=gamma-aodn-org-au +CODEARTIFACT_REPO=maven-aodn-store +ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com +ECR_REPOSITORY=geonetwork4 diff --git a/.github/workflows/build-production.yml b/.github/workflows/build-production.yml new file mode 100644 index 0000000..9898327 --- /dev/null +++ b/.github/workflows/build-production.yml @@ -0,0 +1,130 @@ +name: Build Production + +on: + push: + tags: + - v*.*.* + +permissions: + id-token: write + contents: read + +env: + environment_name: production + +jobs: + build_push: + runs-on: ubuntu-latest + environment: production + outputs: + image_digest: ${{ steps.build_and_push.outputs.digest }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + server-id: 'codeartifact' + server-password: 'CODEARTIFACT_AUTH_TOKEN' + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Get CodeArtifact Repository Authentication Token + run: | + TOKEN=$(aws codeartifact get-authorization-token \ + --domain ${{ vars.CODEARTIFACT_DOMAIN }} \ + --domain-owner ${{ steps.aws_auth.outputs.aws-account-id }} \ + --region ${{ vars.AWS_REGION }} \ + --query authorizationToken \ + --output text) + echo "CODEARTIFACT_AUTH_TOKEN=$TOKEN" >> "$GITHUB_ENV" + + - name: Get CodeArtifact Repository URL + run: | + REPO_URL=$(aws codeartifact get-repository-endpoint \ + --domain ${{ vars.CODEARTIFACT_DOMAIN }} \ + --repository ${{ vars.CODEARTIFACT_REPO }} \ + --format maven \ + --region ${{ vars.AWS_REGION }} \ + --output text) + echo "CODEARTIFACT_REPO_URL=$REPO_URL" >> "$GITHUB_ENV" + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Publish Artifacts to CodeArtifact Repository + run: mvn -B deploy + env: + CODEARTIFACT_AUTH_TOKEN: ${{ env.CODEARTIFACT_AUTH_TOKEN }} + CODEARTIFACT_REPO_URL: ${{ env.CODEARTIFACT_REPO_URL }} + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build and Push Docker Image + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . + # Only building for AMD64 for now + # platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.ref_name }} + + - name: Push Image Digest to SSM + run: | + aws ssm put-parameter \ + --name "/apps/sample-django-app/${{ env.environment_name }}/image_digest" \ + --type "String" \ + --value "$digest" \ + --overwrite + env: + digest: ${{ steps.build_and_push.outputs.digest }} + + trigger_deploy: + runs-on: ubuntu-latest + needs: [build_push] + steps: + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" + + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: '${{ env.environment_name }}' + } + }) diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml new file mode 100644 index 0000000..0389809 --- /dev/null +++ b/.github/workflows/build-staging.yml @@ -0,0 +1,128 @@ +name: Build Staging + +on: + push: + branches: + - main + paths-ignore: + - '**/*.md' + - '.github/environment/**' + +permissions: + id-token: write + contents: read + +env: + environment_name: staging + +jobs: + build_push: + runs-on: ubuntu-latest + environment: staging + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + server-id: 'codeartifact' + server-password: 'CODEARTIFACT_AUTH_TOKEN' + + - name: Configure AWS Credentials + id: aws_auth + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: ${{ vars.AWS_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + + - name: Get CodeArtifact Repository Authentication Token + run: | + TOKEN=$(aws codeartifact get-authorization-token \ + --domain ${{ vars.CODEARTIFACT_DOMAIN }} \ + --domain-owner ${{ steps.aws_auth.outputs.aws-account-id }} \ + --region ${{ vars.AWS_REGION }} \ + --query authorizationToken \ + --output text) + echo "CODEARTIFACT_AUTH_TOKEN=$TOKEN" >> "$GITHUB_ENV" + + - name: Get CodeArtifact Repository URL + run: | + REPO_URL=$(aws codeartifact get-repository-endpoint \ + --domain ${{ vars.CODEARTIFACT_DOMAIN }} \ + --repository ${{ vars.CODEARTIFACT_REPO }} \ + --format maven \ + --region ${{ vars.AWS_REGION }} \ + --output text) + echo "CODEARTIFACT_REPO_URL=$REPO_URL" >> "$GITHUB_ENV" + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Login to ECR + uses: docker/login-action@v3 + with: + registry: ${{ vars.ECR_REGISTRY }} + + - name: Build and Push Docker Image + id: build_and_push + uses: docker/build-push-action@v5 + with: + context: . + # Only building for AMD64 for now + # platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.sha }} + ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest + + - name: Push Image Digest to SSM + run: | + aws ssm put-parameter \ + --name "/apps/geonetwork4/${{ env.environment_name }}/image_digest" \ + --type "String" \ + --value "$digest" \ + --overwrite + env: + digest: ${{ steps.build_and_push.outputs.digest }} + + trigger_deploy: + runs-on: ubuntu-latest + needs: [build_push] + steps: + - name: Generate App Token + uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: "appdeploy" + + - name: Trigger Deploy Workflow + uses: actions/github-script@v7 + with: + github-token: ${{ steps.app-token.outputs.token }} + retries: 3 + retry-exempt-status-codes: 204 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'aodn', + repo: 'appdeploy', + workflow_id: 'deploy.yml', + ref: 'main', + inputs: { + app_name: 'geonetwork4', + environment: '${{ env.environment_name }}' + } + }) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..64730fc --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,20 @@ +name: Run Pre-commit Checks + +on: + pull_request: + branches: + - master + +permissions: + id-token: write + contents: read + +jobs: + pre_commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..17144b0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,69 @@ +name: Test + +on: + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + - '.github/environment/**' + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +permissions: + id-token: write + contents: read + +jobs: + build_test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Setup Docker Structure Test + run: > + curl -LO + https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64 + && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64 + /usr/local/bin/container-structure-test + + - name: Set Image Tag + id: set_image_tag + run: | + branch_name=${{ github.head_ref || github.ref_name }} + tag=${{ env.TAG_PREFIX}}-${branch_name//\//-} + echo "$tag" + echo "image_tag=$tag" >> $GITHUB_OUTPUT + env: + TAG_PREFIX: test + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn -B package --file pom.xml + + - name: Build Docker Image + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: image:${{ github.sha }} + + - name: Test Docker Image + run: | + container-structure-test test --image image:${{ github.sha }} --config tests/config.yaml diff --git a/.gitignore b/.gitignore index de1853a..b8b70e1 100644 --- a/.gitignore +++ b/.gitignore @@ -26,9 +26,10 @@ replay_pid* # Ingore local generated folders elasticdata/ gn4_data/ -src/main/generated/ .env .idea/ -**/target/ \ No newline at end of file +**/target/ + +**/.git-versioned-pom.xml diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml new file mode 100644 index 0000000..540c45f --- /dev/null +++ b/.mvn/extensions.xml @@ -0,0 +1,10 @@ + + + + me.qoomon + maven-git-versioning-extension + 9.7.0 + + + diff --git a/.mvn/maven-git-versioning-extension.xml b/.mvn/maven-git-versioning-extension.xml new file mode 100644 index 0000000..a54262e --- /dev/null +++ b/.mvn/maven-git-versioning-extension.xml @@ -0,0 +1,25 @@ + + + + + .+ + ${ref}-SNAPSHOT + + ${ref} + + + + + .*)]]> + ${ref.version} + + + + + + ${commit} + + + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f670082 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + # Security + - id: detect-private-key + - repo: https://github.com/gitleaks/gitleaks + rev: v8.18.0 + hooks: + - id: gitleaks diff --git a/README.md b/README.md index 7d2105b..2efa2ea 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repo is used to customize a GeoNetwork4 to be used by AODN. The key customi build a jar file with the code we want to add and insert it use COPY function to copy it to the lib folder during the Dockerfile build. -The jar file contains a hack where we name a @Configuration class with the same package as +The jar file contains a hack where we name a @Configuration class with the same package as geonetwork4 base package (org.fao.geonet), so that the initial component-scan will pick up this class. From there we add additional component-scan to our custom classes. This avoided the need to alter the xml like what we did before plus we are using a Docker base image of GeoNetwork4. @@ -54,6 +54,6 @@ Once you have the json, you can generate code like the one here in Java to acces ## Use of S3 -You can see a config file related to S3, however we do not use it because after experiment it, it +You can see a config file related to S3, however we do not use it because after experiment it, it didn't support well as the GN4 will issue warning on file not found with relative folder name. The -code is just keep as a record. +code is just keep as a record. diff --git a/geonetwork-api/pom.xml b/geonetwork-api/pom.xml index 2d54d17..6884594 100644 --- a/geonetwork-api/pom.xml +++ b/geonetwork-api/pom.xml @@ -5,7 +5,7 @@ geonetwork4 au.org.aodn - 1.0-SNAPSHOT + 0.0.0 ../pom.xml 4.0.0 @@ -13,6 +13,10 @@ geonetwork-api jar + + false + + org.geonetwork-opensource @@ -88,4 +92,4 @@ - \ No newline at end of file + diff --git a/geonetwork-api/src/main/resources/schema/gn4-api.json b/geonetwork-api/src/main/resources/schema/gn4-api.json index f8a176b..42db60a 100644 --- a/geonetwork-api/src/main/resources/schema/gn4-api.json +++ b/geonetwork-api/src/main/resources/schema/gn4-api.json @@ -22476,4 +22476,4 @@ } } } -} \ No newline at end of file +} diff --git a/geonetwork/pom.xml b/geonetwork/pom.xml index 64af8a6..2bc958e 100644 --- a/geonetwork/pom.xml +++ b/geonetwork/pom.xml @@ -5,7 +5,7 @@ geonetwork4 au.org.aodn - 1.0-SNAPSHOT + 0.0.0 ../pom.xml 4.0.0 @@ -13,6 +13,10 @@ geonetwork jar + + false + + org.geonetwork-opensource @@ -97,4 +101,4 @@ - \ No newline at end of file + diff --git a/geonetwork/src/main/resources/application.properties b/geonetwork/src/main/resources/application.properties index 00a396b..7cb1cc4 100644 --- a/geonetwork/src/main/resources/application.properties +++ b/geonetwork/src/main/resources/application.properties @@ -2,4 +2,4 @@ aodn.geonetwork4.esIndexer.protocol=http aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:localhost} aodn.geonetwork4.esIndexer.port=${INDEXER_PORT:80} aodn.geonetwork4.esIndexer.apikey=${INDEXER_APIKEY} -aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.protocol}://${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid} \ No newline at end of file +aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.protocol}://${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid} diff --git a/pom.xml b/pom.xml index 16c8ede..05de931 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ au.org.aodn geonetwork4 pom - 1.0-SNAPSHOT + 0.0.0 geonetwork-api geonetwork @@ -26,11 +26,14 @@ 11 11 + true + 4.2.1 3.0.46 2.2.15 2.0.1.Final 1.12.261 + true @@ -79,4 +82,17 @@ - \ No newline at end of file + + + + codeartifact + codeartifact + ${env.CODEARTIFACT_REPO_URL} + + + codeartifact + codeartifact + ${env.CODEARTIFACT_REPO_URL} + + + diff --git a/tests/README-tests.md b/tests/README-tests.md new file mode 100644 index 0000000..c850d8e --- /dev/null +++ b/tests/README-tests.md @@ -0,0 +1,17 @@ +# Docker Image Structural Testing +The configuration file `test/config.yaml` defines various "structural" checks to perform on the docker image created by `../Dockerfile`. + +The test require the use of the [container-structure-test](https://github.com/GoogleContainerTools/container-structure-test) from Google. + +Follow their documentation to install. + +## Usage +Generate your docker image using your preferred method e.g. +```shell +docker build -t myimage . +``` + +To run the tests against the image, run the following: +```shell +container-structure-test test --image myimage --config tests/config.yaml +``` diff --git a/tests/config.yaml b/tests/config.yaml new file mode 100644 index 0000000..7981f83 --- /dev/null +++ b/tests/config.yaml @@ -0,0 +1,10 @@ +schemaVersion: 2.0.0 + +commandTests: + - name: "say hello world" + command: "bash" + args: + - -c + - | + echo hello && + echo world