Devops cicd

.github/environment/README.md

@@ -0,0 +1,13 @@
+## Github Deployment Environments
+Github deployment environments are used to define unique settings for each environment i.e. staging and production
+
+The build and push workflows need to know which AWS account to push updated docker images to.
+
+### DotEnv Files
+The .env files in this directory are here as a record of the "variables" and their values.
+
+The variables can be updated from these files using the following command:
+```bash
+gh variable set -R aodn/<repo name> -e <environment name> -f <environment>.env
+
+```

.github/environment/production.env

@@ -0,0 +1,3 @@
+AWS_REGION=ap-southeast-2
+ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
+ECR_REPOSITORY=geonetwork4

.github/environment/staging.env

@@ -0,0 +1,3 @@
+AWS_REGION=ap-southeast-2
+ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
+ECR_REPOSITORY=geonetwork4

.github/workflows/build-production.yml

@@ -0,0 +1,102 @@
+name: Build Production
+
+on:
+  push:
+    tags:
+      - v*.*.*
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  environment_name: production
+
+jobs:
+  build_push:
+    runs-on: ubuntu-latest
+    environment: production
+    outputs:
+      image_digest: ${{ steps.build_and_push.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+
+      - name: Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.ECR_REGISTRY }}
+
+      - name: Build and Push Docker Image
+        id: build_and_push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          #          Only building for AMD64 for now
+          #          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.ref_name }}
+
+      - name: Push Image Digest to SSM
+        run: |
+          aws ssm put-parameter \
+            --name "/apps/sample-django-app/${{ env.environment_name }}/image_digest" \
+            --type "String" \
+            --value "$digest" \
+            --overwrite
+        env:
+          digest: ${{ steps.build_and_push.outputs.digest }}
+
+  trigger_deploy:
+    runs-on: ubuntu-latest
+    needs: [build_push]
+    steps:
+      - name: Generate App Token
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.DEPLOY_APP_ID }}
+          private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "appdeploy"
+
+      - name: Trigger Deploy Workflow
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ steps.app-token.outputs.token }}
+          retries: 3
+          retry-exempt-status-codes: 204
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: 'aodn',
+              repo: 'appdeploy',
+              workflow_id: 'deploy.yml',
+              ref: 'main',
+              inputs: {
+                app_name: 'geonetwork4',
+                environment: '${{ env.environment_name }}'
+              }
+            })

.github/workflows/build-staging.yml

@@ -0,0 +1,113 @@
+name: Build Staging
+
+on:
+  push:
+    branches:
+      - master
+      - main
+    paths-ignore:
+      - '**/*.md'
+      - '.github/environment/**'
+  # This is only here for testing purposes while in a PR
+  #TODO remove before merge to main
+  pull_request:
+    branches:
+      - master
+      - main
+    paths-ignore:
+      - '**/*.md'
+      - '.github/environment/**'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  environment_name: staging
+
+jobs:
+  build_push:
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+
+      - name: Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.ECR_REGISTRY }}
+
+      - name: Build and Push Docker Image
+        id: build_and_push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          #          Only building for AMD64 for now
+          #          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.sha }}
+
+      - name: Push Image Digest to SSM
+        run: |
+          aws ssm put-parameter \
+            --name "/apps/geonetwork4/${{ env.environment_name }}/image_digest" \
+            --type "String" \
+            --value "$digest" \
+            --overwrite
+        env:
+          digest: ${{ steps.build_and_push.outputs.digest }}
+
+  trigger_deploy:
+    runs-on: ubuntu-latest
+    needs: [build_push]
+    steps:
+      - name: Generate App Token
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.DEPLOY_APP_ID }}
+          private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "appdeploy"
+
+      - name: Trigger Deploy Workflow
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ steps.app-token.outputs.token }}
+          retries: 3
+          retry-exempt-status-codes: 204
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: 'aodn',
+              repo: 'appdeploy',
+              workflow_id: 'deploy.yml',
+              ref: 'main',
+              inputs: {
+                app_name: 'geonetwork4',
+                environment: '${{ env.environment_name }}'
+              }
+            })

.github/workflows/pre-commit.yml

@@ -0,0 +1,20 @@
+name: Run Pre-commit Checks
+
+on:
+  pull_request:
+    branches:
+      - master
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  pre_commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - uses: pre-commit/[email protected]

.github/workflows/test.yml

@@ -0,0 +1,69 @@
+name: Test
+
+on:
+  pull_request:
+    branches:
+      - master
+      - main
+    paths-ignore:
+      - '**/*.md'
+      - '.github/environment/**'
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build_test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Setup Docker Structure Test
+        run: >
+          curl -LO
+          https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64
+          && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64
+          /usr/local/bin/container-structure-test
+
+      - name: Set Image Tag
+        id: set_image_tag
+        run: |
+          branch_name=${{ github.head_ref || github.ref_name }}
+          tag=${{ env.TAG_PREFIX}}-${branch_name//\//-}
+          echo "$tag"
+          echo "image_tag=$tag" >> $GITHUB_OUTPUT
+        env:
+          TAG_PREFIX: test
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Build Docker Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          tags: image:${{ github.sha }}
+
+      - name: Test Docker Image
+        run: |
+          container-structure-test test --image image:${{ github.sha }} --config tests/config.yaml

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      # Security
+      - id: detect-private-key
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.18.0
+    hooks:
+      - id: gitleaks

README.md

@@ -4,7 +4,7 @@ This repo is used to customize a GeoNetwork4 to be used by AODN. The key customi
 build a jar file with the code we want to add and insert it use COPY function to copy it to the lib
 folder during the Dockerfile build.
 
-The jar file contains a hack where we name a @Configuration class with the same package as 
+The jar file contains a hack where we name a @Configuration class with the same package as
 geonetwork4 base package (org.fao.geonet), so that the initial component-scan will pick up this
 class. From there we add additional component-scan to our custom classes. This avoided the need
 to alter the xml like what we did before plus we are using a Docker base image of GeoNetwork4.
@@ -29,6 +29,6 @@ Once you have the json, you can generate code like the one here in Java to acces
 
 ## Use of S3
 
-You can see a config file related to S3, however we do not use it because after experiment it, it 
+You can see a config file related to S3, however we do not use it because after experiment it, it
 didn't support well as the GN4 will issue warning on file not found with relative folder name. The
-code is just keep as a record. 
+code is just keep as a record.

pom.xml

@@ -179,4 +179,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
+</project>

src/main/resources/application.properties

@@ -2,4 +2,4 @@ aodn.geonetwork4.esIndexer.protocol=http
 aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:localhost}
 aodn.geonetwork4.esIndexer.port=${INDEXER_PORT:80}
 aodn.geonetwork4.esIndexer.apikey=${INDEXER_APIKEY}
-aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.protocol}://${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid}
+aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.protocol}://${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid}

src/main/resources/schema/gn4-api.json

@@ -22473,4 +22473,4 @@
       }
     }
   }
-}
+}

startEsLocal.sh

@@ -31,4 +31,3 @@ fi
 # If you run in EC2, you need to change the docker-es-compose.yml localhost in cert to the hostname of EC2 and install docker
 # docker-compose and sudo chmod 666 /run/docker.sock
 docker-compose -f docker-es-compose.yml up --detach
-