5320 devops cicd

.github/environment/README.md

@@ -0,0 +1,13 @@
+## Github Deployment Environments
+Github deployment environments are used to define unique settings for each environment i.e. staging and production
+
+The build and push workflows need to know which AWS account to push updated docker images to.
+
+### DotEnv Files
+The .env files in this directory are here as a record of the "variables" and their values.
+
+The variables can be updated from these files using the following command:
+```bash
+gh variable set -R aodn/<repo name> -e <environment name> -f <environment>.env
+
+```

.github/environment/production.env

@@ -0,0 +1,5 @@
+AWS_REGION=ap-southeast-2
+CODEARTIFACT_DOMAIN=gamma-aodn-org-au
+CODEARTIFACT_REPO=maven-aodn-store
+ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
+ECR_REPOSITORY=geonetwork4

.github/environment/staging.env

@@ -0,0 +1,5 @@
+AWS_REGION=ap-southeast-2
+CODEARTIFACT_DOMAIN=gamma-aodn-org-au
+CODEARTIFACT_REPO=maven-aodn-store
+ECR_REGISTRY=450356697252.dkr.ecr.ap-southeast-2.amazonaws.com
+ECR_REPOSITORY=geonetwork4

.github/workflows/build-production.yml

@@ -0,0 +1,130 @@
+name: Build Production
+
+on:
+  push:
+    tags:
+      - v*.*.*
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  environment_name: production
+
+jobs:
+  build_push:
+    runs-on: ubuntu-latest
+    environment: production
+    outputs:
+      image_digest: ${{ steps.build_and_push.outputs.digest }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+          server-id: 'codeartifact'
+          server-password: 'CODEARTIFACT_AUTH_TOKEN'
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+
+      - name: Get CodeArtifact Repository Authentication Token
+        run: |
+          TOKEN=$(aws codeartifact get-authorization-token \
+            --domain ${{ vars.CODEARTIFACT_DOMAIN }} \
+            --domain-owner ${{ steps.aws_auth.outputs.aws-account-id }} \
+            --region ${{ vars.AWS_REGION }} \
+            --query authorizationToken \
+            --output text)
+          echo "CODEARTIFACT_AUTH_TOKEN=$TOKEN" >> "$GITHUB_ENV"
+
+      - name: Get CodeArtifact Repository URL
+        run: |
+          REPO_URL=$(aws codeartifact get-repository-endpoint \
+            --domain ${{ vars.CODEARTIFACT_DOMAIN }} \
+            --repository ${{ vars.CODEARTIFACT_REPO }} \
+            --format maven \
+            --region ${{ vars.AWS_REGION }} \
+            --output text)
+          echo "CODEARTIFACT_REPO_URL=$REPO_URL" >> "$GITHUB_ENV"
+
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Publish Artifacts to CodeArtifact Repository
+        run: mvn -B deploy
+        env:
+          CODEARTIFACT_AUTH_TOKEN: ${{ env.CODEARTIFACT_AUTH_TOKEN }}
+          CODEARTIFACT_REPO_URL: ${{ env.CODEARTIFACT_REPO_URL }}
+
+      - name: Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.ECR_REGISTRY }}
+
+      - name: Build and Push Docker Image
+        id: build_and_push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          #          Only building for AMD64 for now
+          #          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.ref_name }}
+
+      - name: Push Image Digest to SSM
+        run: |
+          aws ssm put-parameter \
+            --name "/apps/sample-django-app/${{ env.environment_name }}/image_digest" \
+            --type "String" \
+            --value "$digest" \
+            --overwrite
+        env:
+          digest: ${{ steps.build_and_push.outputs.digest }}
+
+  trigger_deploy:
+    runs-on: ubuntu-latest
+    needs: [build_push]
+    steps:
+      - name: Generate App Token
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.DEPLOY_APP_ID }}
+          private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "appdeploy"
+
+      - name: Trigger Deploy Workflow
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ steps.app-token.outputs.token }}
+          retries: 3
+          retry-exempt-status-codes: 204
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: 'aodn',
+              repo: 'appdeploy',
+              workflow_id: 'deploy.yml',
+              ref: 'main',
+              inputs: {
+                app_name: 'geonetwork4',
+                environment: '${{ env.environment_name }}'
+              }
+            })

.github/workflows/build-staging.yml

@@ -0,0 +1,128 @@
+name: Build Staging
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**/*.md'
+      - '.github/environment/**'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  environment_name: staging
+
+jobs:
+  build_push:
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+          server-id: 'codeartifact'
+          server-password: 'CODEARTIFACT_AUTH_TOKEN'
+
+      - name: Configure AWS Credentials
+        id: aws_auth
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          audience: sts.amazonaws.com
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+
+      - name: Get CodeArtifact Repository Authentication Token
+        run: |
+          TOKEN=$(aws codeartifact get-authorization-token \
+            --domain ${{ vars.CODEARTIFACT_DOMAIN }} \
+            --domain-owner ${{ steps.aws_auth.outputs.aws-account-id }} \
+            --region ${{ vars.AWS_REGION }} \
+            --query authorizationToken \
+            --output text)
+          echo "CODEARTIFACT_AUTH_TOKEN=$TOKEN" >> "$GITHUB_ENV"
+
+      - name: Get CodeArtifact Repository URL
+        run: |
+          REPO_URL=$(aws codeartifact get-repository-endpoint \
+            --domain ${{ vars.CODEARTIFACT_DOMAIN }} \
+            --repository ${{ vars.CODEARTIFACT_REPO }} \
+            --format maven \
+            --region ${{ vars.AWS_REGION }} \
+            --output text)
+          echo "CODEARTIFACT_REPO_URL=$REPO_URL" >> "$GITHUB_ENV"
+
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.ECR_REGISTRY }}
+
+      - name: Build and Push Docker Image
+        id: build_and_push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          #          Only building for AMD64 for now
+          #          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:${{ github.sha }}
+            ${{ vars.ECR_REGISTRY }}/${{ vars.ECR_REPOSITORY }}:latest
+
+      - name: Push Image Digest to SSM
+        run: |
+          aws ssm put-parameter \
+            --name "/apps/geonetwork4/${{ env.environment_name }}/image_digest" \
+            --type "String" \
+            --value "$digest" \
+            --overwrite
+        env:
+          digest: ${{ steps.build_and_push.outputs.digest }}
+
+  trigger_deploy:
+    runs-on: ubuntu-latest
+    needs: [build_push]
+    steps:
+      - name: Generate App Token
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ vars.DEPLOY_APP_ID }}
+          private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "appdeploy"
+
+      - name: Trigger Deploy Workflow
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ steps.app-token.outputs.token }}
+          retries: 3
+          retry-exempt-status-codes: 204
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: 'aodn',
+              repo: 'appdeploy',
+              workflow_id: 'deploy.yml',
+              ref: 'main',
+              inputs: {
+                app_name: 'geonetwork4',
+                environment: '${{ env.environment_name }}'
+              }
+            })

.github/workflows/pre-commit.yml

@@ -0,0 +1,20 @@
+name: Run Pre-commit Checks
+
+on:
+  pull_request:
+    branches:
+      - master
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  pre_commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - uses: pre-commit/[email protected]

.github/workflows/test.yml

@@ -0,0 +1,69 @@
+name: Test
+
+on:
+  pull_request:
+    branches:
+      - master
+      - main
+    paths-ignore:
+      - '**/*.md'
+      - '.github/environment/**'
+
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build_test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Setup Docker Structure Test
+        run: >
+          curl -LO
+          https://storage.googleapis.com/container-structure-test/latest/container-structure-test-linux-amd64
+          && chmod +x container-structure-test-linux-amd64 && sudo mv container-structure-test-linux-amd64
+          /usr/local/bin/container-structure-test
+
+      - name: Set Image Tag
+        id: set_image_tag
+        run: |
+          branch_name=${{ github.head_ref || github.ref_name }}
+          tag=${{ env.TAG_PREFIX}}-${branch_name//\//-}
+          echo "$tag"
+          echo "image_tag=$tag" >> $GITHUB_OUTPUT
+        env:
+          TAG_PREFIX: test
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Build Docker Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          tags: image:${{ github.sha }}
+
+      - name: Test Docker Image
+        run: |
+          container-structure-test test --image image:${{ github.sha }} --config tests/config.yaml

.gitignore

@@ -26,9 +26,10 @@ replay_pid*
 # Ingore local generated folders
 elasticdata/
 gn4_data/
-src/main/generated/
 .env
 
 .idea/
 
-**/target/
+**/target/
+
+**/.git-versioned-pom.xml