Merge pull request #105 from Cobollatin/feature/batch

Feature/batch
Cobollatin · May 26, 2024 · 9bfd5b9 · 9bfd5b9
2 parents 95ea964 + ba0f456
commit 9bfd5b9
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 10 deletions.
diff --git a/.github/workflows/batch_pipeline.yaml b/.github/workflows/batch_pipeline.yaml
@@ -23,6 +23,14 @@ env:
     IMAGE_NAME: "indexer"
     IMAGE_TAG: ${{ github.sha }}
 jobs:
+    pre_job:
+        runs-on: ubuntu-latest
+        if: github.event_name == 'workflow_dispatch' && github.event.ref != 'refs/heads/main'
+        steps:
+            - name: Invalid branch
+              run: |
+                  echo "Invalid branch"
+                  exit 1
     indexer_ci:
         runs-on: ubuntu-latest
         if: github.event_name == 'pull_request' && github.event.action != 'closed'
@@ -33,6 +41,7 @@ jobs:
     buildImage:
         runs-on: ubuntu-latest
         if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.ref == 'refs/heads/main')
+        needs: pre_job
         steps:
             - uses: actions/[email protected]
             - uses: azure/[email protected]

diff --git a/.github/workflows/tf-drift.yml b/.github/workflows/tf-drift.yml
@@ -25,6 +25,7 @@ jobs:
             TF_VAR_azure_backend_rg: ${{ secrets.AZURE_TF_RESOURCE_GROUP }}
             TF_VAR_azure_backend_sa: ${{ secrets.AZURE_TF_STORAGE_ACCOUNT_NAME }}
             TF_VAR_azure_backend_container: ${{ secrets.AZURE_TF_CONTAINER_NAME }}
+            TF_GITHUB_TOKEN: ${{ secrets.TF_GITHUB_TOKEN }}
             GITHUB_TOKEN: ${{ secrets.TF_GITHUB_TOKEN }}
             GITHUB_OWNER: ${{ github.repository_owner }}
         steps:

diff --git a/.github/workflows/tf-pipeline.yaml b/.github/workflows/tf-pipeline.yaml
@@ -89,7 +89,7 @@ jobs:
             - name: Terraform Plan
               id: tplan
               run: |
-                  plan_output=$(terraform plan -no-color)
+                  plan_output=$(terraform plan -no-color -var "github_token=${{ secrets.GITHUB_TOKEN }}")
                   echo "$plan_output"
                   echo "plan<<EOF" >> $GITHUB_OUTPUT
                   echo "$plan_output" >> $GITHUB_OUTPUT
@@ -99,7 +99,7 @@ jobs:
               if: github.event_name == 'pull_request'
               run: |
                   echo Disable until we have a self-hosted runner
-                  # terraform plan --out plan.tfplan
+                  # terraform plan --out plan.tfplan -var "github_token=${{ secrets.GITHUB_TOKEN }}"
                   # terraform show -json plan.tfplan > tfplan.json
                   # ls
                   # checkov -f tfplan.json --framework terraform_plan
@@ -140,4 +140,4 @@ jobs:
             - name: Terraform Apply
               id: apply
               if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-              run: terraform apply -auto-approve
+              run: terraform apply -auto-approve -var "github_token=${{ secrets.GITHUB_TOKEN }}"
diff --git a/infrastructure/README.md b/infrastructure/README.md
@@ -79,6 +79,7 @@ No modules.
 | [github_actions_public_key.use2_main_acr_github_key](https://registry.terraform.io/providers/integrations/github/latest/docs/data-sources/actions_public_key) | data source |
 | [github_actions_public_key.use2_main_batch_github_key](https://registry.terraform.io/providers/integrations/github/latest/docs/data-sources/actions_public_key) | data source |
 | [github_actions_public_key.use2_main_swa_github_key](https://registry.terraform.io/providers/integrations/github/latest/docs/data-sources/actions_public_key) | data source |
+| [github_repository.use2_acr_github_repos](https://registry.terraform.io/providers/integrations/github/latest/docs/data-sources/repository) | data source |
 
 ## Inputs
 
@@ -90,6 +91,7 @@ No modules.
 | <a name="input_batch_repositories"></a> [batch\_repositories](#input\_batch\_repositories) | The list of repositories with Docker images | `list(string)` | n/a | yes |
 | <a name="input_common_tags"></a> [common\_tags](#input\_common\_tags) | Common tags for all resources | `map(string)` | n/a | yes |
 | <a name="input_environment_name"></a> [environment\_name](#input\_environment\_name) | The name of the environment (e.g., development, staging, production) | `string` | n/a | yes |
+| <a name="input_github_token"></a> [github\_token](#input\_github\_token) | The GitHub token for the GitHub Actions | `string` | n/a | yes |
 | <a name="input_location"></a> [location](#input\_location) | The Azure region where resources will be created | `string` | n/a | yes |
 | <a name="input_location_short"></a> [location\_short](#input\_location\_short) | The short name of the Azure region where resources will be created | `string` | n/a | yes |
 | <a name="input_swa_repository"></a> [swa\_repository](#input\_swa\_repository) | The GitHub repository for the Static Web App | `string` | n/a | yes |

diff --git a/infrastructure/main.tf b/infrastructure/main.tf
@@ -12,6 +12,11 @@ data "azuread_service_principal" "current" {
   client_id = data.azurerm_client_config.current.client_id
 }
 
+data "github_repository" "use2_acr_github_repos" {
+  for_each = toset(var.acr_repositories)
+  name     = each.value
+}
+
 ############################################################################################################################
 # Networking
 resource "azurerm_virtual_network" "use2_main_vnet" {
@@ -131,14 +136,32 @@ resource "azurerm_container_registry_task" "use2_main_acr_indexer_purge_task" {
     task_content = <<EOF
 version: v1.1.0
 steps: 
-  - cmd: acr purge acr purge --filter 'indexer:.*' --keep 10 --untagged
+  - cmd: |
+      PURGE_CMD="acr purge --ago 1d --untagged --keep 5"
+      for i in $(az acr repository list -n ${azurerm_container_registry.use2_main_acr.name} -o tsv);do PURGE_CMD+=" --filter '"$i":.'";done
+      echo $PURGE_CMD
     disableWorkingDirectoryOverride: true
     timeout: 3600
 EOF
   }
   platform {
     os = "Linux"
   }
+  dynamic "source_trigger" {
+    for_each = toset(var.acr_repositories)
+    content {
+      name           = "GithubTrigger"
+      source_type    = "Github"
+      repository_url = data.github_repository.use2_acr_github_repos[source_trigger.key].html_url
+      events         = ["commit"]
+      branch         = "main"
+      authentication {
+        token      = var.github_token
+        token_type = "PAT"
+      }
+    }
+
+  }
   timer_trigger {
     name     = "PurgeTimer"
     schedule = "0 */6 * * *" # Every 6 hours
@@ -362,7 +385,6 @@ resource "azurerm_key_vault_key" "use2_main_sa_kv_key" {
     automatic {
       time_before_expiry = "P30D"
     }
-
     expire_after         = "P90D"
     notify_before_expiry = "P29D"
   }
@@ -590,16 +612,17 @@ resource "azurerm_subnet_network_security_group_association" "use2_bp_subnet_nsg
 }
 
 resource "azurerm_batch_pool" "use2_main_batch_pool" {
-  name                           = "${var.app_name}-${var.location_short}-${var.environment_name}-batch-pool-v1"
+  name                           = "${var.app_name}-${var.location_short}-${var.environment_name}-batch-pool"
   resource_group_name            = azurerm_resource_group.use2_main_rg.name
   account_name                   = azurerm_batch_account.use2_main_batch.name
   node_agent_sku_id              = "batch.node.ubuntu 20.04"
-  vm_size                        = "Standard_A1_V2"
+  vm_size                        = "Standard_B1s" # Standard_A1_V2
   metadata                       = var.common_tags
   max_tasks_per_node             = 1
-  os_disk_placement              = "CacheDisk"
   inter_node_communication       = "Disabled"
   target_node_communication_mode = "Default"
+  # Ephemeral OS disk is not supported for VM size Standard_A1_v2.
+  # os_disk_placement              = "CacheDisk"
   storage_image_reference {
     publisher = "microsoft-azure-batch"
     offer     = "ubuntu-server-container"
@@ -620,14 +643,15 @@ EOF
   }
   data_disks {
     lun                  = 0
-    disk_size_gb         = 10
+    disk_size_gb         = 4
     storage_account_type = "Standard_LRS"
+    caching              = "None"
   }
   mount {
     azure_blob_file_system {
       account_name        = azurerm_storage_account.use2_main_sa.name
       container_name      = azurerm_storage_container.use2_main_batch_container.name
-      relative_mount_path = "ba tch"
+      relative_mount_path = "batch"
       identity_id         = azurerm_user_assigned_identity.use2_main_batch_identity.id
       blobfuse_options    = "--log-level=LOG_INFO"
     }

diff --git a/infrastructure/variables.tf b/infrastructure/variables.tf
@@ -53,3 +53,9 @@ variable "batch_docker_images" {
   description = "The list of Docker images for the batch job" # Specifies a list of Docker images that are used for the batch job.
   type        = list(string)                                  # Indicates that this variable is a list of strings.
 }
+
+variable "github_token" {
+  description = "The GitHub token for the GitHub Actions" # Specifies the GitHub token used for GitHub Actions.
+  type        = string                                    # The value of this variable is a string.
+  sensitive   = true                                      # Indicates that the value of this variable should be treated as sensitive and not displayed in logs or outputs.
+}