Merge branch 'main' of https://github.com/microsoft/AzureTRE into cor…

…e-network-subnets-change

.devcontainer/devcontainer.json

@@ -29,7 +29,7 @@
     // Mount docker socket for docker builds
     "type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock",
     // Mounts the github cli login details from the host machine to the container (~/.config/gh/hosts.yml)
-    "type=bind,source=${env:HOME}${env:USERPROFILE}/.config,target=/home/vscode/.config",
+    "type=bind,source=${env:HOME}${env:USERPROFILE}/.config,target=/home/vscode/.config"
   ],
   "remoteUser": "vscode",
   "containerEnv": {
@@ -277,6 +277,8 @@
         "ms-python.pylance",
         "hashicorp.terraform",
         "github.vscode-pull-request-github",
+        "gitHub.copilot",
+        "github.copilot-chat",
         "getporter.porter-vscode",
         "davidanson.vscode-markdownlint",
         "editorconfig.editorconfig",
@@ -291,5 +293,6 @@
     8000
   ],
   // Run commands after the container is created.
-  "postCreateCommand": "./.devcontainer/scripts/post-create.sh"
+  "postCreateCommand": "./.devcontainer/scripts/post-create.sh",
+  "initializeCommand": ["./.devcontainer/scripts/initialize"]
 }

.devcontainer/scripts/initialize

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+mkdir -p "$HOME/.azure" "$HOME/.config" || true

.devcontainer/scripts/initialize.cmd

@@ -0,0 +1,2 @@
+@echo off
+mkdir %USERPROFILE%\.azure %USERPROFILE%\.config || exit /b 0

.github/workflows/deploy_tre_reusable.yml

@@ -863,3 +863,4 @@ jobs:
         with:
           junit_files: "artifacts/**/*.xml"
           check_name: "E2E Test Results"
+          comment_mode: off

CHANGELOG.md

@@ -37,6 +37,9 @@ ENHANCEMENTS:
 * Allow enablement of Secure Boot and vTPM for Guacamole VMs ([#4235](https://github.com/microsoft/AzureTRE/issues/4235))
 * Surface the server-layout parameter of Guacamole [server-layout](https://guacamole.apache.org/doc/gug/configuring-guacamole.html#session-settings) ([#4234](https://github.com/microsoft/AzureTRE/issues/4234))
 * Add encryption at host for VMs ([#4263](https://github.com/microsoft/AzureTRE/pull/4263))
+* Downgrade certs shared service App Gateway to Basic SKU ([#4300](https://github.com/microsoft/AzureTRE/issues/4300))
+* Airlock function host storage to use the user-assigned managed identity ([#4276](https://github.com/microsoft/AzureTRE/issues/4276))
+* Disable local authentication in EventGrid ([#4254](https://github.com/microsoft/AzureTRE/issues/4254))
 
 BUG FIXES:
 * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112))
@@ -54,6 +57,8 @@ BUG FIXES:
 * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222))
 * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)])
 * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)])
+* Windows R version must be 4.1.2 otherwise post install script doesn't update package mirror URL ([#4288](https://github.com/microsoft/AzureTRE/issues/4288))
+* Recreate tre_output.json if empty. ([[#4292](https://github.com/microsoft/AzureTRE/issues/4292)])
 * Upgrade AzureRM Terraform provider from `3.117.0` to `4.14.0`. ([[PR_link](https://github.com/microsoft/AzureTRE/pull/4255/)])
 * Subnet definitions are now inline in the `azurerm_virtual_network` resource, and NSG associations are set using `security_group` in each subnet block (no separate `azurerm_subnet_network_security_group_association` needed). ([[PR_link](https://github.com/microsoft/AzureTRE/pull/4255/)])
 

airlock_processor/BlobCreatedTrigger/function.json

@@ -13,15 +13,13 @@
     {
       "type": "eventGrid",
       "name": "stepResultEvent",
-      "topicEndpointUri": "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING",
-      "topicKeySetting": "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING",
+      "connection": "EVENT_GRID_STEP_RESULT_CONNECTION",
       "direction": "out"
     },
     {
       "type": "eventGrid",
       "name": "dataDeletionEvent",
-      "topicEndpointUri": "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING",
-      "topicKeySetting": "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING",
+      "connection": "EVENT_GRID_DATA_DELETION_CONNECTION",
       "direction": "out"
     }
   ]

airlock_processor/ScanResultTrigger/function.json

@@ -12,8 +12,7 @@
     {
       "type": "eventGrid",
       "name": "outputEvent",
-      "topicEndpointUri": "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING",
-      "topicKeySetting": "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING",
+      "connection": "EVENT_GRID_STEP_RESULT_CONNECTION",
       "direction": "out"
     }
   ]

airlock_processor/StatusChangedQueueTrigger/function.json

@@ -11,15 +11,13 @@
     {
       "type": "eventGrid",
       "name": "stepResultEvent",
-      "topicEndpointUri": "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING",
-      "topicKeySetting": "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING",
+      "connection": "EVENT_GRID_STEP_RESULT_CONNECTION",
       "direction": "out"
     },
     {
       "type": "eventGrid",
       "name": "dataDeletionEvent",
-      "topicEndpointUri": "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING",
-      "topicKeySetting": "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING",
+      "connection": "EVENT_GRID_DATA_DELETION_CONNECTION",
       "direction": "out"
     }
   ]

airlock_processor/_version.py

@@ -1 +1 @@
-__version__ = "0.8.0"
+__version__ = "0.8.1"

airlock_processor/run_tests_and_exit_succesfully.sh

@@ -6,6 +6,6 @@
 rm -f ../test-results/pytest_airlock_processor*
 mkdir -p ../test-results
 
-if ! pytest --junit-xml ../test-results/pytest_airlock_processor_unit.xml --ignore e2e_tests; then
+if ! python -m pytest --junit-xml ../test-results/pytest_airlock_processor_unit.xml --ignore e2e_tests; then
   touch ../test-results/pytest_airlock_processor_unit_failed
 fi

api_app/_version.py

@@ -1 +1 @@
-__version__ = "0.20.3"
+__version__ = "0.20.4"

api_app/run_tests_and_exit_succesfully.sh

@@ -6,6 +6,6 @@
 rm -f ../test-results/pytest_api*
 mkdir -p ../test-results
 
-if ! pytest --junit-xml ../test-results/pytest_api_unit.xml --ignore e2e_tests -W ignore::pytest.PytestUnraisableExceptionWarning -W ignore::DeprecationWarning; then
+if ! python -m pytest --junit-xml ../test-results/pytest_api_unit.xml --ignore e2e_tests -W ignore::pytest.PytestUnraisableExceptionWarning -W ignore::DeprecationWarning; then
   touch ../test-results/pytest_api_unit_failed
 fi

core/terraform/airlock/airlock_processor.tf

@@ -21,9 +21,8 @@ resource "azurerm_storage_account" "sa_airlock_processor_func_app" {
   allow_nested_items_to_be_public  = false
   cross_tenant_replication_enabled = false
   local_user_enabled               = false
-  # Function Host Storage doesn't seem to be able to use a User Managed ID, which is why we continue to use a key.
-  shared_access_key_enabled = true
-  tags                      = var.tre_core_tags
+  shared_access_key_enabled        = false
+  tags                             = var.tre_core_tags
 
   dynamic "identity" {
     for_each = var.enable_cmk_encryption ? [1] : []
@@ -57,9 +56,7 @@ resource "azurerm_linux_function_app" "airlock_function_app" {
   ftp_publish_basic_authentication_enabled       = false
   webdeploy_publish_basic_authentication_enabled = false
   storage_account_name                           = azurerm_storage_account.sa_airlock_processor_func_app.name
-
-  # Function Host Storage doesn't seem to be able to use a User Managed ID, which is why we continue to use a key.
-  storage_account_access_key = azurerm_storage_account.sa_airlock_processor_func_app.primary_access_key
+  storage_uses_managed_identity                  = true
 
   tags = var.tre_core_tags
 
@@ -69,23 +66,31 @@ resource "azurerm_linux_function_app" "airlock_function_app" {
   }
 
   app_settings = {
-    "SB_CONNECTION_STRING"                       = var.airlock_servicebus.default_primary_connection_string
-    "BLOB_CREATED_TOPIC_NAME"                    = azurerm_servicebus_topic.blob_created.name
-    "TOPIC_SUBSCRIPTION_NAME"                    = azurerm_servicebus_subscription.airlock_processor.name
-    "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING"   = azurerm_eventgrid_topic.step_result.endpoint
-    "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING"   = azurerm_eventgrid_topic.step_result.primary_access_key
-    "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.data_deletion.endpoint
-    "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.data_deletion.primary_access_key
-    "WEBSITES_ENABLE_APP_SERVICE_STORAGE"        = false
-    "AIRLOCK_STATUS_CHANGED_QUEUE_NAME"          = local.status_changed_queue_name
-    "AIRLOCK_SCAN_RESULT_QUEUE_NAME"             = local.scan_result_queue_name
-    "AIRLOCK_DATA_DELETION_QUEUE_NAME"           = local.data_deletion_queue_name
-    "ENABLE_MALWARE_SCANNING"                    = var.enable_malware_scanning
-    "ARM_ENVIRONMENT"                            = var.arm_environment
-    "MANAGED_IDENTITY_CLIENT_ID"                 = azurerm_user_assigned_identity.airlock_id.client_id
-    "TRE_ID"                                     = var.tre_id
-    "WEBSITE_CONTENTOVERVNET"                    = 1
-    "STORAGE_ENDPOINT_SUFFIX"                    = module.terraform_azurerm_environment_configuration.storage_suffix
+    "SB_CONNECTION_STRING"                = var.airlock_servicebus.default_primary_connection_string
+    "BLOB_CREATED_TOPIC_NAME"             = azurerm_servicebus_topic.blob_created.name
+    "TOPIC_SUBSCRIPTION_NAME"             = azurerm_servicebus_subscription.airlock_processor.name
+    "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false
+    "AIRLOCK_STATUS_CHANGED_QUEUE_NAME"   = local.status_changed_queue_name
+    "AIRLOCK_SCAN_RESULT_QUEUE_NAME"      = local.scan_result_queue_name
+    "AIRLOCK_DATA_DELETION_QUEUE_NAME"    = local.data_deletion_queue_name
+    "ENABLE_MALWARE_SCANNING"             = var.enable_malware_scanning
+    "ARM_ENVIRONMENT"                     = var.arm_environment
+    "MANAGED_IDENTITY_CLIENT_ID"          = azurerm_user_assigned_identity.airlock_id.client_id
+    "TRE_ID"                              = var.tre_id
+    "WEBSITE_CONTENTOVERVNET"             = 1
+    "STORAGE_ENDPOINT_SUFFIX"             = module.terraform_azurerm_environment_configuration.storage_suffix
+    "AzureWebJobsStorage__clientId"       = azurerm_user_assigned_identity.airlock_id.client_id
+    "AzureWebJobsStorage__credential"     = "managedidentity"
+
+    "EVENT_GRID_STEP_RESULT_CONNECTION"                           = local.step_result_eventgrid_connection
+    "${local.step_result_eventgrid_connection}__topicEndpointUri" = azurerm_eventgrid_topic.step_result.endpoint
+    "${local.step_result_eventgrid_connection}__credential"       = "managedidentity"
+    "${local.step_result_eventgrid_connection}__clientId"         = azurerm_user_assigned_identity.airlock_id.client_id
+
+    "EVENT_GRID_DATA_DELETION_CONNECTION"                           = local.data_deletion_eventgrid_connection
+    "${local.data_deletion_eventgrid_connection}__topicEndpointUri" = azurerm_eventgrid_topic.data_deletion.endpoint
+    "${local.data_deletion_eventgrid_connection}__credential"       = "managedidentity"
+    "${local.data_deletion_eventgrid_connection}__clientId"         = azurerm_user_assigned_identity.airlock_id.client_id
   }
 
   site_config {

core/terraform/airlock/eventgrid_topics.tf

@@ -6,6 +6,7 @@ resource "azurerm_eventgrid_topic" "step_result" {
   location                      = var.location
   resource_group_name           = var.resource_group_name
   public_network_access_enabled = var.enable_local_debugging
+  local_auth_enabled            = false
 
   identity {
     type = "SystemAssigned"
@@ -60,6 +61,7 @@ resource "azurerm_eventgrid_topic" "status_changed" {
   location                      = var.location
   resource_group_name           = var.resource_group_name
   public_network_access_enabled = var.enable_local_debugging
+  local_auth_enabled            = false
 
   identity {
     type = "SystemAssigned"
@@ -113,6 +115,7 @@ resource "azurerm_eventgrid_topic" "data_deletion" {
   location                      = var.location
   resource_group_name           = var.resource_group_name
   public_network_access_enabled = var.enable_local_debugging
+  local_auth_enabled            = false
 
   identity {
     type = "SystemAssigned"
@@ -163,6 +166,7 @@ resource "azurerm_eventgrid_topic" "scan_result" {
   resource_group_name = var.resource_group_name
   # This is mandatory for the scan result to be published since private networks are not supported yet
   public_network_access_enabled = true
+  local_auth_enabled            = false
 
   identity {
     type = "SystemAssigned"
@@ -323,6 +327,7 @@ resource "azurerm_eventgrid_topic" "airlock_notification" {
   location                      = var.location
   resource_group_name           = var.resource_group_name
   public_network_access_enabled = var.enable_local_debugging
+  local_auth_enabled            = false
 
   identity {
     type = "SystemAssigned"

core/terraform/airlock/identity.tf

@@ -25,7 +25,7 @@ resource "azurerm_role_assignment" "servicebus_receiver" {
   principal_id         = azurerm_user_assigned_identity.airlock_id.principal_id
 }
 
-resource "azurerm_role_assignment" "eventgrid_data_sender" {
+resource "azurerm_role_assignment" "eventgrid_data_sender_status_changed" {
   scope                = azurerm_eventgrid_topic.status_changed.id
   role_definition_name = "EventGrid Data Sender"
   principal_id         = var.api_principal_id
@@ -37,6 +37,18 @@ resource "azurerm_role_assignment" "eventgrid_data_sender_notification" {
   principal_id         = var.api_principal_id
 }
 
+resource "azurerm_role_assignment" "eventgrid_data_sender_step_result" {
+  scope                = azurerm_eventgrid_topic.step_result.id
+  role_definition_name = "EventGrid Data Sender"
+  principal_id         = azurerm_user_assigned_identity.airlock_id.principal_id
+}
+
+resource "azurerm_role_assignment" "eventgrid_data_sender_data_deletion" {
+  scope                = azurerm_eventgrid_topic.data_deletion.id
+  role_definition_name = "EventGrid Data Sender"
+  principal_id         = azurerm_user_assigned_identity.airlock_id.principal_id
+}
+
 resource "azurerm_role_assignment" "airlock_blob_data_contributor" {
   count                = length(local.airlock_sa_blob_data_contributor)
   scope                = local.airlock_sa_blob_data_contributor[count.index]
@@ -52,3 +64,11 @@ resource "azurerm_role_assignment" "api_sa_data_contributor" {
   role_definition_name = "Storage Blob Data Contributor"
   principal_id         = var.api_principal_id
 }
+
+# Permissions needed for the Function Host to work correctly.
+resource "azurerm_role_assignment" "function_host_storage" {
+  for_each             = toset(["Storage Account Contributor", "Storage Blob Data Owner", "Storage Queue Data Contributor"])
+  scope                = azurerm_storage_account.sa_airlock_processor_func_app.id
+  role_definition_name = each.value
+  principal_id         = azurerm_user_assigned_identity.airlock_id.principal_id
+}

core/terraform/airlock/locals.tf

@@ -60,4 +60,7 @@ locals {
     azurerm_storage_account.sa_import_in_progress.id,
     azurerm_storage_account.sa_export_approved.id
   ]
+
+  step_result_eventgrid_connection   = "EVENT_GRID_STEP_RESULT_CONNECTION"
+  data_deletion_eventgrid_connection = "EVENT_GRID_DATA_DELETION_CONNECTION"
 }

core/terraform/outputs.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-if [ ! -f ../tre_output.json ]; then
+if [ ! -f ../tre_output.json ] || [ ! -s ../tre_output.json ]; then
   # Connect to the remote backend of Terraform
   export TF_LOG=""
   # shellcheck disable=SC2154

core/version.txt

@@ -1 +1 @@
-__version__ = "0.11.20"
+__version__ = "0.11.23"

e2e_tests/config.py

@@ -1,11 +1,9 @@
+import warnings
 from starlette.config import Config
 
+warnings.filterwarnings("ignore", message="Config file '.env' not found.")
 
-try:
-    config = Config('.env')
-# Workaround needed until FastAPI uses Starlette >= 3.7.1
-except FileNotFoundError:
-    config = Config()
+config = Config('.env')
 
 # Resource Info
 RESOURCE_LOCATION: str = config("RESOURCE_LOCATION", default="")

e2e_tests/conftest.py

@@ -12,23 +12,13 @@
 
 
 LOGGER = logging.getLogger(__name__)
-pytestmark = pytest.mark.asyncio
+pytestmark = pytest.mark.asyncio(loop_scope="session")
 
 
 def pytest_addoption(parser):
     parser.addoption("--verify", action="store", default="true")
 
 
-@pytest.fixture(scope="session")
-def event_loop():
-    try:
-        loop = asyncio.get_running_loop()
-    except RuntimeError:
-        loop = asyncio.new_event_loop()
-    yield loop
-    loop.close()
-
-
 @pytest.fixture(scope="session")
 def verify(pytestconfig):
     if pytestconfig.getoption("verify").lower() == "true":

e2e_tests/pytest.ini

@@ -10,6 +10,7 @@ markers =
     workspace_services
 
 asyncio_mode = auto
+asyncio_default_fixture_loop_scope = session
 
 log_cli = 1
 log_cli_level = INFO

e2e_tests/test_airlock.py

@@ -14,7 +14,7 @@
 from helpers import get_admin_token
 
 
-pytestmark = pytest.mark.asyncio
+pytestmark = pytest.mark.asyncio(loop_scope="session")
 LOGGER = logging.getLogger(__name__)
 BLOB_FILE_PATH = "./test_airlock_sample.txt"
 BLOB_NAME = os.path.basename(BLOB_FILE_PATH)

e2e_tests/test_performance.py

@@ -8,7 +8,7 @@
 
 from helpers import get_admin_token
 
-pytestmark = pytest.mark.asyncio
+pytestmark = pytest.mark.asyncio(loop_scope="session")
 
 
 @pytest.mark.performance

e2e_tests/test_provisioned_health_api.py

@@ -5,7 +5,7 @@
 from resources import strings
 
 
-pytestmark = pytest.mark.asyncio
+pytestmark = pytest.mark.asyncio(loop_scope="session")
 
 
 @pytest.mark.smoke

e2e_tests/test_ui.py

@@ -4,7 +4,7 @@
 import config
 
 
-pytestmark = pytest.mark.asyncio
+pytestmark = pytest.mark.asyncio(loop_scope="session")
 
 
 @pytest.mark.smoke

e2e_tests/test_workspace_service_templates.py

@@ -8,7 +8,7 @@
 from resources import strings
 from helpers import get_admin_token
 
-pytestmark = pytest.mark.asyncio
+pytestmark = pytest.mark.asyncio(loop_scope="session")
 
 workspace_service_templates = [
     (strings.AZUREML_SERVICE),

e2e_tests/test_workspace_services.py

@@ -5,7 +5,7 @@
 from resources.resource import get_resource, post_resource
 from resources import strings
 
-pytestmark = pytest.mark.asyncio
+pytestmark = pytest.mark.asyncio(loop_scope="session")
 
 workspace_services = [
     strings.AZUREML_SERVICE,