Refactor Azure Monitor in core (#2375)

* destroy stale cicd environments

* fix

* fix

* setup azmonitor & appinsights natively as possible

* update version

* updates

* version change

CHANGELOG.md

@@ -16,14 +16,14 @@ ENHANCEMENTS:
 
 BUG FIXES:
 
-*
+* Azure monitor resourced provided by Terraform and don't allow ingestion over internet ([#2375](https://github.com/microsoft/AzureTRE/pull/2375)).
 
 ## 0.4.1 (August 03, 2022)
 
 **BREAKING CHANGES & MIGRATIONS**:
 
 * Guacamole workspace service configures firewall requirements with deployment pipeline ([#2371](https://github.com/microsoft/AzureTRE/pull/2371)). **Migration** is manual - update the templateVersion of `tre-shared-service-firewall` in Cosmos to `0.4.0` in order to use this capability.
-* Workspace now has an AirlockManager role that has the permissions to review airlock requests  ([#2349](https://github.com/microsoft/AzureTRE/pull/2349)).
+* Workspace now has an AirlockManager role that has the permissions to review airlock requests ([#2349](https://github.com/microsoft/AzureTRE/pull/2349)).
 
 FEATURES:
 

devops/scripts/destroy_env_no_terraform.sh

@@ -95,23 +95,10 @@ echo "Looking for diagnostic settings..."
 # using xargs to run in parallel.
 az resource list --resource-group "${core_tre_rg}" --query '[].[id]' -o tsv | xargs -P 10 -I {} bash -c 'delete_resource_diagnostic "{}"'
 
+tre_id=${core_tre_rg#"rg-"}
 
 # purge keyvault if possible (makes it possible to reuse the same tre_id later)
 # this has to be done before we delete the resource group since we might not wait for it to complete
-
-# DEBUG START
-# This section is to aid debugging an issue where keyvaults aren't being deleted and purged
-echo "keyvault properties:"
-az keyvault list --resource-group "${core_tre_rg}" --query "[].properties"
-echo "keyvault purge protection evaluation result:"
-az keyvault list --resource-group "${core_tre_rg}" --query "[?properties.enablePurgeProtection==``null``] | length (@)"
-
-if [[ -n ${SHOW_KEYVAULT_DEBUG_ON_DESTROY:-} ]]; then
-  az keyvault list --resource-group "${core_tre_rg}" --query "[].properties" --debug
-fi
-# DEBUG END
-
-tre_id=${core_tre_rg#"rg-"}
 keyvault_name="kv-${tre_id}"
 keyvault=$(az keyvault show --name "${keyvault_name}" --resource-group "${core_tre_rg}" || echo 0)
 if [ "${keyvault}" != "0" ]; then
@@ -150,6 +137,14 @@ else
   echo "Resource group ${core_tre_rg} doesn't have a keyvault without purge protection."
 fi
 
+# linked storage accounts don't get deleted with the workspace
+workspace_name="log-${tre_id}"
+workspace=$(az monitor log-analytics workspace show --workspace-name "${workspace_name}" --resource-group "${core_tre_rg}" || echo 0)
+if [ "${workspace}" != "0" ]; then
+  az monitor log-analytics workspace linked-storage list -g "${core_tre_rg}" --workspace-name "${workspace_name}" -o tsv --query '[].id' \
+  | xargs -P 10 -I {} az rest --method delete --uri "{}?api-version=2020-08-01"
+fi
+
 # this will find the mgmt, core resource groups as well as any workspace ones
 # we are reverse-sorting to first delete the workspace groups (might not be
 # good enough because we use no-wait sometimes)

templates/core/terraform/azure-monitor/app_insights_byo_storage.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "app_insights_name": {
+      "type": "String"
+    },
+    "storage_account_resource_id": {
+      "type": "String"
+    }
+  },
+  "variables": {},
+  "resources": [
+    {
+      "name": "[concat(parameters('app_insights_name'), '/serviceprofiler')]",
+      "type": "microsoft.insights/components/linkedStorageAccounts",
+      "apiVersion": "2020-03-01-preview",
+      "properties": {
+        "linkedStorageAccount": "[parameters('storage_account_resource_id')]"
+      }
+    }
+  ]
+}