Merge pull request #4919 from dfe-analytical-services/dev

Merge Dev into Master

Author: N-moh

.github/workflows/block-autosquash-commits.yml

@@ -11,6 +11,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Block auto-squash commits
-        uses: xt0rted/block-autosquash-commits-action@f1ab4d69a90297fde61bc263d2e048eb8480d293
+        uses: xt0rted/block-autosquash-commits-action@87d56ca2c3f01b9247504ead67a1a3c2ca62b030
         with:
           repo-token: ${{ secrets.GH_TOKEN }}

.gitignore

@@ -13,7 +13,7 @@ bin
 obj
 /data/ees-mssql
 /data/public-api-db
-/data/public-api-parquet
+/data/public-api-data
 dfe-meta.db
 
 ## CSharp

docker-compose.yml

@@ -43,6 +43,7 @@ services:
 
   public-api-db:
     image: postgres:16.1-alpine
+    command: postgres -c max_prepared_transactions=100
     ports:
       - "5432:5432"
     volumes:

infrastructure/templates/public-api/api-infrastructure-pipeline.yml

@@ -1,11 +1,5 @@
 trigger: none
 
-resources:
-  pipelines:
-    - pipeline: EESBuildPipeline
-      source: Explore Education Statistics
-      trigger: none
-
 parameters:
   - name: deployContainerApp
     displayName: Can we deploy the Container App yet?  This is dependent on the user-assigned Managed Identity for the API Container App being created with the AcrPull role, and the database users added to PSQL.
@@ -14,6 +8,27 @@ parameters:
     displayName: Does the PostgreSQL Flexible Server require any updates? False by default to avoid unnecessarily lengthy deploys.
     default: false
 
+# This param is helpful for debugging to allow the selection of a particular branch from which to base a deploy from this pipeline.
+# This should be removed in the long term in favour of using the "Resources" selection from the "Run pipeline" dialog.
+#
+#  - name: buildBranchToDeploy
+#    displayName: Build branch to deploy. This allows a person who is manually running the pipeline to specify the use of the latest EESBuildPipeline build that was run against that branch.
+#    default: 'Branch from latest pipeline run'
+
+resources:
+  pipelines:
+    - pipeline: EESBuildPipeline
+      source: Explore Education Statistics
+      trigger:
+        branches:
+          - refs/heads/dev
+          - refs/heads/test
+          - refs/heads/master
+# This param is helpful for debugging to allow the selection of a particular branch from which to base a deploy from this pipeline.
+# This should be removed in the long term in favour of using the "Resources" selection from the "Run pipeline" dialog.
+#
+#      branch: ${{ replace(parameters.buildBranchToDeploy, 'Branch from latest pipeline run', '') }}
+
 variables:
   - group: Public API Infrastructure - common
   - name: isDev

infrastructure/templates/public-api/application/virtualNetwork.bicep

@@ -13,31 +13,31 @@ param dataProcessorFunctionAppNameSuffix string
 @description('Specifies the name suffix of the Container App Environment')
 param containerAppEnvironmentNameSuffix string
 
-resource vNet 'Microsoft.Network/virtualNetworks@2023-09-01' existing = {
+resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' existing = {
   name: vNetName
 }
 
-resource adminSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = {
+resource adminSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = {
   name: '${subscription}-snet-ees-admin'
   parent: vNet
 }
 
-resource publisherSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = {
+resource publisherSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = {
   name: '${subscription}-snet-ees-publisher'
   parent: vNet
 }
 
-resource dataProcessorSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = {
+resource dataProcessorSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = {
   name: '${resourcePrefix}-snet-fa-${dataProcessorFunctionAppNameSuffix}'
   parent: vNet
 }
 
-resource containerAppEnvironmentSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = {
+resource containerAppEnvironmentSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = {
   name: '${subscription}-ees-snet-cae-${containerAppEnvironmentNameSuffix}'
   parent: vNet
 }
 
-resource psqlFlexibleServerSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-09-01' existing = {
+resource psqlFlexibleServerSubnet 'Microsoft.Network/virtualNetworks/subnets@2023-11-01' existing = {
   name: '${subscription}-ees-snet-psql-flexibleserver'
   parent: vNet
 }

infrastructure/templates/public-api/components/appServiceSlotConfig.bicep

@@ -1,9 +1,6 @@
 @description('Specifies the Web / Function App name that these settings belong to')
 param appName string
 
-@description('Specifies the location of the resources')
-param location string
-
 @description('Specifies the names of slot settings (settings that stick to their slots rather than swap)')
 param slotSpecificSettingKeys string[]
 
@@ -16,28 +13,23 @@ param prodOnlySettings object
 @description('Specifies a set of appsettings that are specific to the staging slot')
 param stagingOnlySettings object
 
+@description('Specifies the name of the staging slot')
+param stagingSlotName string = 'staging'
+
 @description('Specifies any existing appsettings from the staging slot')
 param existingStagingAppSettings object
 
 @description('Specifies any existing appsettings from the production slot')
 param existingProductionAppSettings object
 
-@description('A set of tags with which to tag the resource in Azure')
-param tagValues object
-
-@description('Create a staging slot')
-resource stagingSlot 'Microsoft.Web/sites/slots@2023-01-01' = {
-  name: '${appName}/staging'
-  location: location
-  identity: {
-    type: 'SystemAssigned'
-  }
-  properties: {
-    enabled: true
-    httpsOnly: true
-  }
-  tags: tagValues
-}
+@description('Specifies additional Azure Storage Accounts to make available to the staging slot')
+param azureFileShares {
+  storageName: string
+  storageAccountKey: string
+  storageAccountName: string
+  fileShareName: string
+  mountPath: string
+}[] = []
 
 @description('Set specific appsettings to be slot specific values')
 resource functionSlotConfig 'Microsoft.Web/sites/config@2023-01-01' = {
@@ -52,20 +44,32 @@ resource functionSlotConfig 'Microsoft.Web/sites/config@2023-01-01' = {
 // infrastructure deploys do not reset appsettings back to original values and cause
 // unwanted updates to production appsettings prior to a slot swap deploy process being
 // ready to run.
+//
+// See https://blog.dotnetstudio.nl/posts/2021/04/merge-appsettings-with-bicep.
 var combinedStagingSettings = union(commonSettings, stagingOnlySettings, existingStagingAppSettings)
 var combinedProductionSettings = union(commonSettings, prodOnlySettings, existingProductionAppSettings)
 
 @description('Set appsettings on the staging slot')
 resource appStagingSlotSettings 'Microsoft.Web/sites/slots/config@2023-01-01' = {
-  name: 'appsettings'
-  parent: stagingSlot
+  name: '${appName}/${stagingSlotName}/appsettings'
   properties: combinedStagingSettings
 }
 
+resource azureStorageAccounts 'Microsoft.Web/sites/slots/config@2021-01-15' = {
+  name: '${appName}/${stagingSlotName}/azurestorageaccounts'
+  properties: reduce(azureFileShares, {}, (cur, next) => union(cur, {
+    '${next.storageName}': {
+      type: 'AzureFiles'
+      shareName: next.fileShareName
+      mountPath: next.mountPath
+      accountName: next.storageAccountName
+      accessKey: next.storageAccountKey
+    }
+  }))
+}
+
 @description('Set appsettings on production slot')
 resource appProductionSettings 'Microsoft.Web/sites/config@2023-01-01' = {
   name: '${appName}/appsettings'
   properties: combinedProductionSettings
 }
-
-output stagingSlotPrincipalId string = stagingSlot.identity.principalId

infrastructure/templates/public-api/components/blobStore.bicep

@@ -10,11 +10,11 @@ param deleteRetentionPolicy int = 7
 param storageAccountName string
 
 // Reference an existing Storage Account.
-resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing = {
+resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing = {
   name: storageAccountName
 }
 
-resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-01-01' = {
+resource blobServices 'Microsoft.Storage/storageAccounts/blobServices@2023-04-01' = {
   name: blobStoreName
   parent: storageAccount
   properties: {

infrastructure/templates/public-api/components/containerApp.bicep

@@ -61,25 +61,40 @@ param appSettings {
 param tagValues object
 
 @description('An existing Managed Identity\'s Resource Id with which to associate this Container App')
-param managedIdentityName string
+param userAssignedManagedIdentityId string
 
 @description('Id of the owning Container App Environment')
 param managedEnvironmentId string
 
+@description('Volumes to mount within Containers - used in conjunction with "volumeMounts"')
+param volumes {
+  name: string
+  storageType: string
+  storageName: string
+  mountOptions: string?
+  secrets: {
+    path: string
+    secretRef: string
+  }[]?
+}[] = []
+
+@description('Volume mount points within Containers - used in conjunction with "volumes"')
+param volumeMounts {
+  mountPath: string
+  volumeName: string
+}[] = []
+
+
 var containerImageName = '${acrLoginServer}/${containerAppImageName}'
 var containerApplicationName = toLower('${resourcePrefix}-ca-${containerAppName}')
 
-resource containerAppIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = {
-  name: managedIdentityName
-}
-
-resource containerApp 'Microsoft.App/containerApps@2023-05-01' = {
+resource containerApp 'Microsoft.App/containerApps@2024-03-01' = {
   name: containerApplicationName
   location: location
   identity: {
     type: 'UserAssigned'
     userAssignedIdentities: {
-      '${containerAppIdentity.id}': {}
+      '${userAssignedManagedIdentityId}': {}
     }
   }
   properties: {
@@ -101,7 +116,7 @@ resource containerApp 'Microsoft.App/containerApps@2023-05-01' = {
       registries: [
         {
           server: acrLoginServer
-          identity: containerAppIdentity.id
+          identity: userAssignedManagedIdentityId
         }
       ]
     }
@@ -115,6 +130,7 @@ resource containerApp 'Microsoft.App/containerApps@2023-05-01' = {
             cpu: json(cpuCore)
             memory: '${memorySize}Gi'
           }
+          volumeMounts: volumeMounts
         }
       ]
       scale: {
@@ -131,6 +147,7 @@ resource containerApp 'Microsoft.App/containerApps@2023-05-01' = {
           }
         ]
       }
+      volumes: volumes
     }
     workloadProfileName: 'Consumption'
   }

infrastructure/templates/public-api/components/containerAppEnvironment.bicep

@@ -31,15 +31,24 @@ param tagValues object
 @description('Specifies a suffix to append to the full name of the Container App Environment')
 param containerAppEnvironmentNameSuffix string = ''
 
+@description('Specifies an array of Azure File Shares to be available for Container Apps hosted within this Container App Environment')
+param azureFileStorages {
+  storageName: string
+  storageAccountKey: string
+  storageAccountName: string
+  fileShareName: string
+  accessMode: 'ReadWrite' | 'ReadOnly'
+}[] = []
+
 var containerAppEnvironmentName = empty(containerAppEnvironmentNameSuffix)
   ? '${subscription}-ees-cae'
   : '${subscription}-ees-cae-${containerAppEnvironmentNameSuffix}'
 
-resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2021-06-01' existing = {
+resource logAnalyticsWorkspace 'Microsoft.OperationalInsights/workspaces@2023-09-01' existing = {
   name: logAnalyticsWorkspaceName
 }
 
-resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2023-05-01' = {
+resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2024-03-01' = {
   name: containerAppEnvironmentName
   location: location
   properties: {
@@ -58,6 +67,18 @@ resource containerAppEnvironment 'Microsoft.App/managedEnvironments@2023-05-01'
     workloadProfiles: workloadProfiles
   }
   tags: tagValues
+  
+  resource azureFileStorage 'storages@2022-03-01' = [for storage in azureFileStorages: {
+      name: storage.storageName
+      properties: {
+        azureFile: {
+          accountKey: storage.storageAccountKey
+          accountName: storage.storageAccountName
+          shareName: storage.fileShareName
+          accessMode: storage.accessMode
+        }
+      }
+  }]
 }
 
 output containerAppEnvironmentName string = containerAppEnvironmentName

infrastructure/templates/public-api/components/fileShares.bicep infrastructure/templates/public-api/components/fileShare.bicep

@@ -17,11 +17,11 @@ param storageAccountName string
 var shareName = '${resourcePrefix}-fs-${fileShareName}'
 
 // Reference an existing Storage Account.
-resource storageAccount 'Microsoft.Storage/storageAccounts@2023-01-01' existing = {
+resource storageAccount 'Microsoft.Storage/storageAccounts@2023-04-01' existing = {
   name: storageAccountName
 }
 
-resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2023-01-01' = {
+resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2023-04-01' = {
   name: 'default'
   parent: storageAccount
 }