From 0dca935d7f4cb191566cc1ce71be1b3ae0cea8c3 Mon Sep 17 00:00:00 2001 From: Alexander Sehr Date: Thu, 30 Jun 2022 16:40:12 +0200 Subject: [PATCH] [Utilities] Updated ReadMe Utility to handle bicep test files when generating deployment examples (#1628) * Update to latest * Fixed incorrect param file ref (#1626) * Further updates * Further updates * Updated tool * Upper case title update --- .../jobs.validateModuleDeployment.yml | 4 +- .../validateModuleDeployment/action.yml | 4 +- modules/.global/global.module.tests.ps1 | 2 +- utilities/tools/Set-ModuleReadMe.ps1 | 295 ++++++++++++++---- 4 files changed, 231 insertions(+), 74 deletions(-) diff --git a/.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml b/.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml index bc521b9e03..f03517b844 100644 --- a/.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml +++ b/.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml @@ -253,7 +253,7 @@ jobs: if ($testTemplatePossibleParameters -contains 'namePrefix') { $functionInput['additionalParameters'] += @{ - namePrefix = $projectSettings.parameterFileTokens.localTokens[0].value + namePrefix = ($projectSettings.parameterFileTokens.localTokens | Where-Object { $_.name -eq 'namePrefix' }).value } } } @@ -344,7 +344,7 @@ jobs: if ($testTemplatePossibleParameters -contains 'namePrefix') { $functionInput['additionalParameters'] += @{ - namePrefix = $projectSettings.parameterFileTokens.localTokens[0].value + namePrefix = ($projectSettings.parameterFileTokens.localTokens | Where-Object { $_.name -eq 'namePrefix' }).value } } } diff --git a/.github/actions/templates/validateModuleDeployment/action.yml b/.github/actions/templates/validateModuleDeployment/action.yml index be516ed43a..116d82a934 100644 --- a/.github/actions/templates/validateModuleDeployment/action.yml +++ b/.github/actions/templates/validateModuleDeployment/action.yml @@ -247,7 +247,7 @@ runs: if ($testTemplatePossibleParameters -contains 'namePrefix') { $functionInput['additionalParameters'] += @{ - namePrefix = $projectSettings.parameterFileTokens.localTokens[0].value + namePrefix = ($projectSettings.parameterFileTokens.localTokens | Where-Object { $_.name -eq 'namePrefix' }).value } } } @@ -343,7 +343,7 @@ runs: if ($testTemplatePossibleParameters -contains 'namePrefix') { $functionInput['additionalParameters'] += @{ - namePrefix = $projectSettings.parameterFileTokens.localTokens[0].value + namePrefix = ($projectSettings.parameterFileTokens.localTokens | Where-Object { $_.name -eq 'namePrefix' }).value } } } diff --git a/modules/.global/global.module.tests.ps1 b/modules/.global/global.module.tests.ps1 index e8b08a8ea0..f81f6efe87 100644 --- a/modules/.global/global.module.tests.ps1 +++ b/modules/.global/global.module.tests.ps1 @@ -931,7 +931,7 @@ Describe 'Deployment template tests' -Tag Template { foreach ($moduleFolderPath in $moduleFolderPaths) { if (Test-Path (Join-Path $moduleFolderPath '.test')) { - $testFilePaths = (Get-ChildItem (Join-Path -Path $moduleFolderPath -ChildPath '.testeters.json') -Recurse -Force).FullName + $testFilePaths = (Get-ChildItem (Join-Path -Path $moduleFolderPath -ChildPath '.parameters.json') -Recurse -Force).FullName foreach ($testFilePath in $testFilePaths) { foreach ($token in $enforcedTokenList.Keys) { $parameterFileTokenTestCases += @{ diff --git a/utilities/tools/Set-ModuleReadMe.ps1 b/utilities/tools/Set-ModuleReadMe.ps1 index 8daf716a73..e0dbe22354 100644 --- a/utilities/tools/Set-ModuleReadMe.ps1 +++ b/utilities/tools/Set-ModuleReadMe.ps1 @@ -321,6 +321,9 @@ Optional. A switch to control whether or not to add a ARM-JSON-Parameter file ex .PARAMETER addBicep Optional. A switch to control whether or not to add a Bicep deployment example. Defaults to true. +.PARAMETER ProjectSettings +Optional. Projects settings to draw information from. For example the `namePrefix`. + .EXAMPLE Set-DeploymentExamplesSection -TemplateFileContent @{ resource = @{}; ... } -ReadMeFileContent @('# Title', '', '## Section 1', ...) @@ -342,6 +345,9 @@ function Set-DeploymentExamplesSection { [Parameter(Mandatory = $false)] [bool] $addBicep = $true, + [Parameter(Mandatory = $false)] + [hashtable] $ProjectSettings = @{}, + [Parameter(Mandatory = $false)] [string] $SectionStartIdentifier = '## Deployment examples' ) @@ -351,93 +357,234 @@ function Set-DeploymentExamplesSection { $moduleRoot = Split-Path $TemplateFilePath -Parent $resourceTypeIdentifier = $moduleRoot.Replace('\', '/').Split('/modules/')[1].TrimStart('/') - $parameterFiles = Get-ChildItem (Join-Path $moduleRoot '.test') -Filter '*parameters.json' -Recurse + $resourceType = $resourceTypeIdentifier.Split('/')[1] + $testFilePaths = (Get-ChildItem (Join-Path -Path $moduleRoot -ChildPath '.test') -File).FullName | Where-Object { $_ -match '.+\.[bicep|json]' } - $index = 1 - foreach ($testFilePath in $parameterFiles.FullName) { - $contentInJSONFormat = Get-Content -Path $testFilePath -Encoding 'utf8' | Out-String + $pathIndex = 1 + foreach ($testFilePath in $testFilePaths) { + $rawContentArray = Get-Content -Path $testFilePath + $rawContent = Get-Content -Path $testFilePath -Encoding 'utf8' | Out-String + + $exampleTitle = ((Split-Path $testFilePath -LeafBase) -replace '\.', ' ') -replace ' parameters', '' + $TextInfo = (Get-Culture).TextInfo + $exampleTitle = $TextInfo.ToTitleCase($exampleTitle) $SectionContent += @( - "

Example $index

" + '

Example {0}: {1}

' -f $pathIndex, $exampleTitle ) - if ($addJson) { - $SectionContent += @( - '', - '
', - '', - 'via JSON Parameter file', - '', - '```json', - $contentInJSONFormat.TrimEnd(), - '```', - '', - '
' - ) - } + if ((Split-Path $testFilePath -Extension) -eq '.bicep') { + # Bicep to JSON + # ============= + $bicepTestStartIndex = $rawContentArray.IndexOf("module testDeployment '../deploy.bicep' = {") + + + $bicepTestEndIndex = $bicepTestStartIndex + do { + $bicepTestEndIndex++ + } while ($rawContentArray[$bicepTestEndIndex] -ne '}') + + $rawBicepExample = $rawContentArray[$bicepTestStartIndex..$bicepTestEndIndex] + + # Replace placeholders + $namePrefix = ($ProjectSettings.parameterFileTokens.localTokens | Where-Object { $_.name -eq 'namePrefix' }).value + $serviceShort = ([regex]::Match($rawContent, "(?m)^param serviceShort string = '(.+)'\s*$")).Captures.Groups[1].Value + + $rawBicepExampleString = ($rawBicepExample | Out-String) + $rawBicepExampleString = $rawBicepExampleString -replace '\$\{serviceShort\}', $serviceShort + $rawBicepExampleString = $rawBicepExampleString -replace '\$\{namePrefix\}', $namePrefix + + $rawBicepExample = $rawBicepExampleString -split '\n' + + # Generate content + if ($addBicep) { + $rawBicepExample[0] = "module $resourceType './$resourceTypeIdentifier/deploy.bicep = {'" + $rawBicepExample = $rawBicepExample | Where-Object { $_ -notmatch 'scope: *' } + + $SectionContent += @( + '', + '
' + '' + 'via Bicep module' + '' + '```bicep', + ($rawBicepExample | ForEach-Object { " $_" }).TrimEnd(), + '```', + '', + '
' + '

' + ) + } + + if ($addJson) { + + $paramStartIndex = 0 + do { + $paramStartIndex++ + } while ($rawBicepExample[$paramStartIndex] -notmatch '\s+params: {') + + $paramBlockIndent = ([regex]::Match($rawBicepExample[$paramStartIndex], '^(\s+).*')).Captures.Groups[1].Value.Length + + $paramEndIndex = 0 + do { + $paramEndIndex++ + } while ($rawBicepExample[$paramEndIndex] -notmatch "^\s{$paramBlockIndent}\}\s*$") + + $paramBlock = $rawBicepExample[($paramStartIndex + 1)..($paramEndIndex - 1)] + + $paramInJsonFormat = @( + '{', + $paramBlock + '}' + ) | Out-String + + # Formatting + $paramInJsonFormat = $paramInJsonFormat -replace "'", '"' + $paramInJsonFormat = $paramInJsonFormat -replace '([0-9a-zA-Z]+):', '"$1":' + + $paramInJSONFormatArray = $paramInJsonFormat -split '\n' | Where-Object { $_ } - if ($addBicep) { - $JSONParametersHashTable = (ConvertFrom-Json $contentInJSONFormat -AsHashtable -Depth 99).parameters - - # Handle KeyVaut references - $keyVaultReferences = $JSONParametersHashTable.Keys | Where-Object { $JSONParametersHashTable[$_].Keys -contains 'reference' } - - if ($keyVaultReferences.Count -gt 0) { - $keyVaultReferenceData = @() - foreach ($reference in $keyVaultReferences) { - $resourceIdElem = $JSONParametersHashTable[$reference].reference.keyVault.id -split '/' - $keyVaultReferenceData += @{ - subscriptionId = $resourceIdElem[2] - resourceGroupName = $resourceIdElem[4] - vaultName = $resourceIdElem[-1] - secretName = $JSONParametersHashTable[$reference].reference.secretName - parameterName = $reference + + # Replace resource IDs + for ($index = 0; $index -lt $paramInJSONFormatArray.Count; $index++) { + if ($paramInJSONFormatArray[$index] -like '*:*' -and ($paramInJSONFormatArray[$index] -split ':')[1].Trim() -notmatch '".+"' -and $paramInJSONFormatArray[$index] -like '*.*') { + # In case of a reference like : "virtualWanId": resourceGroupResources.outputs.virtualWWANResourceId + $paramInJSONFormatArray[$index] = '{0}: "<{1}>"' -f ($paramInJSONFormatArray[$index] -split ':')[0], ([regex]::Match(($paramInJSONFormatArray[$index] -split ':')[0], '"(.+)"')).Captures.Groups[1].Value + } + if ($paramInJSONFormatArray[$index] -notlike '*:*' -and $paramInJSONFormatArray[$index] -notlike '*"*"*' -and $paramInJSONFormatArray[$index] -like '*.*') { + # In case of a reference like : [ \n resourceGroupResources.outputs.managedIdentityPrincipalId \n ] + $paramInJSONFormatArray[$index] = '"{0}"' -f $paramInJSONFormatArray[$index].Split('.')[-1].Trim() + } + } + + # Handle comma + for ($index = 0; $index -lt $paramInJSONFormatArray.Count; $index++) { + if ($paramInJSONFormatArray[$index] -match '[\{|\[]') { + # If we're just opening an object/array, skip + continue + } else { + if ((($index -lt $paramInJSONFormatArray.Count - 1) -and $paramInJSONFormatArray[$index + 1] -match '[\]|\}]') -or ($index -eq $paramInJSONFormatArray.Count - 1)) { + # -or ($index -eq $paramInJSONFormatArray.Count - 2 -and $paramInJSONFormatArray[$index + 1] -eq '')) { + # If the next item closes an object/array, or is the last line, skip + continue + } + $paramInJSONFormatArray[$index] = '{0},' -f $paramInJSONFormatArray[$index].Trim() + } + } + + # Add 'value' middle-layer for top-level parameters + $paramInJsonFormatObject = $paramInJSONFormatArray | Out-String | ConvertFrom-Json -AsHashtable -Depth 99 + $paramInJsonFormatObjectWithValue = @{} + foreach ($paramKey in $paramInJsonFormatObject.Keys) { + $paramInJsonFormatObjectWithValue[$paramKey] = @{ + value = $paramInJsonFormatObject[$paramKey] } } + + $jsonExample = [ordered]@{ + '$schema' = 'https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#' + contentVersion = '1.0.0.0' + parameters = $paramInJsonFormatObjectWithValue + } + + $jsonExample = ($jsonExample | ConvertTo-Json -Depth 99) -split '\n' + + $SectionContent += @( + '', + '

' + '' + 'via JSON Parameter file' + '' + '```json', + $jsonExample + '```', + '', + '
' + '

' + ) + } + } else { + # JSON to Bicep + # ============= + # TODO: Support JSON test template files? + + if ($addJson) { + $SectionContent += @( + '', + '

', + '', + 'via JSON Parameter file', + '', + '```json', + $rawContent.TrimEnd(), + '```', + '', + '
' + ) } - $extendedKeyVaultReferences = @() - $counter = 0 - foreach ($reference in ($keyVaultReferenceData | Sort-Object -Property 'vaultName' -Unique)) { - $counter++ - $extendedKeyVaultReferences += @( - "resource kv$counter 'Microsoft.KeyVault/vaults@2019-09-01' existing = {", + if ($addBicep) { + $JSONParametersHashTable = (ConvertFrom-Json $rawContent -AsHashtable -Depth 99).parameters + + # Handle KeyVaut references + $keyVaultReferences = $JSONParametersHashTable.Keys | Where-Object { $JSONParametersHashTable[$_].Keys -contains 'reference' } + + if ($keyVaultReferences.Count -gt 0) { + $keyVaultReferenceData = @() + foreach ($reference in $keyVaultReferences) { + $resourceIdElem = $JSONParametersHashTable[$reference].reference.keyVault.id -split '/' + $keyVaultReferenceData += @{ + subscriptionId = $resourceIdElem[2] + resourceGroupName = $resourceIdElem[4] + vaultName = $resourceIdElem[-1] + secretName = $JSONParametersHashTable[$reference].reference.secretName + parameterName = $reference + } + } + } + + $extendedKeyVaultReferences = @() + $counter = 0 + foreach ($reference in ($keyVaultReferenceData | Sort-Object -Property 'vaultName' -Unique)) { + $counter++ + $extendedKeyVaultReferences += @( + "resource kv$counter 'Microsoft.KeyVault/vaults@2019-09-01' existing = {", (" name: '{0}'" -f $reference.vaultName), (" scope: resourceGroup('{0}','{1}')" -f $reference.subscriptionId, $reference.resourceGroupName), - '}', - '' - ) + '}', + '' + ) - # Add attribute for later correct reference - $keyVaultReferenceData | Where-Object { $_.vaultName -eq $reference.vaultName } | ForEach-Object { - $_['vaultResourceReference'] = "kv$counter" + # Add attribute for later correct reference + $keyVaultReferenceData | Where-Object { $_.vaultName -eq $reference.vaultName } | ForEach-Object { + $_['vaultResourceReference'] = "kv$counter" + } } - } - # Handle VALUE references (i.e. remove them) - $JSONParameters = (ConvertFrom-Json $contentInJSONFormat -Depth 99).PSObject.properties['parameters'].value - $JSONParametersWithoutValue = [ordered]@{} - foreach ($parameter in $JSONParameters.PSObject.Properties) { - if ($parameter.value.PSObject.Properties.name -eq 'value') { - $JSONParametersWithoutValue[$parameter.name] = $parameter.value.PSObject.Properties['value'].value - } else { - # replace key vault references - $matchingTuple = $keyVaultReferenceData | Where-Object { $_.parameterName -eq $parameter.Name } - $JSONParametersWithoutValue[$parameter.name] = "{0}.getSecret('{1}')" -f $matchingTuple.vaultResourceReference, $matchingTuple.secretName + # Handle VALUE references (i.e. remove them) + $JSONParameters = (ConvertFrom-Json $rawContent -Depth 99).PSObject.properties['parameters'].value + $JSONParametersWithoutValue = [ordered]@{} + foreach ($parameter in $JSONParameters.PSObject.Properties) { + if ($parameter.value.PSObject.Properties.name -eq 'value') { + $JSONParametersWithoutValue[$parameter.name] = $parameter.value.PSObject.Properties['value'].value + } else { + # replace key vault references + $matchingTuple = $keyVaultReferenceData | Where-Object { $_.parameterName -eq $parameter.Name } + $JSONParametersWithoutValue[$parameter.name] = "{0}.getSecret('{1}')" -f $matchingTuple.vaultResourceReference, $matchingTuple.secretName + } } - } - $templateParameterObject = $JSONParametersWithoutValue | ConvertTo-Json -Depth 99 - if ($templateParameterObject -ne '{}') { - $contentInBicepFormat = $templateParameterObject -replace '"', "'" # Update any [xyz: "xyz"] to [xyz: 'xyz'] - $contentInBicepFormat = $contentInBicepFormat -replace ',', '' # Update any [xyz: xyz,] to [xyz: xyz] - $contentInBicepFormat = $contentInBicepFormat -replace "'(\w+)':", '$1:' # Update any ['xyz': xyz] to [xyz: xyz] - $contentInBicepFormat = $contentInBicepFormat -replace "'(.+.getSecret\('.+'\))'", '$1' # Update any [xyz: 'xyz.GetSecret()'] to [xyz: xyz.GetSecret()] + $templateParameterObject = $JSONParametersWithoutValue | ConvertTo-Json -Depth 99 + if ($templateParameterObject -ne '{}') { + $contentInBicepFormat = $templateParameterObject -replace '"', "'" # Update any [xyz: "xyz"] to [xyz: 'xyz'] + $contentInBicepFormat = $contentInBicepFormat -replace ',', '' # Update any [xyz: xyz,] to [xyz: xyz] + $contentInBicepFormat = $contentInBicepFormat -replace "'(\w+)':", '$1:' # Update any ['xyz': xyz] to [xyz: xyz] + $contentInBicepFormat = $contentInBicepFormat -replace "'(.+.getSecret\('.+'\))'", '$1' # Update any [xyz: 'xyz.GetSecret()'] to [xyz: xyz.GetSecret()] - $bicepParamsArray = $contentInBicepFormat -split ('\n') - $bicepParamsArray = $bicepParamsArray[1..($bicepParamsArray.count - 2)] + $bicepParamsArray = $contentInBicepFormat -split ('\n') + $bicepParamsArray = $bicepParamsArray[1..($bicepParamsArray.count - 2)] + } } - $resourceType = $resourceTypeIdentifier.Split('/')[1] $SectionContent += @( '', @@ -464,7 +611,7 @@ function Set-DeploymentExamplesSection { '' ) - $index++ + $pathIndex++ } # Build result @@ -641,6 +788,15 @@ function Set-ModuleReadMe { $fullResourcePath = (Split-Path $TemplateFilePath -Parent).Replace('\', '/').split('/modules/')[1] + $root = (Get-Item $PSScriptRoot).Parent.Parent.FullName + $projectSettingsPath = Join-Path $root 'settings.json' + if (Test-Path $projectSettingsPath) { + $projectSettings = Get-Content $projectSettingsPath | ConvertFrom-Json -AsHashtable + } else { + Write-Warning "No settings file found in path [$projectSettingsPath]" + $projectSettings = @{} + } + # Check readme if (-not (Test-Path $ReadMeFilePath) -or ([String]::IsNullOrEmpty((Get-Content $ReadMeFilePath -Raw)))) { # Create new readme file @@ -724,6 +880,7 @@ function Set-ModuleReadMe { $inputObject = @{ ReadMeFileContent = $readMeFileContent TemplateFilePath = $TemplateFilePath + ProjectSettings = $projectSettings } $readMeFileContent = Set-DeploymentExamplesSection @inputObject }