Support compilerfolder and online environments (#3607)

When specifying BcAuthContext and Environment to Run-AlPipeline,
Run-AlPipeline would always create a filesonly container, disallowing
running on Linux.
This PR fixes this plus some bugs found as a result of this.

- Replace all occurrences of
[System.Runtime.InteropServices.Marshal]::PtrToStringAuto with
[System.Runtime.InteropServices.Marshal]::PtrToStringBSTR as the Auto
function doesn't always do what's expected under Linux (We do not use
the Auto function in AL-Go)
- Ensure correct casing of Newtonsoft.Json.dll for Linux (also not a
problem in AL-Go)
- Always add extensionId (when specified) to Properties section in test
results xml
- Also added two new overrides (PipelineInitialize and PipelineFinalize)
requested by COSMO Consult.
- If environment is specified as a Web Client URL, and BcAuthContext
contains username/password in Run-AlPipeline, then tests will run
against this environment. PublishBcContainerApp and
ImportTestToolkitToBcContainer needs to be overridden for this to work
with full pipeline.
- Add parameter CompilerFolder to Run-TestsInBcContainer and
Import-TestToolkitToBcContainer for running tests using CompilerFolder
bits from the host
- Including caching of appinfos in CompilerFolder cache (to save time
when caching on GitHub Actions)

Running Build AND Test under Linux (using CompilerFolder), using an
online environment as "Service Tier" can be seen here:

https://github.com/BusinessCentralDemos/bingmaps.pte/actions/runs/10313615507

Build and test here takes approx. 3 minutes.

This functionality is needed by COSMO to enable using their Docker Swarm
for running tests in AL-Go.

COSMO is aware that AL-Go moves away from using BcContainerHelper and
will subsequently have to change their integration when this has
happened.

---------

Co-authored-by: freddydk <freddydk@users.noreply.github.com>

AppHandling/Compile-AppInNavContainer.ps1

@@ -427,7 +427,7 @@ try {
                 throw "You need to specify credentials when you are not using Windows Authentication"
             }
 
-            $pair = ("$($Credential.UserName):"+[System.Runtime.InteropServices.Marshal]::PtrToStringAuto([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($credential.Password)))
+            $pair = ("$($Credential.UserName):"+[System.Runtime.InteropServices.Marshal]::PtrToStringBSTR([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($credential.Password)))
             $bytes = [System.Text.Encoding]::ASCII.GetBytes($pair)
             $base64 = [System.Convert]::ToBase64String($bytes)
             $basicAuthValue = "Basic $base64"

AppHandling/Get-NavContainerApp.ps1

@@ -76,7 +76,7 @@ try {
             throw "You need to specify credentials when you are not using Windows Authentication"
         }
 
-        $pair = ("$($Credential.UserName):"+[System.Runtime.InteropServices.Marshal]::PtrToStringAuto([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($credential.Password)))
+        $pair = ("$($Credential.UserName):"+[System.Runtime.InteropServices.Marshal]::PtrToStringBSTR([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($credential.Password)))
         $bytes = [System.Text.Encoding]::ASCII.GetBytes($pair)
         $base64 = [System.Convert]::ToBase64String($bytes)
         $basicAuthValue = "Basic $base64"

AppHandling/Get-TestsFromNavContainer.ps1

@@ -182,9 +182,9 @@ try {
 
     $result = Invoke-ScriptInBcContainer -containerName $containerName -usePwsh $false -scriptBlock { Param([string] $tenant, [string] $companyName, [string] $profile, [pscredential] $credential, [string] $accessToken, [string] $testSuite, [string] $testCodeunit, [string] $testCodeunitRange, [string] $PsTestFunctionsPath, [string] $ClientContextPath, $testPage, $version, $culture, $timezone, $debugMode, $ignoreGroups, $usePublicWebBaseUrl, $useUrl, $extensionId, $disabledtests)
 
-        $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Management\NewtonSoft.json.dll"
+        $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Management\Newtonsoft.Json.dll"
         if (!(Test-Path $newtonSoftDllPath)) {
-            $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\NewtonSoft.json.dll"
+            $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Newtonsoft.Json.dll"
         }
         $newtonSoftDllPath = (Get-Item $newtonSoftDllPath).FullName
         $clientDllPath = "C:\Test Assemblies\Microsoft.Dynamics.Framework.UI.Client.dll"

AppHandling/PsTestFunctions.ps1

@@ -44,7 +44,7 @@ function New-ClientContext {
         if ($Credential -eq $null -or $credential -eq [System.Management.Automation.PSCredential]::Empty) {
             throw "You need to specify credentials (Username and AccessToken) if using AAD authentication"
         }
-        $accessToken = [System.Runtime.InteropServices.Marshal]::PtrToStringAuto([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($credential.Password))
+        $accessToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($credential.Password))
         $clientContext = [ClientContext]::new($serviceUrl, $accessToken, $interactionTimeout, $culture, $timezone)
     }
     else {
@@ -726,18 +726,20 @@ function Run-Tests {
                 $JunitTestSuiteProperties = $JUnitDoc.CreateElement("properties")
                 $JUnitTestSuite.AppendChild($JunitTestSuiteProperties) | Out-Null
 
+                if ($extensionid) {
+                    $property = $JUnitDoc.CreateElement("property")
+                    $property.SetAttribute("name","extensionid")
+                    $property.SetAttribute("value", $extensionId)
+                    $JunitTestSuiteProperties.AppendChild($property) | Out-Null
+                }
+
                 if ($process) {
                     $property = $JUnitDoc.CreateElement("property")
                     $property.SetAttribute("name","processinfo.start")
                     $property.SetAttribute("value", $processinfostart)
                     $JunitTestSuiteProperties.AppendChild($property) | Out-Null
 
                     if ($extensionid) {
-                        $property = $JUnitDoc.CreateElement("property")
-                        $property.SetAttribute("name","extensionid")
-                        $property.SetAttribute("value", $extensionId)
-                        $JunitTestSuiteProperties.AppendChild($property) | Out-Null
-
                         $appname = "$(Get-NavAppInfo -ServerInstance $serverInstance | Where-Object { "$($_.AppId)" -eq $extensionId } | ForEach-Object { $_.Name })"
                         if ($appname) {
                             $property = $JUnitDoc.CreateElement("property")

AppHandling/Publish-NavContainerApp.ps1

@@ -233,7 +233,7 @@ try {
                         if (!($credential)) {
                             throw "You need to specify credentials when you are not using Windows Authentication"
                         }
-                        $pair = ("$($Credential.UserName):"+[System.Runtime.InteropServices.Marshal]::PtrToStringAuto([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($credential.Password)))
+                        $pair = ("$($Credential.UserName):"+[System.Runtime.InteropServices.Marshal]::PtrToStringBSTR([System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($credential.Password)))
                         $bytes = [System.Text.Encoding]::ASCII.GetBytes($pair)
                         $base64 = [System.Convert]::ToBase64String($bytes)
                         $HttpClient.DefaultRequestHeaders.Authorization = New-Object System.Net.Http.Headers.AuthenticationHeaderValue("Basic", $base64);

AppHandling/Run-AlPipeline.ps1

@@ -183,6 +183,10 @@
   Information about which product built the app. Will be stamped into the app manifest.
  .Parameter BuildUrl
   The URL for the build job, which built the app. Will be stamped into the app manifest.
+ .Parameter PipelineInitialize
+  Override for Pipeline Initialize
+ .Parameter PipelineFinalize
+  Override for Pipeline Finalize 
  .Parameter DockerPull
   Override function parameter for docker pull
  .Parameter NewBcContainer
@@ -193,7 +197,7 @@
   Override function parameter for Import-TestToolkitToBcContainer
  .Parameter CompileAppInBcContainer
   Override function parameter for Compile-AppInBcContainer
-  .Parameter CompileAppWithBcCompilerFolder
+ .Parameter CompileAppWithBcCompilerFolder
   Override function parameter for Compile-AppWithBcCompilerFolder
  .Parameter PreCompileApp
   Custom script to run before compiling an app.
@@ -363,6 +367,7 @@ Param(
     [string] $sourceCommit = '',
     [string] $buildBy = "BcContainerHelper,$BcContainerHelperVersion",
     [string] $buildUrl = '',
+    [scriptblock] $PipelineInitialize,
     [scriptblock] $DockerPull,
     [scriptblock] $NewBcContainer,
     [scriptblock] $SetBcContainerKeyVaultAadAppAndCertificate,
@@ -383,7 +388,8 @@ Param(
     [scriptblock] $RemoveBcContainer,
     [scriptblock] $GetBestGenericImageName,
     [scriptblock] $GetBcContainerEventLog,
-    [scriptblock] $InstallMissingDependencies
+    [scriptblock] $InstallMissingDependencies,
+    [scriptblock] $PipelineFinalize
 )
 
 function CheckRelativePath([string] $baseFolder, [string] $sharedFolder, $path, $name) {
@@ -475,6 +481,10 @@ function GetInstalledAppIds {
 $telemetryScope = InitTelemetryScope -name $MyInvocation.InvocationName -parameterValues $PSBoundParameters -includeParameters @()
 try {
 
+if ($PipelineInitialize) {
+    Invoke-Command -ScriptBlock $PipelineInitialize
+}
+
 $warningsToShow = @()
 
 if (!$baseFolder -or !(Test-Path $baseFolder -PathType Container)) {
@@ -554,13 +564,19 @@ if ($bcAuthContext) {
         Write-Host -ForegroundColor Yellow "Uninstalling removed apps from online environments are not supported"
         $uninstallRemovedApps = $false
     }
-    $bcAuthContext = Renew-BcAuthContext -bcAuthContext $bcAuthContext
-    $bcEnvironment = Get-BcEnvironments -bcAuthContext $bcAuthContext | Where-Object { $_.name -eq $environment -and $_.type -eq "Sandbox" }
-    if (!($bcEnvironment)) {
-        throw "Environment $environment doesn't exist in the current context or it is not a Sandbox environment."
+    if ($environment -notlike ('https://*')) {
+        $bcAuthContext = Renew-BcAuthContext -bcAuthContext $bcAuthContext
+        $bcEnvironment = Get-BcEnvironments -bcAuthContext $bcAuthContext | Where-Object { $_.name -eq $environment -and $_.type -eq "Sandbox" }
+        if (!($bcEnvironment)) {
+            throw "Environment $environment doesn't exist in the current context or it is not a Sandbox environment."
+        }
+        $parameters = @{
+            bcAuthContext = $bcAuthContext
+            environment = $environment
+        }
+        $bcBaseApp = Get-BcPublishedApps @Parameters | Where-Object { $_.Name -eq "Base Application" -and $_.state -eq "installed" }
+        $artifactUrl = Get-BCArtifactUrl -type Sandbox -country $bcEnvironment.countryCode -version $bcBaseApp.Version -select Closest
     }
-    $bcBaseApp = Get-BcPublishedApps -bcAuthContext $bcauthcontext -environment $environment | Where-Object { $_.Name -eq "Base Application" -and $_.state -eq "installed" }
-    $artifactUrl = Get-BCArtifactUrl -type Sandbox -country $bcEnvironment.countryCode -version $bcBaseApp.Version -select Closest
     $filesOnly = $true
 }
 
@@ -748,14 +764,16 @@ Write-Host -ForegroundColor Yellow "Custom CodeCops"
 if ($customCodeCops) { $customCodeCops | ForEach-Object { Write-Host "- $_" } } else { Write-Host "- None" }
 
 $vsixFile = DetermineVsixFile -vsixFile $vsixFile
-
 $compilerFolder = ''
+$createContainer = $true
 
 if ($useCompilerFolder) {
     # We are using CompilerFolder, no need for a filesOnly Container
     # If we are to create a container, it is for publishing and testing
     $filesOnly = $false
     $updateLaunchJson = ''
+    $createContainer = !($doNotPublishApps -or ($bcAuthContext -and $environment))
+    if (!$createContainer) { $containerName = ''}
 }
 elseif ($doNotPublishApps) {
     # We are not using CompilerFolder, but we are not publishing apps either
@@ -908,7 +926,7 @@ $signApps = ($codeSignCertPfxFile -ne "")
 
 Measure-Command {
 
-if ( $artifactUrl -and !$reUseContainer -and !$doNotPublishApps -and !$filesOnly) {
+if ( $artifactUrl -and !$reUseContainer -and $createContainer) {
 if ($gitHubActions) { Write-Host "::group::Pulling generic image" }
 Measure-Command {
 Write-Host -ForegroundColor Yellow @'
@@ -948,19 +966,36 @@ try {
 $testCountry = $_.Trim()
 $testToolkitInstalled = $false
 
-if ($gitHubActions) { Write-Host "::group::Creating container" }
-Write-Host -ForegroundColor Yellow @'
+if ($useCompilerFolder) {
+    if ($gitHubActions) { Write-Host "::group::Creating CompilerFolder" }
+    Write-Host -ForegroundColor Yellow @'
+
+   _____                _   _                _____                      _ _           ______    _     _
+  / ____|              | | (_)              / ____|                    (_) |         |  ____|  | |   | |
+ | |     _ __ ___  __ _| |_ _ _ __   __ _  | |     ___  _ __ ___  _ __  _| | ___ _ __| |__ ___ | | __| | ___ _ __
+ | |    | '__/ _ \/ _` | __| | '_ \ / _` | | |    / _ \| '_ ` _ \| '_ \| | |/ _ \ '__|  __/ _ \| |/ _` |/ _ \ '__|
+ | |____| | |  __/ (_| | |_| | | | | (_| | | |___| (_) | | | | | | |_) | | |  __/ |  | | | (_) | | (_| |  __/ |
+  \_____|_|  \___|\__,_|\__|_|_| |_|\__, |  \_____\___/|_| |_| |_| .__/|_|_|\___|_|  |_|  \___/|_|\__,_|\___|_|
+                                     __/ |                       | |
+                                    |___/                        |_|
 
-   _____                _   _                               _        _
-  / ____|              | | (_)                             | |      (_)
- | |     _ __ ___  __ _| |_ _ _ __   __ _    ___ ___  _ __ | |_ __ _ _ _ __   ___ _ __
- | |    | '__/ _ \/ _` | __| | '_ \ / _` |  / __/ _ \| '_ \| __/ _` | | '_ \ / _ \ '__|
- | |____| | |  __/ (_| | |_| | | | | (_| | | (__ (_) | | | | |_ (_| | | | | |  __/ |
-  \_____|_|  \___|\__,_|\__|_|_| |_|\__, |  \___\___/|_| |_|\__\__,_|_|_| |_|\___|_|
+'@
+}
+else {
+    if ($gitHubActions) { Write-Host "::group::Creating container" }
+    Write-Host -ForegroundColor Yellow @'
+
+   _____                _   _                _____            _        _
+  / ____|              | | (_)              / ____|          | |      (_)
+ | |     _ __ ___  __ _| |_ _ _ __   __ _  | |     ___  _ __ | |_ __ _ _ _ __   ___ _ __
+ | |    | '__/ _ \/ _` | __| | '_ \ / _` | | |    / _ \| '_ \| __/ _` | | '_ \ / _ \ '__|
+ | |____| | |  __/ (_| | |_| | | | | (_| | | |___| (_) | | | | || (_| | | | | |  __/ |
+  \_____|_|  \___|\__,_|\__|_|_| |_|\__, |  \_____\___/|_| |_|\__\__,_|_|_| |_|\___|_|
                                      __/ |
                                     |___/
 
 '@
+}
 
 Measure-Command {
 
@@ -983,7 +1018,7 @@ Measure-Command {
             -containerName $containerName
         Write-Host "CompilerFolder $compilerFolder created"
     }
-    if ($filesOnly -or !$doNotPublishApps) {
+    if ($createContainer -and ($filesOnly -or !$doNotPublishApps)) {
         # If we are going to build using a filesOnly container or we are going to publish apps, we need a container
         if (Test-BcContainer -containerName $containerName) {
             if ($bcAuthContext) {
@@ -1035,7 +1070,7 @@ Measure-Command {
             }
             Invoke-Command -ScriptBlock $NewBcContainer -ArgumentList $Parameters
 
-            if (-not $bcAuthContext) {
+            if ($createContainer -and -not $bcAuthContext) {
                 if ($keyVaultCertPfxFile -and $KeyVaultClientId -and $keyVaultCertPfxPassword) {
                     $Parameters = @{
                         "containerName" = $containerName
@@ -1329,6 +1364,7 @@ Measure-Command {
     Write-Host -ForegroundColor Yellow "Importing Test Toolkit for additional country $testCountry"
     $Parameters = @{
         "containerName" = $containerName
+        "compilerFolder" = $compilerFolder
         "includeTestLibrariesOnly" = $installTestLibraries
         "includeTestFrameworkOnly" = !$installTestLibraries -and ($installTestFramework -or $installPerformanceToolkit)
         "includeTestRunnerOnly" = !$installTestLibraries -and !$installTestFramework -and ($installTestRunner -or $installPerformanceToolkit)
@@ -1508,6 +1544,7 @@ Measure-Command {
         $measureText = ", test apps and importing test toolkit"
         $Parameters = @{
             "containerName" = $containerName
+            "compilerFolder" = $compilerFolder
             "includeTestLibrariesOnly" = $installTestLibraries
             "includeTestFrameworkOnly" = !$installTestLibraries -and ($installTestFramework -or $installPerformanceToolkit)
             "includeTestRunnerOnly" = !$installTestLibraries -and !$installTestFramework -and ($installTestRunner -or $installPerformanceToolkit)
@@ -1654,7 +1691,7 @@ Write-Host -ForegroundColor Yellow @'
     $Parameters = @{ }
     $CopParameters = @{ }
 
-    if ($bcAuthContext) {
+    if ($bcAuthContext -and !$useCompilerFolder) {
         $Parameters += @{
             "bcAuthContext" = $bcAuthContext
             "environment" = $environment
@@ -2433,6 +2470,7 @@ $testAppIds.Keys | ForEach-Object {
     }
     $Parameters = @{
         "containerName" = $containerName
+        "compilerFolder" = $compilerFolder
         "tenant" = $tenant
         "credential" = $credential
         "companyName" = $companyName
@@ -2461,6 +2499,7 @@ $testAppIds.Keys | ForEach-Object {
         $Parameters += @{
             "bcAuthContext" = $bcAuthContext
             "environment" = $environment
+            "ConnectFromHost" = !$createContainer
         }
     }
 
@@ -2615,7 +2654,11 @@ finally {
     $progressPreference = $prevProgressPreference
 }
 
-if (!$keepContainer) {
+if ($useCompilerFolder -and $compilerFolder) {
+    Remove-BcCompilerFolder -compilerFolder $compilerFolder
+}
+
+if ($createContainer -and !$keepContainer) {
 if ($gitHubActions) { Write-Host "::group::Removing container" }
 if (!($err)) {
 Write-Host -ForegroundColor Yellow @'
@@ -2631,9 +2674,6 @@ Write-Host -ForegroundColor Yellow @'
 }
 Measure-Command {
 
-    if ($useCompilerFolder -and $compilerFolder) {
-        Remove-BcCompilerFolder -compilerFolder $compilerFolder
-    }
     if (!$doNotPublishApps) {
         if (!$filesOnly -and $containerEventLogFile) {
             try {
@@ -2667,6 +2707,10 @@ if ($err) {
 
 } | ForEach-Object { Write-Host -ForegroundColor Yellow "`nAL Pipeline finished in $([int]$_.TotalSeconds) seconds" }
 
+if ($PipelineFinalize) {
+    Invoke-Command -ScriptBlock $PipelineFinalize
+}
+
 }
 catch {
     TrackException -telemetryScope $telemetryScope -errorRecord $_

AppHandling/Run-ConnectionTestToNavContainer.ps1

@@ -135,15 +135,15 @@ try {
     }
 
     if ($connectFromHost) {
-        $newtonSoftDllPath = Join-Path $PsTestToolFolder "NewtonSoft.json.dll"
+        $newtonSoftDllPath = Join-Path $PsTestToolFolder "Newtonsoft.Json.dll"
         $clientDllPath = Join-Path $PsTestToolFolder "Microsoft.Dynamics.Framework.UI.Client.dll"
 
         Invoke-ScriptInBcContainer -containerName $containerName { Param([string] $myNewtonSoftDllPath, [string] $myClientDllPath)
 
             if (!(Test-Path $myNewtonSoftDllPath)) {
-                $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Management\NewtonSoft.json.dll"
+                $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Management\Newtonsoft.Json.dll"
                 if (!(Test-Path $newtonSoftDllPath)) {
-                    $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\NewtonSoft.json.dll"
+                    $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Newtonsoft.Json.dll"
                 }
                 $newtonSoftDllPath = (Get-Item $newtonSoftDllPath).FullName
                 Copy-Item -Path $newtonSoftDllPath -Destination $myNewtonSoftDllPath
@@ -202,9 +202,9 @@ try {
 
         $result = Invoke-ScriptInBcContainer -containerName $containerName -usePwsh $false -scriptBlock { Param([string] $tenant, [string] $companyName, [string] $profile, [pscredential] $credential, [string] $accessToken, [string] $PsTestFunctionsPath, [string] $ClientContextPath, [timespan] $interactionTimeout, $version, $culture, $timezone, $debugMode, $usePublicWebBaseUrl, $useUrl)
 
-            $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Management\NewtonSoft.json.dll"
+            $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Management\Newtonsoft.Json.dll"
             if (!(Test-Path $newtonSoftDllPath)) {
-                $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\NewtonSoft.json.dll"
+                $newtonSoftDllPath = "C:\Program Files\Microsoft Dynamics NAV\*\Service\Newtonsoft.Json.dll"
             }
             $newtonSoftDllPath = (Get-Item $newtonSoftDllPath).FullName
             $clientDllPath = "C:\Test Assemblies\Microsoft.Dynamics.Framework.UI.Client.dll"