diff --git a/images/win/Windows2016-Azure.json b/images/win/Windows2016-Azure.json index 55a1e611e8ab..259fbb53e6e9 100644 --- a/images/win/Windows2016-Azure.json +++ b/images/win/Windows2016-Azure.json @@ -18,6 +18,7 @@ "run_scan_antivirus": "false", "root_folder": "C:", + "toolset_json_path": "{{env `TEMP`}}\\toolset.json", "image_folder": "C:\\image", "commit_file": "C:\\image\\commit.txt", "imagedata_file": "C:\\imagedata.json", @@ -318,6 +319,11 @@ "source": "{{template_dir}}/toolcache-2016.json", "destination": "{{user `root_folder`}}/toolcache.json" }, + { + "type": "file", + "source": "{{template_dir}}/toolset-2016.json", + "destination": "{{user `toolset_json_path`}}" + }, { "type": "powershell", "environment_vars":[ @@ -328,6 +334,15 @@ "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1" ] }, + { + "type": "powershell", + "environment_vars":[ + "TOOLSET_JSON_PATH={{user `toolset_json_path`}}" + ], + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1" + ] + }, { "type": "powershell", "scripts":[ @@ -642,6 +657,15 @@ "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1" ] }, + { + "type": "powershell", + "environment_vars":[ + "TOOLSET_JSON_PATH={{user `toolset_json_path`}}" + ], + "scripts":[ + "{{ template_dir }}/scripts/Installers/Validate-Toolset.ps1" + ] + }, { "type": "powershell", "scripts":[ diff --git a/images/win/Windows2019-Azure.json b/images/win/Windows2019-Azure.json index ab28c810c461..325c87028d09 100644 --- a/images/win/Windows2019-Azure.json +++ b/images/win/Windows2019-Azure.json @@ -18,6 +18,7 @@ "run_scan_antivirus": "false", "root_folder": "C:", + "toolset_json_path": "{{env `TEMP`}}\\toolset.json", "image_folder": "C:\\image", "commit_file": "C:\\image\\commit.txt", "imagedata_file": "C:\\imagedata.json", @@ -291,6 +292,11 @@ "source": "{{template_dir}}/toolcache-2019.json", "destination": "{{user `root_folder`}}/toolcache.json" }, + { + "type": "file", + "source": "{{template_dir}}/toolset-2019.json", + "destination": "{{user `toolset_json_path`}}" + }, { "type": "powershell", "environment_vars":[ @@ -301,6 +307,15 @@ "{{ template_dir }}/scripts/Installers/Download-ToolCache.ps1" ] }, + { + "type": "powershell", + "environment_vars":[ + "TOOLSET_JSON_PATH={{user `toolset_json_path`}}" + ], + "scripts":[ + "{{ template_dir }}/scripts/Installers/Install-Toolset.ps1" + ] + }, { "type": "powershell", "scripts":[ @@ -639,6 +654,15 @@ "{{ template_dir }}/scripts/Installers/Validate-ToolCache.ps1" ] }, + { + "type": "powershell", + "environment_vars":[ + "TOOLSET_JSON_PATH={{user `toolset_json_path`}}" + ], + "scripts":[ + "{{ template_dir }}/scripts/Installers/Validate-Toolset.ps1" + ] + }, { "type": "powershell", "scripts":[ diff --git a/images/win/scripts/ImageHelpers/ImageHelpers.psm1 b/images/win/scripts/ImageHelpers/ImageHelpers.psm1 index f979659d9122..2b716880bf42 100644 --- a/images/win/scripts/ImageHelpers/ImageHelpers.psm1 +++ b/images/win/scripts/ImageHelpers/ImageHelpers.psm1 @@ -15,6 +15,7 @@ Export-ModuleMember -Function @( 'Set-SystemVariable' 'Install-Binary' 'Get-ToolcachePackages' + 'Get-ToolsetContent' 'Get-ToolsByName' 'Add-ContentToMarkdown' 'Add-SoftwareDetailsToMarkdown' diff --git a/images/win/scripts/ImageHelpers/InstallHelpers.ps1 b/images/win/scripts/ImageHelpers/InstallHelpers.ps1 index ee5287805320..c8d70c8bfeed 100644 --- a/images/win/scripts/ImageHelpers/InstallHelpers.ps1 +++ b/images/win/scripts/ImageHelpers/InstallHelpers.ps1 @@ -283,6 +283,11 @@ function Get-ToolcachePackages { Get-Content -Raw $toolcachePath | ConvertFrom-Json } +function Get-ToolsetContent { + $toolsetJson = Get-Content -Path $env:TOOLSET_JSON_PATH -Raw + ConvertFrom-Json -InputObject $toolsetJson +} + function Get-ToolsByName { param ( [Parameter(Mandatory = $True)] diff --git a/images/win/scripts/Installers/Download-ToolCache.ps1 b/images/win/scripts/Installers/Download-ToolCache.ps1 index 4749a3f3f224..6c93e3804574 100644 --- a/images/win/scripts/Installers/Download-ToolCache.ps1 +++ b/images/win/scripts/Installers/Download-ToolCache.ps1 @@ -44,21 +44,6 @@ Function NPMFeed-AuthSetup { $npmrcContent | Out-File -FilePath "$($env:TEMP)/.npmrc" -Encoding utf8 } -Function Set-DefaultPythonVersion { - param( - [Parameter(Mandatory=$true)] - [System.Version] $Version, - [System.String] $Arch = "x64" - ) - - $pythonPath = $Env:AGENT_TOOLSDIRECTORY + "/Python/${Version}*/${Arch}" - $pythonDir = Get-Item -Path $pythonPath - - Write-Host "Use Python ${Version} as a system Python" - Add-MachinePathItem -PathItem $pythonDir.FullName - Add-MachinePathItem -PathItem "$($pythonDir.FullName)\Scripts" -} - Function Set-DefaultRubyVersion { param( [Parameter(Mandatory=$true)] @@ -106,5 +91,4 @@ $ToolVersions.PSObject.Properties | ForEach-Object { } } -Set-DefaultPythonVersion -Version "3.7" Set-DefaultRubyVersion -Version "2.5" \ No newline at end of file diff --git a/images/win/scripts/Installers/Install-Toolset.ps1 b/images/win/scripts/Installers/Install-Toolset.ps1 new file mode 100644 index 000000000000..25bfe58ec18b --- /dev/null +++ b/images/win/scripts/Installers/Install-Toolset.ps1 @@ -0,0 +1,76 @@ +################################################################################ +## File: Install-Toolset.ps1 +## Team: CI-Build +## Desc: Install toolset +################################################################################ + +Function Install-Asset { + param( + [Parameter(Mandatory=$true)] + [object] $ReleaseAsset + ) + + $releaseAssetName = [System.IO.Path]::GetFileNameWithoutExtension($ReleaseAsset.filename) + $assetFolderPath = Join-Path $env:TEMP $releaseAssetName + $assetArchivePath = Start-DownloadWithRetry -Url $ReleaseAsset.download_url -Name $ReleaseAsset.filename + + Write-Host "Extract $($ReleaseAsset.filename) content..." + 7z.exe x $assetArchivePath -o"$assetFolderPath" -y | Out-Null + + Write-Host "Invoke installation script..." + Push-Location -Path $assetFolderPath + Invoke-Expression .\setup.ps1 + Pop-Location +} + +Function Set-DefaultPythonVersion { + param( + [Parameter(Mandatory=$true)] + [object[]] $Toolset + ) + + $python = $Toolset | Where-Object { ($_.name -eq "Python") -and ($_.default -ne "") } ` + | Select-Object default, arch -First 1 + + if ($python.default -ne $null) { + $pythonPath = Join-Path $Env:AGENT_TOOLSDIRECTORY "/Python/$($python.default)/$($python.arch)" -Resolve + + Write-Host "Use Python $($python.default) as a system Python" + Add-MachinePathItem -PathItem $pythonPath + Add-MachinePathItem -PathItem "$pythonPath\Scripts" + } else { + Write-Host "Default Python version not found in toolset file!" + } +} + +$ErrorActionPreference = "Stop" + +Import-Module -Name ImageHelpers -Force + +# Get toolcache content from toolset +$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache + +foreach ($tool in $tools) { + # Get versions manifest for current tool + $assets = Invoke-RestMethod $tool.url + + # Get github release asset for each version + foreach ($toolVersion in $tool.versions) { + $asset = $assets | Where-Object version -like $toolVersion ` + | Sort-Object -Property {[version]$_.version} -Descending ` + | Select-Object -ExpandProperty files ` + | Where-Object { ($_.platform -eq $tool.platform) -and ($_.arch -eq $tool.arch) } ` + | Select-Object -First 1 + + Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..." + if ($asset -ne $null) { + Install-Asset -ReleaseAsset $asset + } else { + Write-Host "Asset was not found in versions manifest" + exit 1 + } + } +} + +# Install default python version +Set-DefaultPythonVersion -Toolset $tools \ No newline at end of file diff --git a/images/win/scripts/Installers/Validate-ToolCache.ps1 b/images/win/scripts/Installers/Validate-ToolCache.ps1 index 5ad500bf444f..526254e2b612 100644 --- a/images/win/scripts/Installers/Validate-ToolCache.ps1 +++ b/images/win/scripts/Installers/Validate-ToolCache.ps1 @@ -63,32 +63,6 @@ function RunTestsByPath { } } -function Get-SystemDefaultPython { - Write-Host "Validate system Python..." - - if (Get-Command -Name 'python') - { - Write-Host "Python $(& python -V 2>&1) on path" - } - else - { - Write-Host "Python is not on path" - exit 1 - } - - $pythonBinVersion = $(& python -V 2>&1) - if ($pythonBinVersion -notlike "Python 3.*") - { - Write-Error "Python 3 is not in the PATH" - exit 1 - } - - $pythonBinOnPath = Split-Path -Path (Get-Command -Name 'python').Path - $description = GetDefaultToolDescription -SoftwareVersion $pythonBinVersion -SoftwareLocation $pythonBinOnPath - - return $description -} - function Get-SystemDefaultRuby { Write-Host "Validate system Ruby..." @@ -196,9 +170,6 @@ function ToolcacheTest { } } - if ($SoftwareName -contains "Python") { - $markdownDescription += Get-SystemDefaultPython - } if ($SoftwareName -contains "Ruby") { $markdownDescription += Get-SystemDefaultRuby } @@ -206,10 +177,6 @@ function ToolcacheTest { Add-SoftwareDetailsToMarkdown -SoftwareName $SoftwareName -DescriptionMarkdown $markdownDescription } -# Python test -$PythonTests = @("python.exe", "Scripts\pip.exe") -ToolcacheTest -SoftwareName "Python" -ExecTests $PythonTests - # PyPy test $PyPyTests = @("python.exe", "bin\pip.exe") ToolcacheTest -SoftwareName "PyPy" -ExecTests $PyPyTests diff --git a/images/win/scripts/Installers/Validate-Toolset.ps1 b/images/win/scripts/Installers/Validate-Toolset.ps1 new file mode 100644 index 000000000000..28963b12a781 --- /dev/null +++ b/images/win/scripts/Installers/Validate-Toolset.ps1 @@ -0,0 +1,111 @@ +################################################################################ +## File: Validate-Toolset.ps1 +## Team: CI-Build +## Desc: Validate Toolset +################################################################################ + +function Run-ExecutableTests { + param ( + [Parameter(Mandatory)] [string[]] $Executables, + [Parameter(Mandatory)] [string] $ToolPath + ) + + foreach ($executable in $Executables) { + $executablePath = Join-Path $ToolPath $executable + + Write-Host "Check $executable..." + if (Test-Path $executablePath) { + Write-Host "$executable is successfully installed: $(& $executablePath --version)" + } else { + Write-Host "$executablePath is not installed!" + exit 1 + } + } +} + +function Validate-SystemDefaultTool { + param ( + [Parameter(Mandatory)] [string] $ToolName, + [Parameter(Mandatory)] [string] $ExpectedVersion + ) + + $binName = $ToolName.ToLower() + + # Check if tool on path + if (Get-Command -Name $binName) { + $versionOnPath = $(& $binName --version 2>&1) | Select-String -Pattern ".*(\d+\.\d+\.\d+)" + $versionBinPath = Split-Path -Path (Get-Command -Name $binName).Path + + # Check if version is correct + if ($versionOnPath.matches.Groups[1].Value -notlike $ExpectedVersion) { + Write-Error "$ToolName $ExpectedVersion is not in the PATH" + exit 1 + } + + Write-Host "$ToolName $versionOnPath on path" + } else { + Write-Host "$ToolName is not on path" + exit 1 + } + + # Add default version description to markdown + $description = "
__System default version:__ $versionOnPath
" + $description += "_Environment:_
" + $description += "* Location: $versionBinPath
" + $description += "* PATH: contains the location of $versionOnPath
" + + return $description +} + +$ErrorActionPreference = "Stop" + +Import-Module -Name ImageHelpers -Force + +# Define executables for cached tools +$toolsExecutables = @{ Python = @("python.exe", "Scripts\pip.exe") } + +# Get toolcache content from toolset +$tools = Get-ToolsetContent | Select-Object -ExpandProperty toolcache + +foreach($tool in $tools) { + $markdownDescription = "" + + $toolPath = Join-Path $env:AGENT_TOOLSDIRECTORY $tool.name + # Get executables for current tool + $toolExecs = $toolsExecutables[$tool.name] + + foreach ($version in $tool.versions) { + # Check if version folder exists + $expectedVersionPath = Join-Path $toolPath $version + if (-not (Test-Path $expectedVersionPath)) { + Write-Host "Expected $($tool.name) $version folder is not found!" + exit 1 + } + + # Take latest installed version in case if toolset version contains wildcards + $foundVersion = Get-Item $expectedVersionPath ` + | Sort-Object -Property {[version]$_.name} -Descending ` + | Select-Object -First 1 + + # Check for required architecture folder + $foundVersionArchPath = Join-Path $foundVersion $tool.arch + if (-not (Test-Path $foundVersionArchPath)) { + Write-Host "Expected $($tool.name)($($tool.arch)) $($foundVersion.name) architecture folder is not found!" + exit 1 + } + + Write-Host "Run validation test for $($tool.name)($($tool.arch)) $($foundVersion.name) executables..." + Run-ExecutableTests -Executables $toolExecs -ToolPath $foundVersionArchPath + + # Add to tool version to markdown + $markdownDescription += "_Version:_ $($foundVersion.name)
" + } + + # Create markdown description for system default tool + if (-not ([string]::IsNullOrEmpty($tool.default))) { + Write-Host "Validate system default $($tool.name)($($tool.arch)) $($tool.default)..." + $markdownDescription += Validate-SystemDefaultTool -ToolName $tool.name -ExpectedVersion $tool.default + } + + Add-SoftwareDetailsToMarkdown -SoftwareName "$($tool.name) ($($tool.arch))" -DescriptionMarkdown $markdownDescription +} \ No newline at end of file diff --git a/images/win/toolcache-2016.json b/images/win/toolcache-2016.json index 8db575c23b3b..1b4f1a6c7c50 100644 --- a/images/win/toolcache-2016.json +++ b/images/win/toolcache-2016.json @@ -1,10 +1,4 @@ { - "@actions/toolcache-python-windows-x64": [ - "2.7", "3.5", "3.6", "3.7", "3.8" - ], - "@actions/toolcache-python-windows-x86": [ - "2.7", "3.5", "3.6", "3.7", "3.8" - ], "@actions/toolcache-ruby-windows-x64": [ "2.4", "2.5", "2.6", "2.7" ], diff --git a/images/win/toolcache-2019.json b/images/win/toolcache-2019.json index 4f946cfaf442..6873c14e5765 100644 --- a/images/win/toolcache-2019.json +++ b/images/win/toolcache-2019.json @@ -1,10 +1,4 @@ { - "@actions/toolcache-python-windows-x64": [ - "2.7", "3.5", "3.6", "3.7", "3.8" - ], - "@actions/toolcache-python-windows-x86": [ - "2.7", "3.5", "3.6", "3.7", "3.8" - ], "@actions/toolcache-ruby-windows-x64": [ "2.4", "2.5", "2.6", "2.7" ], diff --git a/images/win/toolset-2016.json b/images/win/toolset-2016.json new file mode 100644 index 000000000000..da372c498667 --- /dev/null +++ b/images/win/toolset-2016.json @@ -0,0 +1,31 @@ +{ + "toolcache": [ + { + "name": "Python", + "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json", + "arch": "x64", + "platform" : "win32", + "versions": [ + "2.7.*", + "3.5.*", + "3.6.*", + "3.7.*", + "3.8.*" + ], + "default": "3.7.*" + }, + { + "name": "Python", + "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json", + "arch": "x86", + "platform" : "win32", + "versions": [ + "2.7.*", + "3.5.*", + "3.6.*", + "3.7.*", + "3.8.*" + ] + } + ] +} \ No newline at end of file diff --git a/images/win/toolset-2019.json b/images/win/toolset-2019.json new file mode 100644 index 000000000000..da372c498667 --- /dev/null +++ b/images/win/toolset-2019.json @@ -0,0 +1,31 @@ +{ + "toolcache": [ + { + "name": "Python", + "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json", + "arch": "x64", + "platform" : "win32", + "versions": [ + "2.7.*", + "3.5.*", + "3.6.*", + "3.7.*", + "3.8.*" + ], + "default": "3.7.*" + }, + { + "name": "Python", + "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json", + "arch": "x86", + "platform" : "win32", + "versions": [ + "2.7.*", + "3.5.*", + "3.6.*", + "3.7.*", + "3.8.*" + ] + } + ] +} \ No newline at end of file