From 5cfbfcb2e6dd94aa5b4eef5572a775b52ec36176 Mon Sep 17 00:00:00 2001 From: Vladimir Safonkin Date: Wed, 29 Apr 2020 13:00:56 +0300 Subject: [PATCH] Add Python toolcache installation from Github releases for Ubuntu (#704) Change logic to install Python to Ubuntu images to use GitHub releases from https://github.com/actions/python-versions --- .../scripts/installers/Install-Toolset.ps1 | 55 ++++++++++++++++ .../scripts/installers/Validate-Toolset.ps1 | 62 +++++++++++++++++++ .../scripts/installers/hosted-tool-cache.sh | 6 -- .../scripts/installers/test-toolcache.sh | 1 - images/linux/toolcache-1604.json | 3 - images/linux/toolcache-1804.json | 3 - images/linux/toolset-1604.json | 18 ++++++ images/linux/toolset-1804.json | 18 ++++++ images/linux/ubuntu1604.json | 18 ++++++ images/linux/ubuntu1804.json | 18 ++++++ 10 files changed, 189 insertions(+), 13 deletions(-) create mode 100644 images/linux/scripts/installers/Install-Toolset.ps1 create mode 100644 images/linux/scripts/installers/Validate-Toolset.ps1 create mode 100644 images/linux/toolset-1604.json create mode 100644 images/linux/toolset-1804.json diff --git a/images/linux/scripts/installers/Install-Toolset.ps1 b/images/linux/scripts/installers/Install-Toolset.ps1 new file mode 100644 index 000000000000..518a1f9bc808 --- /dev/null +++ b/images/linux/scripts/installers/Install-Toolset.ps1 @@ -0,0 +1,55 @@ +################################################################################ +## File: Install-Toolset.ps1 +## Team: CI-Build +## Desc: Install toolset +################################################################################ + +Function Install-Asset { + param( + [Parameter(Mandatory = $true)] + [object] $ReleaseAsset + ) + + Write-Host "Download $($ReleaseAsset.filename)" + wget $ReleaseAsset.download_url -nv --retry-connrefused --tries=10 + + Write-Host "Extract $($ReleaseAsset.filename) content..." + $assetFolderPath = Join-Path $env:INSTALLER_SCRIPT_FOLDER $($ReleaseAsset.filename) + New-Item -ItemType Directory -Path $assetFolderPath + tar -xzf $ReleaseAsset.filename -C $assetFolderPath + + Write-Host "Invoke installation script..." + Push-Location -Path $assetFolderPath + Invoke-Expression "bash ./setup.sh" + Pop-Location +} + +$ErrorActionPreference = "Stop" + +# Get toolset content +$toolsetJson = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw +$tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache + +foreach ($tool in $tools) { + # Get versions manifest for current tool + $assets = Invoke-RestMethod $tool.url + + # Get github release asset for each version + foreach ($toolVersion in $tool.versions) { + $asset = $assets | Where-Object version -like $toolVersion ` + | Select-Object -ExpandProperty files ` + | Where-Object { ($_.platform -eq $tool.platform) -and ($_.platform_version -eq $tool.platform_version)} ` + | Select-Object -First 1 + + Write-Host "Installing $($tool.name) $toolVersion $($tool.arch)..." + if ($asset -ne $null) { + Install-Asset -ReleaseAsset $asset + } + else { + Write-Host "Asset was not found in versions manifest" + exit 1 + } + } +} + +chown -R "$($env:SUDO_USER):$($env:SUDO_USER)" /opt/hostedtoolcache/Python \ No newline at end of file diff --git a/images/linux/scripts/installers/Validate-Toolset.ps1 b/images/linux/scripts/installers/Validate-Toolset.ps1 new file mode 100644 index 000000000000..13847465ecfe --- /dev/null +++ b/images/linux/scripts/installers/Validate-Toolset.ps1 @@ -0,0 +1,62 @@ +################################################################################ +## File: Validate-Toolset.ps1 +## Team: CI-Build +## Desc: Validate Toolset +################################################################################ + +function Run-ExecutableTests { + param ( + [Parameter(Mandatory)] [string[]] $Executables, + [Parameter(Mandatory)] [string] $ToolPath + ) + + foreach ($executable in $Executables) { + $executablePath = Join-Path $ToolPath $executable + + Write-Host "Check $executable..." + if (Test-Path $executablePath) { + Write-Host "$executable is successfully installed: $(& $executablePath --version)" + } else { + Write-Host "$executablePath is not installed!" + exit 1 + } + } +} + +$ErrorActionPreference = "Stop" + +# Define executables for cached tools +$toolsExecutables = @{ Python = @("python", "bin/pip") } + +# Get toolset content +$toolsetJson = Get-Content -Path "$env:INSTALLER_SCRIPT_FOLDER/toolset.json" -Raw +$tools = ConvertFrom-Json -InputObject $toolsetJson | Select-Object -ExpandProperty toolcache + +foreach($tool in $tools) { + Invoke-Expression "bash -c `"source $env:HELPER_SCRIPTS/document.sh; DocumentInstalledItem '$($tool.name):'`"" + + $toolPath = Join-Path $env:AGENT_TOOLSDIRECTORY $tool.name + # Get executables for current tool + $toolExecs = $toolsExecutables[$tool.name] + + foreach ($version in $tool.versions) { + # Check if version folder exists + $expectedVersionPath = Join-Path $toolPath $version + if (-not (Test-Path $expectedVersionPath)) { + Write-Host "Expected $($tool.name) $version folder is not found!" + exit 1 + } + + # Take latest installed version in case if toolset version contains wildcards + $foundVersion = Get-Item $expectedVersionPath ` + | Sort-Object -Property {[version]$_.name} -Descending ` + | Select-Object -First 1 + $foundVersionPath = Join-Path $foundVersion $tool.arch + + Write-Host "Run validation test for $($tool.name)($($tool.arch)) $($foundVersion.name) executables..." + Run-ExecutableTests -Executables $toolExecs -ToolPath $foundVersionPath + + # Add tool version to documentation + Invoke-Expression "bash -c `"source $env:HELPER_SCRIPTS/document.sh; DocumentInstalledItemIndent '$($tool.name) $($foundVersion.name)'`"" + } +} \ No newline at end of file diff --git a/images/linux/scripts/installers/hosted-tool-cache.sh b/images/linux/scripts/installers/hosted-tool-cache.sh index f9e5b918afad..a0ff4466ba7d 100644 --- a/images/linux/scripts/installers/hosted-tool-cache.sh +++ b/images/linux/scripts/installers/hosted-tool-cache.sh @@ -49,12 +49,6 @@ done; popd -DocumentInstalledItem "Python:" -pythons=$(ls $AGENT_TOOLSDIRECTORY/Python) -for python in $pythons; do - DocumentInstalledItemIndent "Python $python" -done; - DocumentInstalledItem "Ruby:" rubys=$(ls $AGENT_TOOLSDIRECTORY/Ruby) for ruby in $rubys; do diff --git a/images/linux/scripts/installers/test-toolcache.sh b/images/linux/scripts/installers/test-toolcache.sh index 2601db4b48e1..53ed0715e7fd 100644 --- a/images/linux/scripts/installers/test-toolcache.sh +++ b/images/linux/scripts/installers/test-toolcache.sh @@ -69,6 +69,5 @@ done; AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache -Test_Hostedtoolcache_Tool "Python" "x64/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+'" Test_Hostedtoolcache_Tool "Ruby" "x64/bin/ruby -e 'puts RUBY_VERSION' | egrep -o '[0-9]+\.[0-9]+'" Test_Hostedtoolcache_Tool "PyPy" "x64/bin/python -c 'import sys;print(sys.version)'| head -1 | egrep -o '[0-9]+\.[0-9]+' | cut -d '.' -f 1" diff --git a/images/linux/toolcache-1604.json b/images/linux/toolcache-1604.json index 40f4cd9d46f2..96638f0f45ba 100644 --- a/images/linux/toolcache-1604.json +++ b/images/linux/toolcache-1604.json @@ -1,7 +1,4 @@ { - "@actions/toolcache-python-ubuntu-1604-x64": [ - "2.7", "3.5", "3.6", "3.7", "3.8" - ], "@actions/toolcache-ruby-ubuntu-1604-x64": [ "2.4", "2.5", "2.6", "2.7" ], diff --git a/images/linux/toolcache-1804.json b/images/linux/toolcache-1804.json index 26ad4d98e5db..13c697851e03 100644 --- a/images/linux/toolcache-1804.json +++ b/images/linux/toolcache-1804.json @@ -1,7 +1,4 @@ { - "@actions/toolcache-python-ubuntu-1804-x64": [ - "2.7", "3.5", "3.6", "3.7", "3.8" - ], "@actions/toolcache-ruby-ubuntu-1804-x64": [ "2.4", "2.5", "2.6", "2.7" ], diff --git a/images/linux/toolset-1604.json b/images/linux/toolset-1604.json new file mode 100644 index 000000000000..e3953f81bac9 --- /dev/null +++ b/images/linux/toolset-1604.json @@ -0,0 +1,18 @@ +{ + "toolcache": [ + { + "name": "Python", + "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json", + "platform" : "linux", + "platform_version": "16.04", + "arch": "x64", + "versions": [ + "2.7.*", + "3.5.*", + "3.6.*", + "3.7.*", + "3.8.*" + ] + } + ] +} \ No newline at end of file diff --git a/images/linux/toolset-1804.json b/images/linux/toolset-1804.json new file mode 100644 index 000000000000..a854ae6b6ffc --- /dev/null +++ b/images/linux/toolset-1804.json @@ -0,0 +1,18 @@ +{ + "toolcache": [ + { + "name": "Python", + "url" : "https://raw.githubusercontent.com/actions/python-versions/master/versions-manifest.json", + "platform" : "linux", + "platform_version": "18.04", + "arch": "x64", + "versions": [ + "2.7.*", + "3.5.*", + "3.6.*", + "3.7.*", + "3.8.*" + ] + } + ] +} \ No newline at end of file diff --git a/images/linux/ubuntu1604.json b/images/linux/ubuntu1604.json index 62081af54112..cc4eb8f5957a 100644 --- a/images/linux/ubuntu1604.json +++ b/images/linux/ubuntu1604.json @@ -269,6 +269,11 @@ "source": "{{template_dir}}/toolcache-1604.json", "destination": "{{user `installer_script_folder`}}/toolcache.json" }, + { + "type": "file", + "source": "{{template_dir}}/toolset-1604.json", + "destination": "{{user `installer_script_folder`}}/toolset.json" + }, { "type": "shell", "scripts":[ @@ -287,6 +292,19 @@ ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/Install-Toolset.ps1", + "{{template_dir}}/scripts/installers/Validate-Toolset.ps1" + ], + "environment_vars": [ + "METADATA_FILE={{user `metadata_file`}}", + "HELPER_SCRIPTS={{user `helper_script_folder`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} pwsh -f {{ .Path }}'" + }, { "type": "shell", "scripts":[ diff --git a/images/linux/ubuntu1804.json b/images/linux/ubuntu1804.json index 41b758976a54..b50efb289134 100644 --- a/images/linux/ubuntu1804.json +++ b/images/linux/ubuntu1804.json @@ -273,6 +273,11 @@ "source": "{{template_dir}}/toolcache-1804.json", "destination": "{{user `installer_script_folder`}}/toolcache.json" }, + { + "type": "file", + "source": "{{template_dir}}/toolset-1804.json", + "destination": "{{user `installer_script_folder`}}/toolset.json" + }, { "type": "shell", "scripts":[ @@ -291,6 +296,19 @@ ], "execute_command": "sudo sh -c '{{ .Vars }} {{ .Path }}'" }, + { + "type": "shell", + "scripts":[ + "{{template_dir}}/scripts/installers/Install-Toolset.ps1", + "{{template_dir}}/scripts/installers/Validate-Toolset.ps1" + ], + "environment_vars": [ + "METADATA_FILE={{user `metadata_file`}}", + "HELPER_SCRIPTS={{user `helper_script_folder`}}", + "INSTALLER_SCRIPT_FOLDER={{user `installer_script_folder`}}" + ], + "execute_command": "sudo sh -c '{{ .Vars }} pwsh -f {{ .Path }}'" + }, { "type": "shell", "scripts":[