From e70b61892e1eef06fbde852c9999a492f2868104 Mon Sep 17 00:00:00 2001 From: Billy O'Neal Date: Tue, 14 Jan 2025 17:14:11 -0800 Subject: [PATCH] Add a "status" API to DiagnosticContext, overhaul Downloads console output (#1565) Extensive overhaul of our downloads handling and console output; @JavierMatosD and I have gone back and forth several times and yet kept introducing unintended bugs in other places, which led me to believe targeted fixes would no longer cut it. Fixes many longstanding bugs and hopefully makes our console output for this more understandable: * We no longer print 'error' when an asset cache misses but the authoritative download succeeds. This partially undoes #1541. It is good to print errors immediately when they happen, but if a subsequent authoritative download succeeds we need to not print those errors. * We now always and consistently print output from x-script s at the time that actually happens. Resolves https://devdiv.visualstudio.com/DevDiv/_workitems/edit/2300063 * We don't tell the user that proxy settings might fix a hash mismatch problem. * We do tell the user that proxy settings might fix a download from asset cache problem. * We now always tell the user the full command line we tried when invoking an x-script that fails. * We don't crash if an x-script doesn't create the file we expect, or creates a file with the wrong hash. * We now always print what we are doing *before* touching the network, so if we hang the user knows which server is being problematic. Note that this includes storing back to asset caches which we were previously entirely silent about except in case of failure. Other changes: * Removed debug output about asset cache configuration. The output was misleading / wrong depending on readwrite settings, and echoing to the user exactly what they said before we've interpreted it is not useful debug output. (Contrast with other `VcpkgPaths` debug output which tend to be paths we have likely changed from something a user said) Other notes: * This makes all dependencies of #908 speak `DiagnosticContext` so it will be easy to audit that the foreground/background thread behavior is correct after this. * I did test the curl status parsing on old Ubuntu again. Special thanks to @JavierMatosD for his help in review of the first console output attempts and for blowing the dust off this area in the first place. --- .../asset-caching/bad-hash-script.ps1 | 3 + .../asset-caching/failing-script.ps1 | 3 +- .../asset-caching/no-file-script.ps1 | 1 + .../end-to-end-tests-dir/asset-caching.ps1 | 435 +++- include/vcpkg/base/api-stable-format.h | 18 +- include/vcpkg/base/diagnostics.h | 136 +- include/vcpkg/base/downloads.h | 129 +- include/vcpkg/base/file_sink.h | 3 +- include/vcpkg/base/fwd/downloads.h | 6 +- include/vcpkg/base/fwd/message_sinks.h | 4 +- include/vcpkg/base/hash.h | 39 +- include/vcpkg/base/message-data.inc.h | 129 +- include/vcpkg/base/message_sinks.h | 84 +- include/vcpkg/base/strings.h | 1 + include/vcpkg/base/system.process.h | 128 +- include/vcpkg/binarycaching.h | 2 +- include/vcpkg/configure-environment.h | 9 +- include/vcpkg/tools.h | 2 +- include/vcpkg/vcpkgpaths.h | 2 +- locales/messages.json | 86 +- src/vcpkg-test/configparser.cpp | 28 +- src/vcpkg-test/downloads.cpp | 214 +- src/vcpkg-test/strings.cpp | 86 +- src/vcpkg/base/diagnostics.cpp | 270 ++- src/vcpkg/base/downloads.cpp | 1814 +++++++++++------ src/vcpkg/base/files.cpp | 5 +- src/vcpkg/base/hash.cpp | 67 +- src/vcpkg/base/message_sinks.cpp | 249 ++- src/vcpkg/base/strings.cpp | 32 +- src/vcpkg/base/system.process.cpp | 351 ++-- src/vcpkg/binarycaching.cpp | 187 +- src/vcpkg/commands.bootstrap-standalone.cpp | 16 +- src/vcpkg/commands.build.cpp | 2 +- src/vcpkg/commands.ci-verify-versions.cpp | 407 ++-- src/vcpkg/commands.download.cpp | 33 +- src/vcpkg/commands.format-manifest.cpp | 2 +- src/vcpkg/commands.help.cpp | 7 +- src/vcpkg/commands.remove.cpp | 6 +- src/vcpkg/commands.set-installed.cpp | 3 +- src/vcpkg/configure-environment.cpp | 53 +- src/vcpkg/paragraphs.cpp | 2 +- src/vcpkg/postbuildlint.cpp | 448 ++-- src/vcpkg/tools.cpp | 50 +- src/vcpkg/vcpkgpaths.cpp | 15 +- 44 files changed, 3682 insertions(+), 1885 deletions(-) create mode 100644 azure-pipelines/e2e-assets/asset-caching/bad-hash-script.ps1 create mode 100644 azure-pipelines/e2e-assets/asset-caching/no-file-script.ps1 diff --git a/azure-pipelines/e2e-assets/asset-caching/bad-hash-script.ps1 b/azure-pipelines/e2e-assets/asset-caching/bad-hash-script.ps1 new file mode 100644 index 0000000000..0f30f60271 --- /dev/null +++ b/azure-pipelines/e2e-assets/asset-caching/bad-hash-script.ps1 @@ -0,0 +1,3 @@ +Param([string]$File) +Write-Host "Creating file with the wrong hash" +Set-Content -Path $File -Value "This is a file with the wrong hash" -Encoding Ascii -NoNewline diff --git a/azure-pipelines/e2e-assets/asset-caching/failing-script.ps1 b/azure-pipelines/e2e-assets/asset-caching/failing-script.ps1 index c232cc056d..fafc73a496 100644 --- a/azure-pipelines/e2e-assets/asset-caching/failing-script.ps1 +++ b/azure-pipelines/e2e-assets/asset-caching/failing-script.ps1 @@ -1 +1,2 @@ -throw "Script download error" +Write-Host "Script download error" +exit 1 diff --git a/azure-pipelines/e2e-assets/asset-caching/no-file-script.ps1 b/azure-pipelines/e2e-assets/asset-caching/no-file-script.ps1 new file mode 100644 index 0000000000..16bcc8c21d --- /dev/null +++ b/azure-pipelines/e2e-assets/asset-caching/no-file-script.ps1 @@ -0,0 +1 @@ +Write-Host "Not creating a file" diff --git a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 index e00df0f361..7e04e204ff 100644 --- a/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/asset-caching.ps1 @@ -7,15 +7,16 @@ Run-Vcpkg -TestArgs ($commonArgs + @("install", "vcpkg-test-x-script", "--x-bina Throw-IfFailed $env:VCPKG_FORCE_DOWNLOADED_BINARIES = "ON" +$assetCacheRegex = [regex]::Escape($AssetCache) # Testing asset cache miss (not configured) + x-block-origin enabled Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" - +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=clear;x-block-origin", "--downloads-root=$DownloadsRoot")) +Throw-IfNotFailed $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"error: Missing .* and downloads are blocked by x-block-origin." +"A suitable version of cmake was not found \(required v[0-9.]+\)\.", +"Downloading cmake-[0-9.]+-[^.]+\.(zip|tar\.gz)", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+" ) -join "`n" if (-not ($actual -match $expected)) { @@ -23,12 +24,12 @@ if (-not ($actual -match $expected)) { } # Testing asset cache miss (not configured) + x-block-origin disabled -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" - +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=clear;", "--downloads-root=$DownloadsRoot")) +Throw-IfFailed $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Downloading .*." +"A suitable version of cmake was not found \(required v[0-9.]+\)\.", +"Downloading https://github\.com/Kitware/CMake/releases/download/[^ ]+ -> cmake-[0-9.]+-[^.]+\.(zip|tar\.gz)", +"Successfully downloaded cmake-[0-9.]+-[^.]+\.(zip|tar\.gz)" ) -join "`n" if (-not ($actual -match $expected)) { @@ -37,12 +38,23 @@ if (-not ($actual -match $expected)) { # Testing asset cache miss (configured) + x-block-origin enabled Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" - +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot")) +Throw-IfNotFailed $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss for .* and downloads are blocked by x-block-origin" +"A suitable version of cmake was not found \(required v[0-9.]+\)\.", +"Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-z]+", +"error: curl: \(37\) Couldn't open file [^\n]+", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\." ) -join "`n" if (-not ($actual -match $expected)) { @@ -51,15 +63,14 @@ if (-not ($actual -match $expected)) { # Testing asset cache miss (configured) + x-block-origin disabled Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" - +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) +Throw-IfFailed $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache miss; downloading from .*" -"Downloading .*" -"Successfully downloaded .*." -"Successfully stored .* to .*." +"A suitable version of cmake was not found \(required v[0-9\.]+\)\.", +"Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-f]+" +"Asset cache miss; trying authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+", +"Successfully downloaded cmake-[0-9.]+-[^.]+\.(zip|tar\.gz), storing to file://$assetCacheRegex/[0-9a-f]+", +"Store success" ) -join "`n" if (-not ($actual -match $expected)) { @@ -69,159 +80,387 @@ if (-not ($actual -match $expected)) { # Testing asset cache hit Refresh-Downloads Run-Vcpkg -TestArgs ($commonArgs + @('remove', 'vcpkg-internal-e2e-test-port')) -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) -$actual = $actual -replace "`r`n", "`n" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("fetch", "cmake", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;", "--downloads-root=$DownloadsRoot")) $expected = @( -"A suitable version of .* was not found \(required v[0-9\.]+\)." -"Asset cache hit for .*; downloaded from: .*" +"A suitable version of cmake was not found \(required v[0-9\.]+\)\.", +"Trying to download cmake-[0-9.]+-[^.]+\.(zip|tar\.gz) using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Download successful! Asset cache hit, did not try authoritative source https://github\.com/Kitware/CMake/releases/download/[^ ]+" ) -join "`n" if (-not ($actual -match $expected)) { throw "Failure: asset cache hit" } -# Testing asset caching && x-block-orgin promises when --debug is passed (enabled) +# azurl (no), x-block-origin (no), asset-cache (n/a), download (fail) +# Expected: Download failure message, nothing about asset caching Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug")) -if (-not ($actual.Contains("[DEBUG] External asset downloads are blocked (x-block-origin is enabled)") -and $actual.Contains("[DEBUG] Asset caching is enabled."))) { - throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)" -} +$expected = @( +"^Downloading https://localhost:1234/foobar\.html -> example3\.html", +"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\.", +"$" +) -join "`n" -# Testing asset caching && x-block-orgin promises when --debug is passed (disabled) -Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug")) -if (-not ($actual.Contains("[DEBUG] External asset downloads are allowed (x-block-origin is disabled)") -and $actual.Contains("[DEBUG] Asset cache is not configured"))) { - throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html")) +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)" } -# azurl (no), x-block-origin (no), asset-cache (n/a), download (fail) -# Expected: Download failure message, nothing about asset caching +# Also with multiple URLs Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html")) -if (-not ($actual.Contains("error: https://localhost:1234/foobar.html: curl failed to download with exit code 7"))) { +$expected = @( +"^Downloading example3\.html, trying https://localhost:1234/foobar\.html", +"Trying https://localhost:1235/baz\.html", +"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"error: curl: \(7\) Failed to connect to localhost port 1235( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\.", +"$" +) -join "`n" + +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html", "--url", "https://localhost:1235/baz.html")) +Throw-IfNotFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)" } #azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch) -#Expected: Download message with the "you might need to configure a proxy" message and with expected/actual sha +#Expected: Hash check failed message expected/actual sha Refresh-TestRoot +$expected = @( +"^Downloading https://example\.com -> example3\.html", +"[^\n]+example3\.html\.\d+\.part: error: download from https://example\.com had an unexpected hash", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", +"note: Actual : d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", +"$" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com")) -if (-not ($actual.Contains("Failed to download example3.html.") -and - $actual.Contains("If you are using a proxy, please ensure your proxy settings are correct.") -and - $actual.Contains("error: File does not have the expected hash:") -and - $actual.Contains("url: https://example.com") -and - $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and - $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) { +Throw-IfNotFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)" } # azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed) # Expected: Download success message, nothing about asset caching Refresh-TestRoot +$expected = @( +"^Downloading https://example\.com -> example3\.html", +"Successfully downloaded example3\.html", +"$" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com")) -if (-not ($actual.Contains("Downloading example3.html") -and - $actual.Contains("Successfully downloaded example3.html."))) { +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" +} + +# ... also with multiple authoritative URLs +if ($IsWindows) { + # WinHTTP + Refresh-TestRoot + $expected = @( + "^Downloading example3\.html, trying https://nonexistent\.example\.com", + "warning: Download https://nonexistent\.example\.com failed -- retrying after 1000ms", + "warning: Download https://nonexistent\.example\.com failed -- retrying after 2000ms", + "warning: Download https://nonexistent\.example\.com failed -- retrying after 4000ms", + "Trying https://example\.com", + "Successfully downloaded example3\.html", + "$" + ) -join "`n" + + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://nonexistent.example.com", "--url", "https://example.com")) + Throw-IfFailed + if (-not ($actual -match $expected)) { + throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" + } +} + +# Force curl with --header +Refresh-TestRoot +$expected = @( +"^Downloading example3\.html, trying https://nonexistent\.example\.com", +"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in 1 seconds\. 3 retries left\.", +"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in 2 seconds\. 2 retries left\.", +"warning: (Problem : timeout\.|Transient problem: timeout) Will retry in 4 seconds\. 1 retries left\.", +"Trying https://example\.com", +"Successfully downloaded example3\.html", +"$" +) -join "`n" + +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://nonexistent.example.com", "--url", "https://example.com", "--header", "Cache-Control: no-cache")) +Throw-IfFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)" } # azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a) # Expected: Download failure message, nothing about asset caching, x-block-origin complaint Refresh-TestRoot +$expected = @( +"^Downloading example3\.html", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"$" +) -join "`n" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=clear;x-block-origin")) -if (-not ($actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) { +Throw-IfNotFailed + if (-not ($actual -match $expected)) { throw "Failure: azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)" } + # azurl (yes), x-block-origin (no), asset-cache (miss), download (fail) # Expected: Download failure message, asset cache named, nothing about x-block-origin Refresh-TestRoot +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Asset cache miss; trying authoritative source https://localhost:1234/foobar\.html", +"error: curl: \(37\) Couldn't open file [^\n]+", +"error: curl: \(7\) Failed to connect to localhost port 1234( after \d+ ms)?: ((Could not|Couldn't) connect to server|Connection refused)", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\.", +"$" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -if (-not ($actual.Contains("Asset cache miss; downloading from https://localhost:1234/foobar.html") -and - $actual.Contains("Downloading example3.html") -and - $actual.Contains("error: file://$AssetCache") -and - $actual.Contains("curl failed to download with exit code 37"))) { +Throw-IfNotFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)" } # azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a) # Expected: Download success message, asset cache named, nothing about x-block-origin Refresh-TestRoot -$actual = $actual -replace "`r`n", "`n" -Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +Run-Vcpkg -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +Throw-IfFailed +Remove-Item "$downloadsRoot/example3.html" +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Download successful! Asset cache hit, did not try authoritative source https://example\.com", +"$" +) -join "`n" + +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)" +} + +# azurl (yes), x-block-origin (no), asset-cache (hash mismatch), download (success) +# Expected: Asset cache named, nothing about x-block-origin +Remove-Item "$downloadsRoot/example3.html" +Set-Content -Path "$AssetCache/d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a" -Encoding Ascii -NoNewline -Value "The wrong hash content" +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Asset cache miss; trying authoritative source https://example\.com", +"Successfully downloaded example3\.html, storing to file://$assetCacheRegex/[0-9a-f]+", +"Store success", +"$" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) { - throw "Failure: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)" +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)" } # azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch) -# Expected: Download message with "you might need to configure a proxy" and expected/actual sha -Refresh-TestRoot +# Expected: File read failure from the asset cache, hash check mismatch for the download. Proxy message emitted due to the asset cache miss even though it doesn't apply to the cache miss. +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Asset cache miss; trying authoritative source https://example\.com", +"error: curl: \(37\) Couldn't open file [^\n]+", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\." +"[^\n]+example3\.html\.\d+\.part: error: download from https://example\.com had an unexpected hash", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", +"note: Actual : d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", +"$" +) -join "`n" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (yes), x-block-origin (no), asset-cache (sha-mismatch), download (sha-mismatch)" +} + +# azurl (yes), x-block-origin (no), asset-cache (sha-mismatch), download (sha-mismatch) +# Expected: Hash check failed message expected/actual sha +Copy-Item "$AssetCache/d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a" "$AssetCache/d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b" +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Asset cache miss; trying authoritative source https://example\.com", +"[^\n]+example3\.html\.\d+\.part: error: download from file://$assetCacheRegex/[0-9a-z]+ had an unexpected hash", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", +"note: Actual : d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", +"[^\n]+example3\.html\.\d+\.part: error: download from https://example\.com had an unexpected hash", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", +"note: Actual : d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", +"$" +) -join "`n" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and - $actual.Contains("Downloading example3.html") -and - $actual.Contains("error: file://$AssetCache") -and - $actual.Contains("curl failed to download with exit code 37") -and - $actual.Contains("error: File does not have the expected hash:") -and - $actual.Contains("url: https://example.com") -and - $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and - $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) { - throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)" +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (yes), x-block-origin (no), asset-cache (sha-mismatch), download (sha-mismatch)" } # azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed) # Expected: Download success message, asset cache upload, nothing about x-block-origin Refresh-TestRoot +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Asset cache miss; trying authoritative source https://example\.com", +"Successfully downloaded example3\.html, storing to file://$assetCacheRegex/[0-9a-z]+", +"Store success", +"$" +) -join "`n" + $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and - $actual.Contains("Downloading example3.html") -and - $actual.Contains("Successfully downloaded example3.html.") -and - $actual.Contains("Successfully stored example3.html to file://$AssetCache"))) { - throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)" +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)" } # azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a) # Expected: Download failure message, which asset cache was tried, x-block-origin complaint Refresh-TestRoot -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin")) -if (-not ($actual.Contains("Asset cache miss for example3.html and downloads are blocked by x-block-origin.") -and - $actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) { +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"error: curl: \(37\) Couldn't open file [^\n]+", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"note: or https://alternate\.example\.com", +"note: If you are using a proxy, please ensure your proxy settings are correct\.", +"Possible causes are:", +"1\. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to ``https//address:port``\.", +"This is not correct, because ``https://`` prefix claims the proxy is an HTTPS proxy, while your proxy \(v2ray, shadowsocksr, etc\.\.\.\) is an HTTP proxy\.", +"Try setting ``http://address:port`` to both HTTP_PROXY and HTTPS_PROXY instead\." +"2\. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software\. See: https://github\.com/microsoft/vcpkg-tool/pull/77", +"The value set by your proxy might be wrong, or have same ``https://`` prefix issue\.", +"3\. Your proxy's remote server is our of service\.", +"If you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github\.com/Microsoft/vcpkg/issues", +"to report this upstream download server failure\." +"$" +) -join "`n" + +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--url", "https://alternate.example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin")) +Throw-IfNotFailed +if (-not ($actual -match $expected)) { throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)" } # azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a) # Expected: Download success message, asset cache named, nothing about x-block-origin Refresh-TestRoot -Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) -$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin")) -if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) { - throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)" +$expected = @( +"^Trying to download example3\.html using asset cache file://$assetCacheRegex/[0-9a-z]+", +"Download successful! Asset cache hit, did not try authoritative source https://example\.com, or https://alternate\.example\.com", +"$" +) -join "`n" +Run-Vcpkg -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite")) +Throw-IfFailed +Remove-Item "$downloadsRoot/example3.html" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--url", "https://alternate.example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin")) +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)" } # Testing x-download failure with asset cache (x-script) and x-block-origin settings $env:X_VCPKG_ASSET_SOURCES = "clear;x-script,pwsh $PSScriptRoot/../e2e-assets/asset-caching/failing-script.ps1 {url} {sha512} {dst};x-block-origin" +$expected = @( +"^Trying to download example3.html using asset cache script", +"Script download error", +"error: the asset cache script returned nonzero exit code 1", +"note: the full script command line was: pwsh .+/failing-script\.ps1 https://example\.com d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a `"?[^`"]+example3\.html\.\d+\.part`"?", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"$" +) -join "`n" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--url", "https://example.com", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a")) -# Check for the expected messages in order -$expectedOrder = @( - "error: failed with exit code: (1).", - "error: Missing example3.html and downloads are blocked by x-block-origin." -) - -# Verify order -$index = 0 -foreach ($message in $expectedOrder) { - $index = $actual.IndexOf($message, $index) - if ($index -lt 0) { - throw "Failure: Expected message '$message' not found in the correct order." - } - $index += $message.Length +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (no), x-block-origin (yes), asset-cache (hit), download (n/a)" +} + +Refresh-TestRoot +$env:X_VCPKG_ASSET_SOURCES = "clear;x-script,pwsh $PSScriptRoot/../e2e-assets/asset-caching/no-file-script.ps1 {url} {sha512} {dst};x-block-origin" +$expected = @( +"^Trying to download example3.html using asset cache script", +"Not creating a file", +"[^\n]+example3\.html\.\d+\.part: error: the asset cache script returned success but did not create expected result file", +"note: the full script command line was: pwsh .+/no-file-script\.ps1 https://example\.com d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a `"?[^`"]+example3\.html\.\d+\.part`"?", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"$" +) -join "`n" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--url", "https://example.com", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a")) +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (no), x-block-origin (yes), asset-cache (hit), download (n/a)" +} + +Refresh-TestRoot +$env:X_VCPKG_ASSET_SOURCES = "clear;x-script,pwsh $PSScriptRoot/../e2e-assets/asset-caching/bad-hash-script.ps1 -File {dst};x-block-origin" +$expected = @( +"^Trying to download example3.html using asset cache script", +"Creating file with the wrong hash", +"[^\n]+example3\.html\.\d+\.part: error: the asset cache script returned success but the resulting file has an unexpected hash", +"note: the full script command line was: pwsh .+/bad-hash-script\.ps1 -File `"?[^`"]+example3\.html\.\d+\.part`"?", +"note: Expected: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", +"note: Actual : cc9c9070d8a54bfc32d6be2eb01b531f22f657d868200fbcdc7c4cc5f31e92909bd7c83971bebefa918c2c34e53d859ed49a79f4a943f36ec521fc0544b30d9e", +"error: there were no asset cache hits, and x-block-origin blocks trying the authoritative source https://example\.com", +"$" +) -join "`n" +$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--url", "https://example.com", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a")) +Throw-IfNotFailed +if (-not ($actual -match $expected)) { + throw "Failure: azurl (no), x-block-origin (yes), asset-cache (hit), download (n/a)" } # Testing x-download success with asset cache (x-script) and x-block-origin settings Refresh-TestRoot +$expected = @( +"^Trying to download example3.html using asset cache script", +"Download successful! Asset cache hit, did not try authoritative source https://example\.com/hello-world.txt", +"$" +) -join "`n" $env:X_VCPKG_ASSET_SOURCES = "clear;x-script,$TestScriptAssetCacheExe {url} {sha512} {dst};x-block-origin" $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--url", "https://example.com/hello-world.txt", "--sha512", "09e1e2a84c92b56c8280f4a1203c7cffd61b162cfe987278d4d6be9afbf38c0e8934cdadf83751f4e99d111352bffefc958e5a4852c8a7a29c95742ce59288a8")) -if (-not ($actual.Contains("Successfully downloaded example3.html."))) { - throw "Failure: x-script download success message" +Throw-IfFailed +if (-not ($actual -match $expected)) { + throw "Success: x-script download success message" } diff --git a/include/vcpkg/base/api-stable-format.h b/include/vcpkg/base/api-stable-format.h index 17094f5058..ae61b87ac9 100644 --- a/include/vcpkg/base/api-stable-format.h +++ b/include/vcpkg/base/api-stable-format.h @@ -1,6 +1,7 @@ #pragma once -#include +#include +#include #include #include @@ -10,21 +11,22 @@ namespace vcpkg namespace details { template - void api_stable_format_cb(void* f, std::string& s, StringView sv) + bool api_stable_format_cb(void* f, std::string& s, StringView sv) { - (*(F*)(f))(s, sv); + return (*(F*)(f))(s, sv); } - ExpectedL api_stable_format_impl(StringView fmtstr, - void (*cb)(void*, std::string&, StringView), - void* data); + Optional api_stable_format_impl(DiagnosticContext& context, + StringView fmtstr, + bool (*cb)(void*, std::string&, StringView), + void* data); } // This function exists in order to provide an API-stable formatting function similar to `std::format()` that does // not depend on the feature set of fmt or the C++ standard library and thus can be contractual for user interfaces. template - ExpectedL api_stable_format(StringView fmtstr, F&& handler) + Optional api_stable_format(DiagnosticContext& context, StringView fmtstr, F&& handler) { - return details::api_stable_format_impl(fmtstr, &details::api_stable_format_cb, &handler); + return details::api_stable_format_impl(context, fmtstr, &details::api_stable_format_cb, &handler); } } diff --git a/include/vcpkg/base/diagnostics.h b/include/vcpkg/base/diagnostics.h index d7d676f5fe..f5f5415109 100644 --- a/include/vcpkg/base/diagnostics.h +++ b/include/vcpkg/base/diagnostics.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -66,11 +67,22 @@ namespace vcpkg std::string to_string() const; void to_string(std::string& target) const; + MessageLine to_message_line() const; + LocalizedString to_json_reader_string(const std::string& path, const LocalizedString& type) const; DiagKind kind() const noexcept { return m_kind; } + // Returns this DiagnosticLine with kind == Error reduced to Warning. + DiagnosticLine reduce_to_warning() const&; + DiagnosticLine reduce_to_warning() &&; private: + DiagnosticLine(DiagKind kind, + const Optional& origin, + TextRowCol position, + const LocalizedString& message); + DiagnosticLine(DiagKind kind, Optional&& origin, TextRowCol position, LocalizedString&& message); + DiagKind m_kind; Optional m_origin; TextRowCol m_position; @@ -79,8 +91,11 @@ namespace vcpkg struct DiagnosticContext { + // The `report` family are used to report errors or warnings that may result in a function failing + // to do what it is intended to do. Data sent to the `report` family is expected to not be printed + // to the console if a caller decides to handle an error. virtual void report(const DiagnosticLine& line) = 0; - virtual void report(DiagnosticLine&& line) { report(line); } + virtual void report(DiagnosticLine&& line); void report_error(const LocalizedString& message) { report(DiagnosticLine{DiagKind::Error, message}); } void report_error(LocalizedString&& message) { report(DiagnosticLine{DiagKind::Error, std::move(message)}); } @@ -92,15 +107,64 @@ namespace vcpkg this->report_error(std::move(message)); } + template + void report_error_with_log(StringView log_content, VCPKG_DECL_MSG_ARGS) + { + LocalizedString message; + msg::format_to(message, VCPKG_EXPAND_MSG_ARGS); + message.append_raw('\n'); + message.append_raw(log_content); + this->report_error(std::move(message)); + } + + void report_system_error(StringLiteral system_api_name, int error_value); + + // The `status` family are used to report status or progress information that callers are expected + // to show on the console, even if it would decide to handle errors or warnings itself. + // Examples: + // * "Downloading file..." + // * "Building package 1 of 47..." + // + // Some implementations of DiagnosticContext may buffer these messages *anyway* if that makes sense, + // for example, if the work is happening on a background thread. + virtual void statusln(const LocalizedString& message) = 0; + virtual void statusln(LocalizedString&& message) = 0; + virtual void statusln(const MessageLine& message) = 0; + virtual void statusln(MessageLine&& message) = 0; + protected: ~DiagnosticContext() = default; }; + struct PrintingDiagnosticContext final : DiagnosticContext + { + PrintingDiagnosticContext(MessageSink& sink) : sink(sink) { } + + virtual void report(const DiagnosticLine& line) override; + + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + private: + MessageSink& sink; + }; + + // Stores all diagnostics into a vector, while passing through status lines to an underlying MessageSink. struct BufferedDiagnosticContext final : DiagnosticContext { + BufferedDiagnosticContext(MessageSink& status_sink) : status_sink(status_sink) { } + virtual void report(const DiagnosticLine& line) override; virtual void report(DiagnosticLine&& line) override; + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + MessageSink& status_sink; std::vector lines; // Prints all diagnostics to the supplied sink. @@ -111,9 +175,75 @@ namespace vcpkg void to_string(std::string& target) const; bool any_errors() const noexcept; + bool empty() const noexcept; + }; + + // Stores all diagnostics and status messages into a vector. This is generally used for background thread or similar + // scenarios where even status messages can't be immediately printed. + struct FullyBufferedDiagnosticContext final : DiagnosticContext + { + virtual void report(const DiagnosticLine& line) override; + virtual void report(DiagnosticLine&& line) override; + + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + std::vector lines; + + // Prints all diagnostics to the supplied sink. + void print_to(MessageSink& sink) const; + // Converts this message into a string + // Prefer print() if possible because it applies color + std::string to_string() const; + void to_string(std::string& target) const; + + bool empty() const noexcept; + }; + + // DiagnosticContext for attempted operations that may be recovered. + // Stores all diagnostics and passes through all status messages. Afterwards, call commit() to report all + // diagnostics to the outer DiagnosticContext, or handle() to forget them. + struct AttemptDiagnosticContext final : DiagnosticContext + { + AttemptDiagnosticContext(DiagnosticContext& inner_context) : inner_context(inner_context) { } + + virtual void report(const DiagnosticLine& line) override; + virtual void report(DiagnosticLine&& line) override; + + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + void commit(); + void handle(); + + ~AttemptDiagnosticContext(); + + DiagnosticContext& inner_context; + std::vector lines; + }; + + // Wraps another DiagnosticContext and reduces the severity of any reported diagnostics to warning from error. + struct WarningDiagnosticContext final : DiagnosticContext + { + WarningDiagnosticContext(DiagnosticContext& inner_context) : inner_context(inner_context) { } + + virtual void report(const DiagnosticLine& line) override; + virtual void report(DiagnosticLine&& line) override; + + virtual void statusln(const LocalizedString& message) override; + virtual void statusln(LocalizedString&& message) override; + virtual void statusln(const MessageLine& message) override; + virtual void statusln(MessageLine&& message) override; + + DiagnosticContext& inner_context; }; extern DiagnosticContext& console_diagnostic_context; + extern DiagnosticContext& status_only_diagnostic_context; extern DiagnosticContext& null_diagnostic_context; // The following overloads are implementing @@ -191,7 +321,7 @@ namespace vcpkg { using Unwrapper = AdaptContextUnwrapOptional>; using ReturnType = ExpectedL; - BufferedDiagnosticContext bdc; + BufferedDiagnosticContext bdc{out_sink}; decltype(auto) maybe_result = functor(bdc, std::forward(args)...); if (auto result = maybe_result.get()) { @@ -261,7 +391,7 @@ namespace vcpkg { using ReturnType = ExpectedL< typename AdaptContextDetectUniquePtr>::type>; - BufferedDiagnosticContext bdc; + BufferedDiagnosticContext bdc{out_sink}; decltype(auto) maybe_result = functor(bdc, std::forward(args)...); if (maybe_result) { diff --git a/include/vcpkg/base/downloads.h b/include/vcpkg/base/downloads.h index ad7fc72124..6349177a90 100644 --- a/include/vcpkg/base/downloads.h +++ b/include/vcpkg/base/downloads.h @@ -15,7 +15,17 @@ namespace vcpkg { - struct SplitURIView + struct SanitizedUrl + { + SanitizedUrl() = default; + SanitizedUrl(StringView raw_url, View secrets); + const std::string& to_string() const noexcept { return m_sanitized_url; } + + private: + std::string m_sanitized_url; + }; + + struct SplitUrlView { StringView scheme; Optional authority; @@ -23,12 +33,7 @@ namespace vcpkg }; // e.g. {"https","//example.org", "/index.html"} - ExpectedL split_uri_view(StringView uri); - - void verify_downloaded_file_hash(const ReadOnlyFilesystem& fs, - StringView sanitized_url, - const Path& downloaded_path, - StringView sha512); + Optional parse_split_url_view(StringView raw_url); View azure_blob_headers(); @@ -36,33 +41,37 @@ namespace vcpkg // -w "PREFIX%{http_code} %{exitcode} %{errormsg}" // with specific handling for curl version < 7.75.0 which does not understand %{exitcode} %{errormsg} // If the line is malformed for any reason, no entry to http_codes is added. - void parse_curl_status_line(std::vector>& http_codes, StringLiteral prefix, StringView this_line); - - std::vector> download_files(View> url_pairs, - View headers, - View secrets); - - bool submit_github_dependency_graph_snapshot(const Optional& maybe_github_server_url, + // Returns: true if the new version of curl's output with exitcode and errormsg was parsed; otherwise, false. + bool parse_curl_status_line(DiagnosticContext& context, + std::vector& http_codes, + StringLiteral prefix, + StringView this_line); + + std::vector download_files_no_cache(DiagnosticContext& context, + View> url_pairs, + View headers, + View secrets); + + bool submit_github_dependency_graph_snapshot(DiagnosticContext& context, + const Optional& maybe_github_server_url, const std::string& github_token, const std::string& github_repository, const Json::Object& snapshot); - ExpectedL put_file(const ReadOnlyFilesystem&, - StringView url, - const std::vector& secrets, - View headers, - const Path& file, - StringView method = "PUT"); - - ExpectedL invoke_http_request(StringView method, - View headers, - StringView url, - StringView data = {}); + + Optional invoke_http_request(DiagnosticContext& context, + StringLiteral method, + View headers, + StringView url, + StringView data = {}); std::string format_url_query(StringView base_url, View query_params); - std::vector> url_heads(View urls, View headers, View secrets); + std::vector url_heads(DiagnosticContext& context, + View urls, + View headers, + View secrets); - struct DownloadManagerConfig + struct AssetCachingSettings { Optional m_read_url_template; std::vector m_read_headers; @@ -74,37 +83,35 @@ namespace vcpkg }; // Handles downloading and uploading to a content addressable mirror - struct DownloadManager - { - DownloadManager() = default; - explicit DownloadManager(const DownloadManagerConfig& config) : m_config(config) { } - explicit DownloadManager(DownloadManagerConfig&& config) : m_config(std::move(config)) { } - - void download_file(const Filesystem& fs, - const std::string& url, - View headers, - const Path& download_path, - const Optional& sha512, - MessageSink& progress_sink) const; - - // Returns url that was successfully downloaded from - std::string download_file(const Filesystem& fs, - View urls, - View headers, - const Path& download_path, - const Optional& sha512, - MessageSink& progress_sink) const; - - ExpectedL put_file_to_mirror(const ReadOnlyFilesystem& fs, - const Path& file_to_put, - StringView sha512) const; - - bool get_block_origin() const; - bool asset_cache_configured() const; - - private: - DownloadManagerConfig m_config; - }; + bool download_file_asset_cached(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const std::string& url, + View headers, + const Path& download_path, + const Optional& maybe_sha512); + + bool download_file_asset_cached(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + View urls, + View headers, + const Path& download_path, + const Optional& maybe_sha512); + + bool store_to_asset_cache(DiagnosticContext& context, + StringView raw_url, + const SanitizedUrl& sanitized_url, + StringLiteral method, + View headers, + const Path& file); + + bool store_to_asset_cache(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Path& file_to_put, + StringView sha512); Optional try_parse_curl_max5_size(StringView sv); @@ -112,8 +119,8 @@ namespace vcpkg { unsigned int total_percent; unsigned long long total_size; - unsigned int recieved_percent; - unsigned long long recieved_size; + unsigned int received_percent; + unsigned long long received_size; unsigned int transfer_percent; unsigned long long transfer_size; unsigned long long average_download_speed; // bytes per second @@ -134,3 +141,5 @@ namespace vcpkg // is likely to contain query parameters or similar. std::string url_encode_spaces(StringView url); } + +VCPKG_FORMAT_WITH_TO_STRING(vcpkg::SanitizedUrl); diff --git a/include/vcpkg/base/file_sink.h b/include/vcpkg/base/file_sink.h index 7f2c20c9ed..ef1730104b 100644 --- a/include/vcpkg/base/file_sink.h +++ b/include/vcpkg/base/file_sink.h @@ -14,6 +14,7 @@ namespace vcpkg : m_log_file(log_file), m_out_file(fs.open_for_write(m_log_file, append_to_file, VCPKG_LINE_INFO)) { } - void print(Color c, StringView sv) override; + void println(const MessageLine& line) override; + void println(MessageLine&& line) override; }; } diff --git a/include/vcpkg/base/fwd/downloads.h b/include/vcpkg/base/fwd/downloads.h index 1f7a59ca3a..c51accb232 100644 --- a/include/vcpkg/base/fwd/downloads.h +++ b/include/vcpkg/base/fwd/downloads.h @@ -2,8 +2,8 @@ namespace vcpkg { - struct SplitURIView; - struct DownloadManager; - struct DownloadManagerConfig; + struct SanitizedUrl; + struct SplitUrlView; + struct AssetCachingSettings; struct CurlProgressData; } diff --git a/include/vcpkg/base/fwd/message_sinks.h b/include/vcpkg/base/fwd/message_sinks.h index 541fa35e6a..6def368015 100644 --- a/include/vcpkg/base/fwd/message_sinks.h +++ b/include/vcpkg/base/fwd/message_sinks.h @@ -2,6 +2,8 @@ namespace vcpkg { + struct MessageLineSegment; + struct MessageLine; struct MessageSink; extern MessageSink& null_sink; @@ -10,5 +12,5 @@ namespace vcpkg extern MessageSink& stderr_sink; struct FileSink; - struct CombiningSink; + struct TeeSink; } diff --git a/include/vcpkg/base/hash.h b/include/vcpkg/base/hash.h index f8a5f71e69..baedbd2b97 100644 --- a/include/vcpkg/base/hash.h +++ b/include/vcpkg/base/hash.h @@ -4,12 +4,26 @@ #include #include +#include #include #include namespace vcpkg::Hash { + enum class HashPrognosis + { + Success, + FileNotFound, + OtherError, + }; + + struct HashResult + { + HashPrognosis prognosis = HashPrognosis::Success; + std::string hash; + }; + enum class Algorithm { Sha256, @@ -33,5 +47,28 @@ namespace vcpkg::Hash std::string get_bytes_hash(const void* first, const void* last, Algorithm algo); std::string get_string_hash(StringView s, Algorithm algo); std::string get_string_sha256(StringView s); - ExpectedL get_file_hash(const ReadOnlyFilesystem& fs, const Path& target, Algorithm algo); + + // Tries to open `path` for reading, and hashes the contents using the requested algorithm. + // Returns a HashResult with the following outcomes: + // HashPrognosis::Success: The entire file was read and hashed. The result hash is stored in `hash`. + // HashPrognosis::FileNotFound: The file does not exist. `hash` is empty string. + // HashPrognosis::OtherError: An error occurred while reading the file. `hash` is empty string. + HashResult get_file_hash(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const Path& path, + Algorithm algo); + + // Tries to open `path` for reading, and hashes the contents using the requested algorithm. + // If the file exists and could be completely read, returns an engaged optional with the stringized hash. + // Otherwise, returns an disengaged optional. + // Note that the file not existing is interpreted as an error that will be reported to `context`. + Optional get_file_hash_required(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const Path& path, + Algorithm algo); + + // Tries to open `path` for reading, and hashes the contents using the requested algorithm. + // If the file exists and could be completely read, returns an engaged optional with the stringized hash. + // Otherwise, returns the read operation error. + ExpectedL get_file_hash(const ReadOnlyFilesystem& fs, const Path& path, Algorithm algo); } diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index f0c4961ad2..c216b38f25 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -248,19 +248,56 @@ DECLARE_MESSAGE(ArtifactsSwitchOsx, (), "", "Forces host detection to MacOS when DECLARE_MESSAGE(ArtifactsSwitchX64, (), "", "Forces host detection to x64 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when acquiring artifacts") DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts") -DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}") -DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}") +DECLARE_MESSAGE(AssetCacheConsult, (msg::path, msg::url), "", "Trying to download {path} using asset cache {url}") +DECLARE_MESSAGE(AssetCacheConsultScript, (msg::path), "", "Trying to download {path} using asset cache script") +DECLARE_MESSAGE(AssetCacheHit, (), "", "Download successful! Asset cache hit.") +DECLARE_MESSAGE(AssetCacheHitUrl, + (msg::url), + "", + "Download successful! Asset cache hit, did not try authoritative source {url}") +DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; trying authoritative source {url}") DECLARE_MESSAGE(AssetCacheMissBlockOrigin, - (msg::path), + (msg::url), "x-block-origin is a vcpkg term. Do not translate", - "Asset cache miss for {path} and downloads are blocked by x-block-origin.") -DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}.") -DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}") + "there were no asset cache hits, and x-block-origin blocks trying the authoritative source {url}") +DECLARE_MESSAGE(AssetCacheMissNoUrls, + (msg::sha), + "", + "Asset cache missed looking for {sha} and no authoritative URL is known") DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments, (msg::value), "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", "unexpected arguments: '{value}' does not accept arguments") -DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (msg::path, msg::url), "", "Successfully stored {path} to {url}.") +DECLARE_MESSAGE(AssetCacheScriptBadVariable, + (msg::value, msg::list), + "{value} is the script template passed to x-script, {list} is the name of the unknown replacement", + "the script template {value} contains unknown replacement {list}") +DECLARE_MESSAGE(AssetCacheScriptBadVariableHint, + (msg::list), + "{list} is the name of the unknown replacement", + "if you want this on the literal command line, use {{{list}}}") +DECLARE_MESSAGE(AssetCacheScriptCommandLine, (), "", "the full script command line was") +DECLARE_MESSAGE(AssetCacheScriptNeedsSha, + (msg::value, msg::url), + "{value} is the script template the user supplied to x-script", + "the script template {value} requires a SHA, but no SHA is known for attempted download of {url}") +DECLARE_MESSAGE(AssetCacheScriptNeedsUrl, + (msg::value, msg::sha), + "{value} is the script template the user supplied to x-script", + "the script template {value} requires a URL, but no URL is known for attempted download of {sha}") +DECLARE_MESSAGE(AssetCacheScriptFailed, + (msg::exit_code), + "", + "the asset cache script returned nonzero exit code {exit_code}") +DECLARE_MESSAGE(AssetCacheScriptFailedToWriteFile, + (), + "", + "the asset cache script returned success but did not create expected result file") +DECLARE_MESSAGE(AssetCacheScriptFailedToWriteCorrectHash, + (), + "", + "the asset cache script returned success but the resulting file has an unexpected hash") +DECLARE_MESSAGE(AssetCacheSuccesfullyStored, (), "", "Store success") DECLARE_MESSAGE(AssetSourcesArg, (), "", "Asset caching sources. See 'vcpkg help assetcaching'") DECLARE_MESSAGE(ASemanticVersionString, (), "", "a semantic version string") DECLARE_MESSAGE(ASetOfFeatures, (), "", "a set of features") @@ -1034,53 +1071,59 @@ DECLARE_MESSAGE(DownloadAvailable, "", "A downloadable copy of this tool is available and can be used by unsetting {env_var}.") DECLARE_MESSAGE(DownloadedSources, (msg::spec), "", "Downloaded sources for {spec}") -DECLARE_MESSAGE(DownloadFailedCurl, - (msg::url, msg::exit_code), - "", - "{url}: curl failed to download with exit code {exit_code}") -DECLARE_MESSAGE(DownloadFailedHashMismatch, - (msg::url, msg::path, msg::expected, msg::actual), - "{expected} and {actual} are SHA512 hashes in hex format.", - "File does not have the expected hash:\n" - "url: {url}\n" - "File: {path}\n" - "Expected hash: {expected}\n" - "Actual hash: {actual}") +DECLARE_MESSAGE(DownloadFailedHashMismatch, (msg::url), "", "download from {url} had an unexpected hash") +DECLARE_MESSAGE(DownloadFailedHashMismatchExpectedHash, (msg::sha), "", "Expected: {sha}") +DECLARE_MESSAGE(DownloadFailedHashMismatchActualHash, (msg::sha), "", "Actual : {sha}") DECLARE_MESSAGE(DownloadFailedRetrying, - (msg::value), + (msg::value, msg::url), "{value} is a number of milliseconds", - "Download failed -- retrying after {value}ms") + "Download {url} failed -- retrying after {value}ms") DECLARE_MESSAGE(DownloadFailedStatusCode, (msg::url, msg::value), "{value} is an HTTP status code", "{url}: failed: status code {value}") DECLARE_MESSAGE(DownloadFailedProxySettings, - (msg::path, msg::url), + (), "", - "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are " - "correct.\nPossible causes are:\n" - "1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable " - "to `https//address:port`.\nThis is not correct, because `https://` prefix " - "claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\n" + "If you are using a proxy, please ensure your proxy settings are correct.\n" + "Possible causes are:\n" + "1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to " + "`https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS " + "proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\n" "Try setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n" - "2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings " - "set by your proxy software. See, {url}\n" + "2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your " + "proxy software. See: https://github.com/microsoft/vcpkg-tool/pull/77\n" "The value set by your proxy might be wrong, or have same `https://` prefix issue.\n" "3. Your proxy's remote server is our of service.\n" - "If you've tried directly download the link, and believe this is not a temporay download server " + "If you've tried directly download the link, and believe this is not a temporary download server " "failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\n" "to report this upstream download server failure.") DECLARE_MESSAGE(DownloadingPortableToolVersionX, (msg::tool_name, msg::version), "", "A suitable version of {tool_name} was not found (required v{version}).") -DECLARE_MESSAGE(DownloadWinHttpError, - (msg::system_api, msg::exit_code, msg::url), - "", - "{url}: {system_api} failed with exit code {exit_code}") +DECLARE_MESSAGE(DownloadingAssetShaToFile, (msg::sha, msg::path), "", "Downloading asset cache entry {sha} -> {path}") +DECLARE_MESSAGE(DownloadingAssetShaWithoutAssetCache, + (msg::sha, msg::path), + "", + "requested download of asset cache entry {sha} -> {path}, but no asset caches are configured") +DECLARE_MESSAGE(DownloadingFile, (msg::path), "", "Downloading {path}") +DECLARE_MESSAGE(DownloadingFileFirstAuthoritativeSource, (msg::path, msg::url), "", "Downloading {path}, trying {url}") +DECLARE_MESSAGE(DownloadingUrlToFile, (msg::url, msg::path), "", "Downloading {url} -> {path}") DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundle, (msg::version), "", "Downloading standalone bundle {version}.") DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundleLatest, (), "", "Downloading latest standalone bundle.") +DECLARE_MESSAGE(DownloadOrUrl, (msg::url), "", "or {url}") +DECLARE_MESSAGE(DownloadTryingAuthoritativeSource, (msg::url), "", "Trying {url}") DECLARE_MESSAGE(DownloadRootsDir, (msg::env_var), "", "Downloads directory (default: {env_var})") +DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}") +DECLARE_MESSAGE(DownloadSuccesfulUploading, + (msg::path, msg::url), + "", + "Successfully downloaded {path}, storing to {url}") +DECLARE_MESSAGE(DownloadWinHttpError, + (msg::system_api, msg::exit_code, msg::url), + "", + "{url}: {system_api} failed with exit code {exit_code}.") DECLARE_MESSAGE(DuplicatedKeyInObj, (msg::value), "{value} is a json property/object", @@ -1265,10 +1308,6 @@ DECLARE_MESSAGE(FailedToDeleteInsideDueToFile, "printed after this", "failed to remove_all_inside({value}) due to {path}: ") DECLARE_MESSAGE(FailedToDetermineCurrentCommit, (), "", "Failed to determine the current commit:") -DECLARE_MESSAGE(MissingAssetBlockOrigin, - (msg::path), - "x-block-origin is a vcpkg term. Do not translate", - "Missing {path} and downloads are blocked by x-block-origin.") DECLARE_MESSAGE(MissingShaVariable, (), "{{sha}} should not be translated", @@ -1356,7 +1395,7 @@ DECLARE_MESSAGE(FetchingRegistryInfo, (msg::url, msg::value), "{value} is a reference", "Fetching registry information from {url} ({value})...") -DECLARE_MESSAGE(FileNotFound, (msg::path), "", "{path}: file not found") +DECLARE_MESSAGE(FileNotFound, (), "", "file not found") DECLARE_MESSAGE(FileReadFailed, (msg::path, msg::byte_offset, msg::count), "", @@ -1431,10 +1470,6 @@ DECLARE_MESSAGE(GraphCycleDetected, (msg::package_name), "A list of package names comprising the cycle will be printed after this message.", "Cycle detected within graph at {package_name}:") -DECLARE_MESSAGE(HashFileFailureToRead, - (msg::path), - "Printed after ErrorMessage and before the specific failing filesystem operation (like file not found)", - "failed to read file \"{path}\" for hashing: ") DECLARE_MESSAGE(HashPortManyFiles, (msg::package_name, msg::count), "", @@ -2216,7 +2251,6 @@ DECLARE_MESSAGE(NonZeroRemainingArgs, "the command '{command_name}' does not accept any additional arguments") DECLARE_MESSAGE(NoOutdatedPackages, (), "", "There are no outdated packages.") DECLARE_MESSAGE(NoRegistryForPort, (msg::package_name), "", "no registry configured for port {package_name}") -DECLARE_MESSAGE(NoUrlsAndHashSpecified, (msg::sha), "", "No urls specified to download SHA: {sha}") DECLARE_MESSAGE(NoUrlsAndNoHashSpecified, (), "", "No urls specified and no hash specified.") DECLARE_MESSAGE(NugetOutputNotCapturedBecauseInteractiveSpecified, (), @@ -2613,6 +2647,7 @@ DECLARE_MESSAGE(ProgramReturnedNonzeroExitCode, (msg::tool_name, msg::exit_code), "The program's console output is appended after this.", "{tool_name} failed with exit code: ({exit_code}).") +DECLARE_MESSAGE(ProgramPathReturnedNonzeroExitCode, (msg::exit_code), "", "failed with exit code {exit_code}") DECLARE_MESSAGE( ProvideExportType, (), @@ -2752,6 +2787,11 @@ DECLARE_MESSAGE(SystemRootMustAlwaysBePresent, "", "Expected the SystemRoot environment variable to be always set on Windows.") DECLARE_MESSAGE(SystemTargetsInstallFailed, (msg::path), "", "failed to install system targets file to {path}") +DECLARE_MESSAGE( + ToolHashMismatch, + (msg::tool_name, msg::expected, msg::actual), + "{expected} and {actual} are SHA512 hashes in hex format.", + "{tool_name} appears to be already downloaded, but has an incorrect hash. Expected {expected} but was {actual}") DECLARE_MESSAGE(ToolFetchFailed, (msg::tool_name), "", "Could not fetch {tool_name}.") DECLARE_MESSAGE(ToolInWin10, (), "", "This utility is bundled with Windows 10 or later.") DECLARE_MESSAGE(ToolOfVersionXNotFound, @@ -3240,6 +3280,7 @@ DECLARE_MESSAGE(WhileParsingVersionsForPort, (msg::package_name, msg::path), "", "while parsing versions for {package_name} from {path}") +DECLARE_MESSAGE(WhileRunningAssetCacheScriptCommandLine, (), "", "while running asset cache script command line") DECLARE_MESSAGE(WhileValidatingVersion, (msg::version), "", "while validating version: {version}") DECLARE_MESSAGE(WindowsOnlyCommand, (), "", "This command only supports Windows.") DECLARE_MESSAGE(WroteNuGetPkgConfInfo, (msg::path), "", "Wrote NuGet package config information to {path}") diff --git a/include/vcpkg/base/message_sinks.h b/include/vcpkg/base/message_sinks.h index 4d854364a0..85cd345840 100644 --- a/include/vcpkg/base/message_sinks.h +++ b/include/vcpkg/base/message_sinks.h @@ -4,56 +4,54 @@ #include +#include +#include + namespace vcpkg { + struct MessageLineSegment + { + Color color; + std::string text; + }; + + struct MessageLine + { + MessageLine() = default; + MessageLine(const MessageLine&) = default; + MessageLine(MessageLine&&) = default; + + explicit MessageLine(const LocalizedString& ls); + explicit MessageLine(LocalizedString&& ls); + + void print(Color color, StringView text); + void print(StringView text); + const std::vector& get_segments() const noexcept; + + std::string to_string() const; + void to_string(std::string& target) const; + + private: + std::vector segments; + }; struct MessageSink { - virtual void print(Color c, StringView sv) = 0; + virtual void println(const MessageLine& line) = 0; + virtual void println(MessageLine&& line) = 0; - void println() { this->print(Color::none, "\n"); } - void print(const LocalizedString& s) { this->print(Color::none, s); } - void println(Color c, const LocalizedString& s) - { - this->print(c, s); - this->print(Color::none, "\n"); - } - inline void println(const LocalizedString& s) - { - this->print(Color::none, s); - this->print(Color::none, "\n"); - } - inline void println(Color c, LocalizedString&& s) { this->print(c, s.append_raw('\n')); } - inline void println(LocalizedString&& s) { this->print(Color::none, s.append_raw('\n')); } - void println_warning(const LocalizedString& s); - void println_error(const LocalizedString& s); + virtual void println(const LocalizedString& s); + virtual void println(LocalizedString&& s); + + virtual void println(Color c, const LocalizedString& s); + virtual void println(Color c, LocalizedString&& s); - template - void print(VCPKG_DECL_MSG_ARGS) - { - this->print(msg::format(VCPKG_EXPAND_MSG_ARGS)); - } template void println(VCPKG_DECL_MSG_ARGS) { this->println(msg::format(VCPKG_EXPAND_MSG_ARGS)); } - template - void println_warning(VCPKG_DECL_MSG_ARGS) - { - this->println_warning(msg::format(VCPKG_EXPAND_MSG_ARGS)); - } - template - void println_error(VCPKG_DECL_MSG_ARGS) - { - this->println_error(msg::format(VCPKG_EXPAND_MSG_ARGS)); - } - template - void print(Color c, VCPKG_DECL_MSG_ARGS) - { - this->print(c, msg::format(VCPKG_EXPAND_MSG_ARGS)); - } template void println(Color c, VCPKG_DECL_MSG_ARGS) { @@ -68,11 +66,17 @@ namespace vcpkg ~MessageSink() = default; }; - struct CombiningSink : MessageSink + struct TeeSink final : MessageSink { MessageSink& m_first; MessageSink& m_second; - CombiningSink(MessageSink& first, MessageSink& second) : m_first(first), m_second(second) { } - void print(Color c, StringView sv) override; + TeeSink(MessageSink& first, MessageSink& second) : m_first(first), m_second(second) { } + + virtual void println(const MessageLine& line) override; + virtual void println(MessageLine&& line) override; + virtual void println(const LocalizedString& line) override; + virtual void println(LocalizedString&& line) override; + virtual void println(Color color, const LocalizedString& line) override; + virtual void println(Color color, LocalizedString&& line) override; }; } diff --git a/include/vcpkg/base/strings.h b/include/vcpkg/base/strings.h index af7ca77acf..236d5c8b56 100644 --- a/include/vcpkg/base/strings.h +++ b/include/vcpkg/base/strings.h @@ -129,6 +129,7 @@ namespace vcpkg::Strings void inplace_ascii_to_lowercase(std::string& s); [[nodiscard]] std::string ascii_to_lowercase(StringView s); [[nodiscard]] std::string ascii_to_uppercase(StringView s); + void append_ascii_lowercase(std::string& target, StringView s); bool case_insensitive_ascii_starts_with(StringView s, StringView pattern); bool case_insensitive_ascii_ends_with(StringView s, StringView pattern); diff --git a/include/vcpkg/base/system.process.h b/include/vcpkg/base/system.process.h index e362dd997c..198d2e1f3f 100644 --- a/include/vcpkg/base/system.process.h +++ b/include/vcpkg/base/system.process.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -119,8 +120,23 @@ namespace vcpkg std::string stdin_content; }; - ExpectedL cmd_execute(const Command& cmd); - ExpectedL cmd_execute(const Command& cmd, const ProcessLaunchSettings& settings); + Optional cmd_execute(DiagnosticContext& context, const Command& cmd); + inline ExpectedL cmd_execute(const Command& cmd) + { + return adapt_context_to_expected( + static_cast (*)(DiagnosticContext&, const Command&)>(cmd_execute), cmd); + } + Optional cmd_execute(DiagnosticContext& context, + const Command& cmd, + const ProcessLaunchSettings& settings); + inline ExpectedL cmd_execute(const Command& cmd, const ProcessLaunchSettings& settings) + { + return adapt_context_to_expected( + static_cast (*)( + DiagnosticContext&, const Command&, const ProcessLaunchSettings&)>(cmd_execute), + cmd, + settings); + } #if defined(_WIN32) Environment cmd_execute_and_capture_environment(const Command& cmd, const Environment& env); @@ -128,25 +144,97 @@ namespace vcpkg void cmd_execute_background(const Command& cmd_line); - ExpectedL cmd_execute_and_capture_output(const Command& cmd); - ExpectedL cmd_execute_and_capture_output(const Command& cmd, - const RedirectedProcessLaunchSettings& settings); + Optional cmd_execute_and_capture_output(DiagnosticContext& context, const Command& cmd); + inline ExpectedL cmd_execute_and_capture_output(const Command& cmd) + { + return adapt_context_to_expected( + static_cast (*)(DiagnosticContext&, const Command&)>( + cmd_execute_and_capture_output), + cmd); + } + Optional cmd_execute_and_capture_output(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings); + inline ExpectedL cmd_execute_and_capture_output(const Command& cmd, + const RedirectedProcessLaunchSettings& settings) + { + return adapt_context_to_expected( + static_cast (*)( + DiagnosticContext&, const Command&, const RedirectedProcessLaunchSettings&)>( + cmd_execute_and_capture_output), + cmd, + settings); + } std::vector> cmd_execute_and_capture_output_parallel(View commands); std::vector> cmd_execute_and_capture_output_parallel( View commands, const RedirectedProcessLaunchSettings& settings); - ExpectedL cmd_execute_and_stream_lines(const Command& cmd, - const std::function& per_line_cb); - ExpectedL cmd_execute_and_stream_lines(const Command& cmd, - const RedirectedProcessLaunchSettings& settings, - const std::function& per_line_cb); - - ExpectedL cmd_execute_and_stream_data(const Command& cmd, - const std::function& data_cb); - ExpectedL cmd_execute_and_stream_data(const Command& cmd, + Optional cmd_execute_and_stream_lines(DiagnosticContext& context, + const Command& cmd, + const std::function& per_line_cb); + inline ExpectedL cmd_execute_and_stream_lines(const Command& cmd, + const std::function& per_line_cb) + { + return adapt_context_to_expected( + static_cast (*)( + DiagnosticContext&, const Command&, const std::function&)>( + cmd_execute_and_stream_lines), + cmd, + per_line_cb); + } + Optional cmd_execute_and_stream_lines(DiagnosticContext& context, + const Command& cmd, const RedirectedProcessLaunchSettings& settings, - const std::function& data_cb); + const std::function& per_line_cb); + inline ExpectedL cmd_execute_and_stream_lines(const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& per_line_cb) + { + return adapt_context_to_expected( + static_cast (*)(DiagnosticContext&, + const Command&, + const RedirectedProcessLaunchSettings&, + const std::function&)>( + cmd_execute_and_stream_lines), + cmd, + settings, + per_line_cb); + } + + Optional cmd_execute_and_stream_data(DiagnosticContext& context, + const Command& cmd, + const std::function& data_cb); + inline ExpectedL cmd_execute_and_stream_data(const Command& cmd, + const std::function& data_cb) + { + return adapt_context_to_expected( + static_cast (*)( + DiagnosticContext&, const Command&, const std::function&)>( + cmd_execute_and_stream_data), + cmd, + data_cb); + } + + Optional cmd_execute_and_stream_data(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& data_cb); + + inline ExpectedL cmd_execute_and_stream_data(const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& data_cb) + { + return adapt_context_to_expected( + static_cast (*)(DiagnosticContext&, + const Command&, + const RedirectedProcessLaunchSettings&, + const std::function&)>( + cmd_execute_and_stream_data), + cmd, + settings, + data_cb); + } uint64_t get_subproccess_stats(); @@ -170,9 +258,15 @@ namespace vcpkg // If exit code is 0, returns a 'success' ExpectedL. // Otherwise, returns an ExpectedL containing error text - ExpectedL flatten(const ExpectedL&, StringView tool_name); + ExpectedL flatten(const ExpectedL& maybe_exit, StringView tool_name); // If exit code is 0, returns a 'success' ExpectedL containing the output // Otherwise, returns an ExpectedL containing error text - ExpectedL flatten_out(ExpectedL&&, StringView tool_name); + ExpectedL flatten_out(ExpectedL&& maybe_exit, StringView tool_name); + + // Checks that `maybe_exit` implies a process that returned 0. If so, returns a pointer to the process' output. + // Otherwise, records an error in `context` and returns nullptr. + std::string* check_zero_exit_code(DiagnosticContext& context, + Optional& maybe_exit, + StringView exe_path); } diff --git a/include/vcpkg/binarycaching.h b/include/vcpkg/binarycaching.h index f9057847df..dbd0e82a6c 100644 --- a/include/vcpkg/binarycaching.h +++ b/include/vcpkg/binarycaching.h @@ -225,7 +225,7 @@ namespace vcpkg bool m_needs_zip_file = false; }; - ExpectedL parse_download_configuration(const Optional& arg); + ExpectedL parse_download_configuration(const Optional& arg); std::string generate_nuget_packages_config(const ActionPlan& action, StringView prefix); diff --git a/include/vcpkg/configure-environment.h b/include/vcpkg/configure-environment.h index 74807999f6..3f53416a30 100644 --- a/include/vcpkg/configure-environment.h +++ b/include/vcpkg/configure-environment.h @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include @@ -16,9 +18,10 @@ namespace vcpkg { - ExpectedL download_vcpkg_standalone_bundle(const DownloadManager& download_manager, - const Filesystem& fs, - const Path& download_root); + Optional download_vcpkg_standalone_bundle(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const Path& download_root); int run_configure_environment_command(const VcpkgPaths& paths, View args); diff --git a/include/vcpkg/tools.h b/include/vcpkg/tools.h index 6542f324e3..32b5860a02 100644 --- a/include/vcpkg/tools.h +++ b/include/vcpkg/tools.h @@ -60,7 +60,7 @@ namespace vcpkg ExpectedL find_system_cmake(const ReadOnlyFilesystem& fs); std::unique_ptr get_tool_cache(const Filesystem& fs, - std::shared_ptr downloader, + const AssetCachingSettings& asset_cache_settings, Path downloads, Path config_path, Path tools, diff --git a/include/vcpkg/vcpkgpaths.h b/include/vcpkg/vcpkgpaths.h index ea7de94962..c764a418dc 100644 --- a/include/vcpkg/vcpkgpaths.h +++ b/include/vcpkg/vcpkgpaths.h @@ -108,7 +108,7 @@ namespace vcpkg std::string get_toolver_diagnostics() const; const Filesystem& get_filesystem() const; - const DownloadManager& get_download_manager() const; + const AssetCachingSettings& get_asset_cache_settings() const; const ToolCache& get_tool_cache() const; const Path& get_tool_exe(StringView tool, MessageSink& status_messages) const; const std::string& get_tool_version(StringView tool, MessageSink& status_messages) const; diff --git a/locales/messages.json b/locales/messages.json index 6527ae9212..fc195de5f8 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -178,16 +178,35 @@ "ArtifactsSwitchWindows": "Forces host detection to Windows when acquiring artifacts", "ArtifactsSwitchX64": "Forces host detection to x64 when acquiring artifacts", "ArtifactsSwitchX86": "Forces host detection to x86 when acquiring artifacts", - "AssetCacheHit": "Asset cache hit for {path}; downloaded from: {url}", - "_AssetCacheHit.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", - "AssetCacheMiss": "Asset cache miss; downloading from {url}", + "AssetCacheConsult": "Trying to download {path} using asset cache {url}", + "_AssetCacheConsult.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheConsultScript": "Trying to download {path} using asset cache script", + "_AssetCacheConsultScript.comment": "An example of {path} is /foo/bar.", + "AssetCacheHit": "Download successful! Asset cache hit.", + "AssetCacheHitUrl": "Download successful! Asset cache hit, did not try authoritative source {url}", + "_AssetCacheHitUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheMiss": "Asset cache miss; trying authoritative source {url}", "_AssetCacheMiss.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", - "AssetCacheMissBlockOrigin": "Asset cache miss for {path} and downloads are blocked by x-block-origin.", - "_AssetCacheMissBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", + "AssetCacheMissBlockOrigin": "there were no asset cache hits, and x-block-origin blocks trying the authoritative source {url}", + "_AssetCacheMissBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheMissNoUrls": "Asset cache missed looking for {sha} and no authoritative URL is known", + "_AssetCacheMissNoUrls.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", "AssetCacheProviderAcceptsNoArguments": "unexpected arguments: '{value}' does not accept arguments", "_AssetCacheProviderAcceptsNoArguments.comment": "{value} is a asset caching provider name such as azurl, clear, or x-block-origin", - "AssetCacheSuccesfullyStored": "Successfully stored {path} to {url}.", - "_AssetCacheSuccesfullyStored.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheScriptBadVariable": "the script template {value} contains unknown replacement {list}", + "_AssetCacheScriptBadVariable.comment": "{value} is the script template passed to x-script, {list} is the name of the unknown replacement", + "AssetCacheScriptBadVariableHint": "if you want this on the literal command line, use {{{list}}}", + "_AssetCacheScriptBadVariableHint.comment": "{list} is the name of the unknown replacement", + "AssetCacheScriptCommandLine": "the full script command line was", + "AssetCacheScriptFailed": "the asset cache script returned nonzero exit code {exit_code}", + "_AssetCacheScriptFailed.comment": "An example of {exit_code} is 127.", + "AssetCacheScriptFailedToWriteCorrectHash": "the asset cache script returned success but the resulting file has an unexpected hash", + "AssetCacheScriptFailedToWriteFile": "the asset cache script returned success but did not create expected result file", + "AssetCacheScriptNeedsSha": "the script template {value} requires a SHA, but no SHA is known for attempted download of {url}", + "_AssetCacheScriptNeedsSha.comment": "{value} is the script template the user supplied to x-script An example of {url} is https://github.com/microsoft/vcpkg.", + "AssetCacheScriptNeedsUrl": "the script template {value} requires a URL, but no URL is known for attempted download of {sha}", + "_AssetCacheScriptNeedsUrl.comment": "{value} is the script template the user supplied to x-script An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", + "AssetCacheSuccesfullyStored": "Store success", "AssetSourcesArg": "Asset caching sources. See 'vcpkg help assetcaching'", "AttemptingToSetBuiltInBaseline": "attempting to set builtin-baseline in vcpkg.json while overriding the default-registry in vcpkg-configuration.json.\nthe default-registry from vcpkg-configuration.json will be used.", "AuthenticationMayRequireManualAction": "One or more {vendor} credential providers requested manual action. Add the binary source 'interactive' to allow interactivity.", @@ -594,28 +613,43 @@ "DocumentedFieldsSuggestUpdate": "If these are documented fields that should be recognized try updating the vcpkg tool.", "DownloadAvailable": "A downloadable copy of this tool is available and can be used by unsetting {env_var}.", "_DownloadAvailable.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", - "DownloadFailedCurl": "{url}: curl failed to download with exit code {exit_code}", - "_DownloadFailedCurl.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {exit_code} is 127.", - "DownloadFailedHashMismatch": "File does not have the expected hash:\nurl: {url}\nFile: {path}\nExpected hash: {expected}\nActual hash: {actual}", - "_DownloadFailedHashMismatch.comment": "{expected} and {actual} are SHA512 hashes in hex format. An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.", - "DownloadFailedProxySettings": "Failed to download {path}.\nIf you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See, {url}\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is our of service.\nIf you've tried directly download the link, and believe this is not a temporay download server failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\nto report this upstream download server failure.", - "_DownloadFailedProxySettings.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", - "DownloadFailedRetrying": "Download failed -- retrying after {value}ms", - "_DownloadFailedRetrying.comment": "{value} is a number of milliseconds", + "DownloadFailedHashMismatch": "download from {url} had an unexpected hash", + "_DownloadFailedHashMismatch.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadFailedHashMismatchActualHash": "Actual : {sha}", + "_DownloadFailedHashMismatchActualHash.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", + "DownloadFailedHashMismatchExpectedHash": "Expected: {sha}", + "_DownloadFailedHashMismatchExpectedHash.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", + "DownloadFailedProxySettings": "If you are using a proxy, please ensure your proxy settings are correct.\nPossible causes are:\n1. You are actually using an HTTP proxy, but setting HTTPS_PROXY variable to `https//address:port`.\nThis is not correct, because `https://` prefix claims the proxy is an HTTPS proxy, while your proxy (v2ray, shadowsocksr, etc...) is an HTTP proxy.\nTry setting `http://address:port` to both HTTP_PROXY and HTTPS_PROXY instead.\n2. If you are using Windows, vcpkg will automatically use your Windows IE Proxy Settings set by your proxy software. See: https://github.com/microsoft/vcpkg-tool/pull/77\nThe value set by your proxy might be wrong, or have same `https://` prefix issue.\n3. Your proxy's remote server is our of service.\nIf you've tried directly download the link, and believe this is not a temporary download server failure, please submit an issue at https://github.com/Microsoft/vcpkg/issues\nto report this upstream download server failure.", + "DownloadFailedRetrying": "Download {url} failed -- retrying after {value}ms", + "_DownloadFailedRetrying.comment": "{value} is a number of milliseconds An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadFailedStatusCode": "{url}: failed: status code {value}", "_DownloadFailedStatusCode.comment": "{value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadOrUrl": "or {url}", + "_DownloadOrUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadRootsDir": "Downloads directory (default: {env_var})", "_DownloadRootsDir.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.", - "DownloadSuccesful": "Successfully downloaded {path}.", + "DownloadSuccesful": "Successfully downloaded {path}", "_DownloadSuccesful.comment": "An example of {path} is /foo/bar.", - "DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}", + "DownloadSuccesfulUploading": "Successfully downloaded {path}, storing to {url}", + "_DownloadSuccesfulUploading.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadTryingAuthoritativeSource": "Trying {url}", + "_DownloadTryingAuthoritativeSource.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}.", "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadedSources": "Downloaded sources for {spec}", "_DownloadedSources.comment": "An example of {spec} is zlib:x64-windows.", + "DownloadingAssetShaToFile": "Downloading asset cache entry {sha} -> {path}", + "_DownloadingAssetShaToFile.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e. An example of {path} is /foo/bar.", + "DownloadingAssetShaWithoutAssetCache": "requested download of asset cache entry {sha} -> {path}, but no asset caches are configured", + "_DownloadingAssetShaWithoutAssetCache.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e. An example of {path} is /foo/bar.", + "DownloadingFile": "Downloading {path}", + "_DownloadingFile.comment": "An example of {path} is /foo/bar.", + "DownloadingFileFirstAuthoritativeSource": "Downloading {path}, trying {url}", + "_DownloadingFileFirstAuthoritativeSource.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.", "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}).", "_DownloadingPortableToolVersionX.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", - "DownloadingUrl": "Downloading {url}", - "_DownloadingUrl.comment": "An example of {url} is https://github.com/microsoft/vcpkg.", + "DownloadingUrlToFile": "Downloading {url} -> {path}", + "_DownloadingUrlToFile.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.", "DownloadingVcpkgStandaloneBundle": "Downloading standalone bundle {version}.", "_DownloadingVcpkgStandaloneBundle.comment": "An example of {version} is 1.3.8.", "DownloadingVcpkgStandaloneBundleLatest": "Downloading latest standalone bundle.", @@ -788,8 +822,7 @@ "FieldKindDidNotHaveExpectedValue": "\"kind\" did not have an expected value: (expected one of: {expected}; found {actual})", "_FieldKindDidNotHaveExpectedValue.comment": "{expected} is a list of literal kinds the user must type, separated by commas, {actual} is what the user supplied", "FileIsNotExecutable": "this file does not appear to be executable", - "FileNotFound": "{path}: file not found", - "_FileNotFound.comment": "An example of {path} is /foo/bar.", + "FileNotFound": "file not found", "FileReadFailed": "Failed to read {count} bytes from {path} at offset {byte_offset}.", "_FileReadFailed.comment": "An example of {path} is /foo/bar. An example of {byte_offset} is 42. An example of {count} is 42.", "FileSeekFailed": "Failed to seek to position {byte_offset} in {path}.", @@ -850,8 +883,6 @@ "_GitUnexpectedCommandOutputCmd.comment": "An example of {command_line} is vcpkg install zlib.", "GraphCycleDetected": "Cycle detected within graph at {package_name}:", "_GraphCycleDetected.comment": "A list of package names comprising the cycle will be printed after this message. An example of {package_name} is zlib.", - "HashFileFailureToRead": "failed to read file \"{path}\" for hashing: ", - "_HashFileFailureToRead.comment": "Printed after ErrorMessage and before the specific failing filesystem operation (like file not found) An example of {path} is /foo/bar.", "HashPortManyFiles": "{package_name} contains {count} files. Hashing these contents may take a long time when determining the ABI hash for binary caching. Consider reducing the number of files. Common causes of this are accidentally checking out source or build files into a port's directory.", "_HashPortManyFiles.comment": "An example of {package_name} is zlib. An example of {count} is 42.", "HeaderOnlyUsage": "{package_name} is header-only and can be used from CMake via:", @@ -1163,8 +1194,6 @@ "MissingAndroidHomeDir": "ANDROID_NDK_HOME directory does not exist: {path}", "_MissingAndroidHomeDir.comment": "An example of {path} is /foo/bar.", "MissingArgFormatManifest": "format-manifest was passed --convert-control without '--all'.\nThis doesn't do anything: control files passed explicitly are converted automatically.", - "MissingAssetBlockOrigin": "Missing {path} and downloads are blocked by x-block-origin.", - "_MissingAssetBlockOrigin.comment": "x-block-origin is a vcpkg term. Do not translate An example of {path} is /foo/bar.", "MissingClosingParen": "missing closing )", "MissingDependency": "Package {spec} is installed, but dependency {package_name} is not.", "_MissingDependency.comment": "An example of {spec} is zlib:x64-windows. An example of {package_name} is zlib.", @@ -1202,8 +1231,6 @@ "NoOutdatedPackages": "There are no outdated packages.", "NoRegistryForPort": "no registry configured for port {package_name}", "_NoRegistryForPort.comment": "An example of {package_name} is zlib.", - "NoUrlsAndHashSpecified": "No urls specified to download SHA: {sha}", - "_NoUrlsAndHashSpecified.comment": "An example of {sha} is eb32643dd2164c72b8a660ef52f1e701bb368324ae461e12d70d6a9aefc0c9573387ee2ed3828037ed62bb3e8f566416a2d3b3827a3928f0bff7c29f7662293e.", "NoUrlsAndNoHashSpecified": "No urls specified and no hash specified.", "NonExactlyArgs": "the command '{command_name}' requires exactly {expected} arguments, but {actual} were provided", "_NonExactlyArgs.comment": "{expected} and {actual} are integers An example of {command_name} is install.", @@ -1366,6 +1393,8 @@ "_PortsUpdated.comment": "An example of {count} is 42.", "PrebuiltPackages": "There are packages that have not been built. To build them run:", "PreviousIntegrationFileRemains": "Previous integration file was not removed.", + "ProgramPathReturnedNonzeroExitCode": "failed with exit code {exit_code}", + "_ProgramPathReturnedNonzeroExitCode.comment": "An example of {exit_code} is 127.", "ProgramReturnedNonzeroExitCode": "{tool_name} failed with exit code: ({exit_code}).", "_ProgramReturnedNonzeroExitCode.comment": "The program's console output is appended after this. An example of {tool_name} is aria2. An example of {exit_code} is 127.", "ProvideExportType": "At least one of the following options are required: --raw --nuget --ifw --zip --7zip --chocolatey --prefab.", @@ -1451,6 +1480,8 @@ "_ToolDataFileSchemaVersionNotSupported.comment": "An example of {version} is 1.3.8.", "ToolFetchFailed": "Could not fetch {tool_name}.", "_ToolFetchFailed.comment": "An example of {tool_name} is aria2.", + "ToolHashMismatch": "{tool_name} appears to be already downloaded, but has an incorrect hash. Expected {expected} but was {actual}", + "_ToolHashMismatch.comment": "{expected} and {actual} are SHA512 hashes in hex format. An example of {tool_name} is aria2.", "ToolInWin10": "This utility is bundled with Windows 10 or later.", "ToolOfVersionXNotFound": "A suitable version of {tool_name} was not found (required v{version}) and unable to automatically download a portable one. Please install a newer version of {tool_name}", "_ToolOfVersionXNotFound.comment": "An example of {tool_name} is aria2. An example of {version} is 1.3.8.", @@ -1706,6 +1737,7 @@ "_WhileLookingForSpec.comment": "An example of {spec} is zlib:x64-windows.", "WhileParsingVersionsForPort": "while parsing versions for {package_name} from {path}", "_WhileParsingVersionsForPort.comment": "An example of {package_name} is zlib. An example of {path} is /foo/bar.", + "WhileRunningAssetCacheScriptCommandLine": "while running asset cache script command line", "WhileValidatingVersion": "while validating version: {version}", "_WhileValidatingVersion.comment": "An example of {version} is 1.3.8.", "WindowsOnlyCommand": "This command only supports Windows.", diff --git a/src/vcpkg-test/configparser.cpp b/src/vcpkg-test/configparser.cpp index cc8832b344..8e1a60d367 100644 --- a/src/vcpkg-test/configparser.cpp +++ b/src/vcpkg-test/configparser.cpp @@ -614,19 +614,19 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") CHECK(parse_download_configuration("x-azurl,ftp://magic,none")); { - DownloadManagerConfig empty; + AssetCachingSettings empty; CHECK(empty.m_write_headers.empty()); CHECK(empty.m_read_headers.empty()); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123,foo").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/?foo"); CHECK(dm.m_read_headers.empty()); CHECK(dm.m_write_url_template == nullopt); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123/,foo").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/?foo"); CHECK(dm.m_read_headers.empty()); @@ -634,7 +634,7 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") CHECK(dm.m_secrets == std::vector{"foo"}); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123,?foo").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/?foo"); CHECK(dm.m_read_headers.empty()); @@ -642,14 +642,14 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") CHECK(dm.m_secrets == std::vector{"?foo"}); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/"); CHECK(dm.m_read_headers.empty()); CHECK(dm.m_write_url_template == nullopt); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123,,readwrite").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/"); CHECK(dm.m_read_headers.empty()); @@ -657,7 +657,7 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") Test::check_ranges(dm.m_write_headers, azure_blob_headers()); } { - DownloadManagerConfig dm = + AssetCachingSettings dm = parse_download_configuration("x-azurl,https://abc/123,foo,readwrite").value_or_exit(VCPKG_LINE_INFO); CHECK(dm.m_read_url_template == "https://abc/123/?foo"); CHECK(dm.m_read_headers.empty()); @@ -665,6 +665,16 @@ TEST_CASE ("AssetConfigParser azurl provider", "[assetconfigparser]") Test::check_ranges(dm.m_write_headers, azure_blob_headers()); CHECK(dm.m_secrets == std::vector{"foo"}); } + { + AssetCachingSettings dm = + parse_download_configuration("x-script,powershell {SHA} {URL}").value_or_exit(VCPKG_LINE_INFO); + CHECK(!dm.m_read_url_template.has_value()); + CHECK(dm.m_read_headers.empty()); + CHECK(!dm.m_write_url_template.has_value()); + CHECK(dm.m_write_headers.empty()); + CHECK(dm.m_secrets.empty()); + CHECK(dm.m_script.value_or_exit(VCPKG_LINE_INFO) == "powershell {SHA} {URL}"); + } } TEST_CASE ("AssetConfigParser clear provider", "[assetconfigparser]") @@ -679,7 +689,7 @@ TEST_CASE ("AssetConfigParser clear provider", "[assetconfigparser]") return std::move(v); }; - DownloadManagerConfig empty; + AssetCachingSettings empty; CHECK(value_or(parse_download_configuration("x-azurl,https://abc/123,foo;clear"), empty).m_read_url_template == nullopt); @@ -698,7 +708,7 @@ TEST_CASE ("AssetConfigParser x-block-origin provider", "[assetconfigparser]") return std::move(v); }; - DownloadManagerConfig empty; + AssetCachingSettings empty; CHECK(!value_or(parse_download_configuration({}), empty).m_block_origin); CHECK(value_or(parse_download_configuration("x-block-origin"), empty).m_block_origin); diff --git a/src/vcpkg-test/downloads.cpp b/src/vcpkg-test/downloads.cpp index 2dcb46abb5..31eef37073 100644 --- a/src/vcpkg-test/downloads.cpp +++ b/src/vcpkg-test/downloads.cpp @@ -5,63 +5,111 @@ using namespace vcpkg; -TEST_CASE ("split_uri_view", "[downloads]") +TEST_CASE ("parse_split_url_view", "[downloads]") { { - auto x = split_uri_view("https://github.com/Microsoft/vcpkg"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "https"); - REQUIRE(x.get()->authority.value_or("") == "//github.com"); - REQUIRE(x.get()->path_query_fragment == "/Microsoft/vcpkg"); + auto x = parse_split_url_view("https://github.com/Microsoft/vcpkg"); + if (auto v = x.get()) + { + REQUIRE(v->scheme == "https"); + REQUIRE(v->authority.value_or("") == "//github.com"); + REQUIRE(v->path_query_fragment == "/Microsoft/vcpkg"); + } + else + { + FAIL(); + } } { - auto x = split_uri_view(""); - REQUIRE(!x.has_value()); + REQUIRE(!parse_split_url_view("").has_value()); + REQUIRE(!parse_split_url_view("hello").has_value()); } { - auto x = split_uri_view("hello"); - REQUIRE(!x.has_value()); + auto x = parse_split_url_view("file:"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(!y->authority.has_value()); + REQUIRE(y->path_query_fragment == ""); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("file:"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "file"); - REQUIRE(!x.get()->authority.has_value()); - REQUIRE(x.get()->path_query_fragment == ""); + auto x = parse_split_url_view("file:path"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(!y->authority.has_value()); + REQUIRE(y->path_query_fragment == "path"); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("file:path"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "file"); - REQUIRE(!x.get()->authority.has_value()); - REQUIRE(x.get()->path_query_fragment == "path"); + auto x = parse_split_url_view("file:/path"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(!y->authority.has_value()); + REQUIRE(y->path_query_fragment == "/path"); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("file:/path"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "file"); - REQUIRE(!x.get()->authority.has_value()); - REQUIRE(x.get()->path_query_fragment == "/path"); + auto x = parse_split_url_view("file://user:pw@host"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(y->authority.value_or("") == "//user:pw@host"); + REQUIRE(y->path_query_fragment == ""); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("file://user:pw@host"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "file"); - REQUIRE(x.get()->authority.value_or({}) == "//user:pw@host"); - REQUIRE(x.get()->path_query_fragment == ""); + auto x = parse_split_url_view("ftp://host:port/"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "ftp"); + REQUIRE(y->authority.value_or("") == "//host:port"); + REQUIRE(y->path_query_fragment == "/"); + } + else + { + FAIL(); + } } { - auto x = split_uri_view("ftp://host:port/"); - REQUIRE(x.has_value()); - REQUIRE(x.get()->scheme == "ftp"); - REQUIRE(x.get()->authority.value_or({}) == "//host:port"); - REQUIRE(x.get()->path_query_fragment == "/"); + auto x = parse_split_url_view("file://D:\\work\\testing\\asset-cache/" + "562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42b69e4aeb3" + "1a4879e1bcf6f7c2d41f2ace12180ea83ba7af48879d40ab"); + if (auto y = x.get()) + { + REQUIRE(y->scheme == "file"); + REQUIRE(y->authority.value_or("") == "//D:\\work\\testing\\asset-cache"); + REQUIRE(y->path_query_fragment == "/562de7b577c99fe347b00437d14ce375a8e5a60504909cb67d2f73c372d39a2f76d2b42" + "b69e4aeb31a4879e1bcf6f7c2d41f2ace12180ea83ba7af48879d40ab"); + } + else + { + FAIL(); + } } } TEST_CASE ("parse_curl_status_line", "[downloads]") { - std::vector> http_codes; + std::vector http_codes; StringLiteral malformed_examples[] = { "asdfasdf", // wrong prefix "curl: unknown --write-out variable: 'exitcode'", // wrong prefixes, and also what old curl does @@ -72,43 +120,46 @@ TEST_CASE ("parse_curl_status_line", "[downloads]") "prefix42 2a", // non numeric exitcode }; + FullyBufferedDiagnosticContext bdc; for (auto&& malformed : malformed_examples) { - parse_curl_status_line(http_codes, "prefix", malformed); + REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", malformed)); REQUIRE(http_codes.empty()); + REQUIRE(bdc.empty()); } // old curl output - parse_curl_status_line(http_codes, "prefix", "prefix200 "); - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].value_or_exit(VCPKG_LINE_INFO) == 200); + REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix200 ")); + REQUIRE(http_codes == std::vector{200}); + REQUIRE(bdc.empty()); http_codes.clear(); - parse_curl_status_line(http_codes, "prefix", "prefix404 "); - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].value_or_exit(VCPKG_LINE_INFO) == 404); + REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix404 ")); + REQUIRE(http_codes == std::vector{404}); + REQUIRE(bdc.empty()); http_codes.clear(); - parse_curl_status_line(http_codes, "prefix", "prefix0 "); // a failure, but we don't know that yet - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].value_or_exit(VCPKG_LINE_INFO) == 0); + REQUIRE(!parse_curl_status_line(bdc, http_codes, "prefix", "prefix0 ")); // a failure, but we don't know that yet + REQUIRE(http_codes == std::vector{0}); + REQUIRE(bdc.empty()); http_codes.clear(); // current curl output - parse_curl_status_line(http_codes, "prefix", "prefix200 0 "); - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].value_or_exit(VCPKG_LINE_INFO) == 200); + REQUIRE(parse_curl_status_line(bdc, http_codes, "prefix", "prefix200 0 ")); + REQUIRE(http_codes == std::vector{200}); + REQUIRE(bdc.empty()); http_codes.clear(); - parse_curl_status_line(http_codes, - "prefix", - "prefix0 60 schannel: SNI or certificate check failed: SEC_E_WRONG_PRINCIPAL (0x80090322) " - "- The target principal name is incorrect."); - REQUIRE(http_codes.size() == 1); - REQUIRE(http_codes[0].error().data() == - "curl operation failed with error code 60. schannel: SNI or certificate check failed: " + REQUIRE(parse_curl_status_line( + bdc, + http_codes, + "prefix", + "prefix0 60 schannel: SNI or certificate check failed: SEC_E_WRONG_PRINCIPAL (0x80090322) " + "- The target principal name is incorrect.")); + REQUIRE(http_codes == std::vector{0}); + REQUIRE(bdc.to_string() == + "error: curl operation failed with error code 60. schannel: SNI or certificate check failed: " "SEC_E_WRONG_PRINCIPAL (0x80090322) - The target principal name is incorrect."); - http_codes.clear(); } TEST_CASE ("download_files", "[downloads]") @@ -116,40 +167,29 @@ TEST_CASE ("download_files", "[downloads]") auto const dst = Test::base_temporary_directory() / "download_files"; auto const url = [&](std::string l) -> auto { return std::pair(l, dst); }; + FullyBufferedDiagnosticContext bdc; std::vector headers; std::vector secrets; - auto results = - download_files(std::vector{url("unknown://localhost:9/secret"), url("http://localhost:9/not-exists/secret")}, - headers, - secrets); - REQUIRE(results.size() == 2); - if (auto first_result = results[0].get()) - { - // old curl - REQUIRE(*first_result == 0); - } - else - { - // current curl - REQUIRE(results[0].error().data() == - "curl operation failed with error code 1. Protocol \"unknown\" not supported"); - } - - auto&& second_error = results[1].error().data(); - std::puts(second_error.c_str()); - // curl operation failed with error code 7. Failed to connect to localhost port 9 after 2241 ms: Could not connect - // to server - if (second_error == "curl operation failed with error code 7.") + auto results = download_files_no_cache( + bdc, + std::vector{url("unknown://localhost:9/secret"), url("http://localhost:9/not-exists/secret")}, + headers, + secrets); + REQUIRE(results == std::vector{0, 0}); + auto all_errors = bdc.to_string(); + if (all_errors == "error: curl operation failed with error code 7.") { - // old curl + // old curl, this is OK! } else { // new curl REQUIRE_THAT( - second_error, - Catch::Matches("curl operation failed with error code 7. Failed to connect to localhost port 9 after " - "[0-9]+ ms: Could not connect to server", + all_errors, + Catch::Matches("error: curl operation failed with error code 1\\. Protocol \"unknown\" not supported( or " + "disabled in libcurl)?\n" + "error: curl operation failed with error code 7\\. Failed to connect to localhost port 9 " + "after [0-9]+ ms: (Could not|Couldn't) connect to server", Catch::CaseSensitive::Yes)); } } @@ -235,8 +275,8 @@ TEST_CASE ("try_parse_curl_progress_data", "[downloads]") .value_or_exit(VCPKG_LINE_INFO); REQUIRE(out.total_percent == 0); REQUIRE(out.total_size == 0); - REQUIRE(out.recieved_percent == 0); - REQUIRE(out.recieved_size == 0); + REQUIRE(out.received_percent == 0); + REQUIRE(out.received_size == 0); REQUIRE(out.transfer_percent == 0); REQUIRE(out.transfer_size == 0); REQUIRE(out.average_upload_speed == 0); @@ -250,8 +290,8 @@ TEST_CASE ("try_parse_curl_progress_data", "[downloads]") .value_or_exit(VCPKG_LINE_INFO); REQUIRE(out.total_percent == 2); REQUIRE(out.total_size == 190 * 1024 * 1024); - REQUIRE(out.recieved_percent == 2); - REQUIRE(out.recieved_size == 3935 * 1024); + REQUIRE(out.received_percent == 2); + REQUIRE(out.received_size == 3935 * 1024); REQUIRE(out.transfer_percent == 0); REQUIRE(out.transfer_size == 0); REQUIRE(out.average_upload_speed == 0); diff --git a/src/vcpkg-test/strings.cpp b/src/vcpkg-test/strings.cpp index de6a284db9..d31a7d956f 100644 --- a/src/vcpkg-test/strings.cpp +++ b/src/vcpkg-test/strings.cpp @@ -204,36 +204,62 @@ TEST_CASE ("inplace_replace_all(char)", "[strings]") TEST_CASE ("api_stable_format(sv,append_f)", "[strings]") { - std::string target; - auto res = api_stable_format("{", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(!res.has_value()); - res = api_stable_format("}", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(!res.has_value()); - res = api_stable_format("{ {", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(!res.has_value()); - res = api_stable_format("{ {}", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(!res.has_value()); - - res = api_stable_format("}}", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(*res.get() == "}"); - res = api_stable_format("{{", [](std::string&, StringView) { CHECK(false); }); - REQUIRE(*res.get() == "{"); - - res = api_stable_format("{x}{y}{z}", [](std::string& out, StringView t) { - CHECK((t == "x" || t == "y" || t == "z")); - Strings::append(out, t, t); - }); - REQUIRE(*res.get() == "xxyyzz"); - res = api_stable_format("{x}}}", [](std::string& out, StringView t) { - CHECK(t == "x"); - Strings::append(out, "hello"); - }); - REQUIRE(*res.get() == "hello}"); - res = api_stable_format("123{x}456", [](std::string& out, StringView t) { - CHECK(t == "x"); - Strings::append(out, "hello"); - }); - REQUIRE(*res.get() == "123hello456"); + for (auto&& invalid_format_string : {"{", "}", "{ {", "{ {}"}) + { + FullyBufferedDiagnosticContext bdc_invalid{}; + auto res = api_stable_format(bdc_invalid, invalid_format_string, [](std::string&, StringView) { + CHECK(false); + return true; + }); + REQUIRE(bdc_invalid.to_string() == fmt::format("error: invalid format string: {}", invalid_format_string)); + } + + FullyBufferedDiagnosticContext bdc{}; + { + auto res = api_stable_format(bdc, "}}", [](std::string&, StringView) { + CHECK(false); + return true; + }); + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "}"); + } + { + auto res = api_stable_format(bdc, "{{", [](std::string&, StringView) { + CHECK(false); + return true; + }); + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "{"); + } + { + auto res = api_stable_format(bdc, "{x}{y}{z}", [](std::string& out, StringView t) { + CHECK((t == "x" || t == "y" || t == "z")); + Strings::append(out, t, t); + return true; + }); + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "xxyyzz"); + } + { + auto res = api_stable_format(bdc, "{x}}}", [](std::string& out, StringView t) { + CHECK(t == "x"); + Strings::append(out, "hello"); + return true; + }); + + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "hello}"); + } + { + auto res = api_stable_format(bdc, "123{x}456", [](std::string& out, StringView t) { + CHECK(t == "x"); + Strings::append(out, "hello"); + return true; + }); + + REQUIRE(bdc.empty()); + REQUIRE(res.value_or_exit(VCPKG_LINE_INFO) == "123hello456"); + } } TEST_CASE ("lex compare less", "[strings]") diff --git a/src/vcpkg/base/diagnostics.cpp b/src/vcpkg/base/diagnostics.cpp index 2ecb4e8052..81105ff425 100644 --- a/src/vcpkg/base/diagnostics.cpp +++ b/src/vcpkg/base/diagnostics.cpp @@ -1,6 +1,9 @@ #include #include #include +#include + +#include #include @@ -48,42 +51,56 @@ namespace const auto prefix = prefixes[diag_index]; target.append(prefix->data(), prefix->size()); } -} -namespace vcpkg -{ - void DiagnosticLine::print_to(MessageSink& sink) const + template + void joined_line_to_string(const std::vector& lines, std::string& target) { - std::string buf; - append_file_prefix(buf, m_origin, m_position); - switch (m_kind) + auto first = lines.begin(); + const auto last = lines.end(); + if (first == last) { - case DiagKind::None: - // intentionally blank - break; - case DiagKind::Message: buf.append(MessagePrefix.data(), MessagePrefix.size()); break; - case DiagKind::Error: + return; + } + + for (;;) + { + first->to_string(target); + if (++first == last) { - sink.print(Color::none, buf); - sink.print(Color::error, "error"); - buf.assign(ColonSpace.data(), ColonSpace.size()); + return; } - break; - case DiagKind::Warning: + + target.push_back('\n'); + } + } + + bool diagnostic_lines_any_errors(const std::vector& lines) + { + for (auto&& line : lines) + { + if (line.kind() == DiagKind::Error) { - sink.print(Color::none, buf); - sink.print(Color::warning, "warning"); - buf.assign(ColonSpace.data(), ColonSpace.size()); + return true; } - break; - case DiagKind::Note: buf.append(NotePrefix.data(), NotePrefix.size()); break; - default: Checks::unreachable(VCPKG_LINE_INFO); } - buf.append(m_message.data()); - buf.push_back('\n'); - sink.print(Color::none, buf); + return false; } +} + +namespace vcpkg +{ + void DiagnosticContext::report(DiagnosticLine&& line) { report(line); } + + void DiagnosticContext::report_system_error(StringLiteral system_api_name, int error_value) + { + report_error(msgSystemApiErrorMessage, + msg::system_api = system_api_name, + msg::exit_code = error_value, + msg::error_msg = std::system_category().message(error_value)); + } + + void DiagnosticLine::print_to(MessageSink& sink) const { sink.println(to_message_line()); } std::string DiagnosticLine::to_string() const { std::string result; @@ -97,6 +114,40 @@ namespace vcpkg target.append(m_message.data()); } + MessageLine DiagnosticLine::to_message_line() const + { + MessageLine ret; + { + std::string file_prefix; + append_file_prefix(file_prefix, m_origin, m_position); + ret.print(file_prefix); + } + switch (m_kind) + { + case DiagKind::None: + // intentionally blank + break; + case DiagKind::Message: ret.print(MessagePrefix); break; + case DiagKind::Error: + { + ret.print(Color::error, "error"); + ret.print(ColonSpace); + } + break; + case DiagKind::Warning: + { + ret.print(Color::warning, "warning"); + ret.print(ColonSpace); + } + break; + case DiagKind::Note: ret.print(NotePrefix); break; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + + ret.print(m_message); + return ret; + } + LocalizedString DiagnosticLine::to_json_reader_string(const std::string& path, const LocalizedString& type) const { std::string result; @@ -110,8 +161,41 @@ namespace vcpkg return LocalizedString::from_raw(result); } - void BufferedDiagnosticContext::report(const DiagnosticLine& line) { lines.push_back(line); } + DiagnosticLine DiagnosticLine::reduce_to_warning() const& + { + return DiagnosticLine{m_kind == DiagKind::Error ? DiagKind::Warning : m_kind, m_origin, m_position, m_message}; + } + DiagnosticLine DiagnosticLine::reduce_to_warning() && + { + return DiagnosticLine{m_kind == DiagKind::Error ? DiagKind::Warning : m_kind, + std::move(m_origin), + m_position, + std::move(m_message)}; + } + DiagnosticLine::DiagnosticLine(DiagKind kind, + const Optional& origin, + TextRowCol position, + const LocalizedString& message) + : m_kind(kind), m_origin(origin), m_position(position), m_message(message) + { + } + DiagnosticLine::DiagnosticLine(DiagKind kind, + Optional&& origin, + TextRowCol position, + LocalizedString&& message) + : m_kind(kind), m_origin(std::move(origin)), m_position(position), m_message(std::move(message)) + { + } + + void PrintingDiagnosticContext::report(const DiagnosticLine& line) { line.print_to(sink); } + + void PrintingDiagnosticContext::statusln(const LocalizedString& message) { sink.println(message); } + void PrintingDiagnosticContext::statusln(LocalizedString&& message) { sink.println(std::move(message)); } + void PrintingDiagnosticContext::statusln(const MessageLine& message) { sink.println(message); } + void PrintingDiagnosticContext::statusln(MessageLine&& message) { sink.println(std::move(message)); } + + void BufferedDiagnosticContext::report(const DiagnosticLine& line) { lines.push_back(line); } void BufferedDiagnosticContext::report(DiagnosticLine&& line) { lines.push_back(std::move(line)); } void BufferedDiagnosticContext::print_to(MessageSink& sink) const { @@ -120,79 +204,135 @@ namespace vcpkg line.print_to(sink); } } + // Converts this message into a string // Prefer print() if possible because it applies color // Not safe to use in the face of concurrent calls to report() - std::string BufferedDiagnosticContext::to_string() const + std::string BufferedDiagnosticContext::to_string() const { return adapt_to_string(*this); } + void BufferedDiagnosticContext::to_string(std::string& target) const { joined_line_to_string(lines, target); } + + bool BufferedDiagnosticContext::any_errors() const noexcept { return diagnostic_lines_any_errors(lines); } + bool BufferedDiagnosticContext::empty() const noexcept { return lines.empty(); } + + void BufferedDiagnosticContext::statusln(const LocalizedString& message) { status_sink.println(message); } + void BufferedDiagnosticContext::statusln(LocalizedString&& message) { status_sink.println(std::move(message)); } + void BufferedDiagnosticContext::statusln(const MessageLine& message) { status_sink.println(message); } + void BufferedDiagnosticContext::statusln(MessageLine&& message) { status_sink.println(std::move(message)); } + + void FullyBufferedDiagnosticContext::report(const DiagnosticLine& line) { lines.push_back(line.to_message_line()); } + void FullyBufferedDiagnosticContext::report(DiagnosticLine&& line) { - std::string result; - this->to_string(result); - return result; + lines.push_back(std::move(line).to_message_line()); } - void BufferedDiagnosticContext::to_string(std::string& target) const + + void FullyBufferedDiagnosticContext::statusln(const LocalizedString& message) { lines.emplace_back(message); } + void FullyBufferedDiagnosticContext::statusln(LocalizedString&& message) { lines.emplace_back(std::move(message)); } + void FullyBufferedDiagnosticContext::statusln(const MessageLine& message) { lines.emplace_back(message); } + void FullyBufferedDiagnosticContext::statusln(MessageLine&& message) { lines.emplace_back(std::move(message)); } + + void FullyBufferedDiagnosticContext::print_to(MessageSink& sink) const { - auto first = lines.begin(); - const auto last = lines.end(); - if (first == last) + for (auto&& line : lines) { - return; + sink.println(line); } + } - for (;;) - { - first->to_string(target); - if (++first == last) - { - return; - } + std::string FullyBufferedDiagnosticContext::to_string() const { return adapt_to_string(*this); } + void FullyBufferedDiagnosticContext::to_string(std::string& target) const { joined_line_to_string(lines, target); } - target.push_back('\n'); + bool FullyBufferedDiagnosticContext::empty() const noexcept { return lines.empty(); } + + void AttemptDiagnosticContext::report(const DiagnosticLine& line) { lines.push_back(line); } + void AttemptDiagnosticContext::report(DiagnosticLine&& line) { lines.push_back(std::move(line)); } + + void AttemptDiagnosticContext::statusln(const LocalizedString& message) { inner_context.statusln(message); } + void AttemptDiagnosticContext::statusln(LocalizedString&& message) { inner_context.statusln(std::move(message)); } + void AttemptDiagnosticContext::statusln(const MessageLine& message) { inner_context.statusln(message); } + void AttemptDiagnosticContext::statusln(MessageLine&& message) { inner_context.statusln(std::move(message)); } + + void AttemptDiagnosticContext::commit() + { + for (auto& line : lines) + { + inner_context.report(std::move(line)); } + + lines.clear(); } - bool BufferedDiagnosticContext::any_errors() const noexcept + void AttemptDiagnosticContext::handle() { lines.clear(); } + + AttemptDiagnosticContext::~AttemptDiagnosticContext() { - for (auto&& line : lines) + if (!lines.empty()) { - if (line.kind() == DiagKind::Error) +#if defined(NDEBUG) + for (auto& line : lines) { - return true; + inner_context.report(std::move(line)); } +#else // ^^^ NDEBUG // !NDEBUG vvv + Checks::unreachable(VCPKG_LINE_INFO, "Uncommitted diagnostics in ~AttemptDiagnosticContext"); +#endif // ^^^ !NDEBUG } + } - return false; + void WarningDiagnosticContext::report(const DiagnosticLine& line) + { + inner_context.report(line.reduce_to_warning()); + } + void WarningDiagnosticContext::report(DiagnosticLine&& line) + { + inner_context.report(std::move(line).reduce_to_warning()); } + + void WarningDiagnosticContext::statusln(const LocalizedString& message) { inner_context.statusln(message); } + void WarningDiagnosticContext::statusln(LocalizedString&& message) { inner_context.statusln(std::move(message)); } + void WarningDiagnosticContext::statusln(const MessageLine& message) { inner_context.statusln(message); } + void WarningDiagnosticContext::statusln(MessageLine&& message) { inner_context.statusln(std::move(message)); } } // namespace vcpkg namespace { - struct ConsoleDiagnosticContext : DiagnosticContext + struct NullDiagnosticContext final : DiagnosticContext + { + // these are all intentionally empty + virtual void report(const DiagnosticLine&) override { } + virtual void statusln(const LocalizedString&) override { } + virtual void statusln(LocalizedString&&) override { } + virtual void statusln(const MessageLine&) override { } + virtual void statusln(MessageLine&&) override { } + }; + + NullDiagnosticContext null_diagnostic_context_instance; + + struct ConsoleDiagnosticContext final : DiagnosticContext { virtual void report(const DiagnosticLine& line) override { line.print_to(out_sink); } + virtual void statusln(const LocalizedString& message) override { out_sink.println(message); } + virtual void statusln(LocalizedString&& message) override { out_sink.println(std::move(message)); } + virtual void statusln(const MessageLine& message) override { out_sink.println(message); } + virtual void statusln(MessageLine&& message) override { out_sink.println(std::move(message)); } }; ConsoleDiagnosticContext console_diagnostic_context_instance; -} // unnamed namespace - -namespace vcpkg -{ - DiagnosticContext& console_diagnostic_context = console_diagnostic_context_instance; -} -namespace -{ - struct NullDiagnosticContext : DiagnosticContext + struct StatusOnlyDiagnosticContext final : DiagnosticContext { - virtual void report(const DiagnosticLine&) override - { - // intentionally empty - } + virtual void report(const DiagnosticLine&) override { } + virtual void statusln(const LocalizedString& message) override { out_sink.println(message); } + virtual void statusln(LocalizedString&& message) override { out_sink.println(std::move(message)); } + virtual void statusln(const MessageLine& message) override { out_sink.println(message); } + virtual void statusln(MessageLine&& message) override { out_sink.println(std::move(message)); } }; - NullDiagnosticContext null_diagnostic_context_instance; -} + StatusOnlyDiagnosticContext status_only_diagnostic_context_instance; +} // unnamed namespace namespace vcpkg { DiagnosticContext& null_diagnostic_context = null_diagnostic_context_instance; + DiagnosticContext& console_diagnostic_context = console_diagnostic_context_instance; + DiagnosticContext& status_only_diagnostic_context = status_only_diagnostic_context_instance; } diff --git a/src/vcpkg/base/downloads.cpp b/src/vcpkg/base/downloads.cpp index 8f573a3a36..25238f2b41 100644 --- a/src/vcpkg/base/downloads.cpp +++ b/src/vcpkg/base/downloads.cpp @@ -4,9 +4,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -15,60 +17,103 @@ #include -namespace vcpkg +#include + +using namespace vcpkg; + +namespace { - static std::string replace_secrets(std::string input, View secrets) + constexpr StringLiteral vcpkg_curl_user_agent_header = + "User-Agent: vcpkg/" VCPKG_BASE_VERSION_AS_STRING "-" VCPKG_VERSION_AS_STRING " (curl)"; + + void add_curl_headers(Command& cmd, View headers) + { + cmd.string_arg("-H").string_arg(vcpkg_curl_user_agent_header); + for (auto&& header : headers) + { + cmd.string_arg("-H").string_arg(header); + } + } + + void replace_secrets(std::string& target, View secrets) { const auto replacement = msg::format(msgSecretBanner); for (const auto& secret : secrets) { - Strings::inplace_replace_all(input, secret, replacement); + Strings::inplace_replace_all(target, secret, replacement); } + } +} - return input; +namespace vcpkg +{ + SanitizedUrl::SanitizedUrl(StringView raw_url, View secrets) + : m_sanitized_url(raw_url.data(), raw_url.size()) + { + replace_secrets(m_sanitized_url, secrets); } #if defined(_WIN32) - struct WinHttpHandle + struct FormatMessageHLocalAlloc { - HINTERNET h; + LPWSTR buffer = nullptr; - WinHttpHandle() : h(0) { } - explicit WinHttpHandle(HINTERNET h_) : h(h_) { } - WinHttpHandle(const WinHttpHandle&) = delete; - WinHttpHandle(WinHttpHandle&& other) : h(other.h) { other.h = 0; } - WinHttpHandle& operator=(const WinHttpHandle&) = delete; - WinHttpHandle& operator=(WinHttpHandle&& other) + ~FormatMessageHLocalAlloc() { - auto cpy = std::move(other); - std::swap(h, cpy.h); - return *this; + if (buffer) + { + LocalFree(buffer); + } } + }; - ~WinHttpHandle() + static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, + const SanitizedUrl& sanitized_url, + DWORD last_error) + { + const HMODULE winhttp_module = GetModuleHandleW(L"winhttp.dll"); + FormatMessageHLocalAlloc alloc; + DWORD tchars_excluding_terminating_null = 0; + if (winhttp_module) { - if (h) + tchars_excluding_terminating_null = + FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_HMODULE, + winhttp_module, + last_error, + 0, + reinterpret_cast(&alloc.buffer), + 0, + nullptr); + } + + auto result = msg::format( + msgDownloadWinHttpError, msg::system_api = api_name, msg::exit_code = last_error, msg::url = sanitized_url); + if (tchars_excluding_terminating_null && alloc.buffer) + { + while (tchars_excluding_terminating_null != 0 && + (alloc.buffer[tchars_excluding_terminating_null - 1] == L'\r' || + alloc.buffer[tchars_excluding_terminating_null - 1] == L'\n')) { - WinHttpCloseHandle(h); + --tchars_excluding_terminating_null; } + + tchars_excluding_terminating_null = static_cast( + std::remove(alloc.buffer, alloc.buffer + tchars_excluding_terminating_null, L'\r') - alloc.buffer); + result.append_raw(' ').append_raw(Strings::to_utf8(alloc.buffer, tchars_excluding_terminating_null)); } - }; - static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, StringView url, DWORD last_error) - { - return msg::format_error( - msgDownloadWinHttpError, msg::system_api = api_name, msg::exit_code = last_error, msg::url = url); + return result; } - static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, StringView url) + static LocalizedString format_winhttp_last_error_message(StringLiteral api_name, const SanitizedUrl& sanitized_url) { - return format_winhttp_last_error_message(api_name, url, GetLastError()); + return format_winhttp_last_error_message(api_name, sanitized_url, GetLastError()); } - static void maybe_emit_winhttp_progress(const Optional& maybe_content_length, + static void maybe_emit_winhttp_progress(MessageSink& machine_readable_progress, + const Optional& maybe_content_length, std::chrono::steady_clock::time_point& last_write, - unsigned long long total_downloaded_size, - MessageSink& progress_sink) + unsigned long long total_downloaded_size) { if (const auto content_length = maybe_content_length.get()) { @@ -77,174 +122,240 @@ namespace vcpkg { const double percent = (static_cast(total_downloaded_size) / static_cast(*content_length)) * 100; - progress_sink.print(Color::none, fmt::format("{:.2f}%\n", percent)); + machine_readable_progress.println(LocalizedString::from_raw(fmt::format("{:.2f}%", percent))); last_write = now; } } } - struct WinHttpRequest + struct WinHttpHandle { - static ExpectedL make(HINTERNET hConnect, - StringView url_path, - StringView sanitized_url, - bool https, - const wchar_t* method = L"GET") - { - WinHttpRequest ret; - ret.m_sanitized_url.assign(sanitized_url.data(), sanitized_url.size()); - // Create an HTTP request handle. - { - auto h = WinHttpOpenRequest(hConnect, - method, - Strings::to_utf16(url_path).c_str(), - nullptr, - WINHTTP_NO_REFERER, - WINHTTP_DEFAULT_ACCEPT_TYPES, - https ? WINHTTP_FLAG_SECURE : 0); - if (!h) - { - return format_winhttp_last_error_message("WinHttpOpenRequest", sanitized_url); - } + WinHttpHandle() = default; + WinHttpHandle(const WinHttpHandle&) = delete; + WinHttpHandle& operator=(const WinHttpHandle&) = delete; - ret.m_hRequest = WinHttpHandle{h}; + void require_null_handle() const + { + if (h) + { + Checks::unreachable(VCPKG_LINE_INFO, "WinHTTP handle type confusion"); } + } - // Send a request. - auto bResults = WinHttpSendRequest( - ret.m_hRequest.h, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0); - - if (!bResults) + void require_created_handle() const + { + if (!h) { - return format_winhttp_last_error_message("WinHttpSendRequest", sanitized_url); + Checks::unreachable(VCPKG_LINE_INFO, "WinHTTP handle not created"); } + } - // End the request. - bResults = WinHttpReceiveResponse(ret.m_hRequest.h, NULL); - if (!bResults) + bool Connect(DiagnosticContext& context, + const WinHttpHandle& session, + StringView hostname, + INTERNET_PORT port, + const SanitizedUrl& sanitized_url) + { + require_null_handle(); + session.require_created_handle(); + h = WinHttpConnect(session.h, Strings::to_utf16(hostname).c_str(), port, 0); + if (h) { - return format_winhttp_last_error_message("WinHttpReceiveResponse", sanitized_url); + return true; } - return ret; + context.report_error(format_winhttp_last_error_message("WinHttpConnect", sanitized_url)); + return false; } - ExpectedL query_status() const + bool Open(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + _In_opt_z_ LPCWSTR pszAgentW, + _In_ DWORD dwAccessType, + _In_opt_z_ LPCWSTR pszProxyW, + _In_opt_z_ LPCWSTR pszProxyBypassW, + _In_ DWORD dwFlags) { - DWORD status_code; - DWORD size = sizeof(status_code); + require_null_handle(); + h = WinHttpOpen(pszAgentW, dwAccessType, pszProxyW, pszProxyBypassW, dwFlags); + if (h) + { + return true; + } - auto succeeded = WinHttpQueryHeaders(m_hRequest.h, - WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, - WINHTTP_HEADER_NAME_BY_INDEX, - &status_code, - &size, - WINHTTP_NO_HEADER_INDEX); - if (succeeded) + context.report_error(format_winhttp_last_error_message("WinHttpOpen", sanitized_url)); + return false; + } + + bool OpenRequest(DiagnosticContext& context, + const WinHttpHandle& hConnect, + const SanitizedUrl& sanitized_url, + IN LPCWSTR pwszVerb, + StringView path_query_fragment, + IN LPCWSTR pwszVersion, + IN LPCWSTR pwszReferrer OPTIONAL, + IN LPCWSTR FAR* ppwszAcceptTypes OPTIONAL, + IN DWORD dwFlags) + { + require_null_handle(); + h = WinHttpOpenRequest(hConnect.h, + pwszVerb, + Strings::to_utf16(path_query_fragment).c_str(), + pwszVersion, + pwszReferrer, + ppwszAcceptTypes, + dwFlags); + if (h) { - return status_code; + return true; } - return format_winhttp_last_error_message("WinHttpQueryHeaders", m_sanitized_url); + context.report_error(format_winhttp_last_error_message("WinHttpOpenRequest", sanitized_url)); + return false; } - ExpectedL> query_content_length() const + bool SendRequest(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + _In_reads_opt_(dwHeadersLength) LPCWSTR lpszHeaders, + IN DWORD dwHeadersLength, + _In_reads_bytes_opt_(dwOptionalLength) LPVOID lpOptional, + IN DWORD dwOptionalLength, + IN DWORD dwTotalLength, + IN DWORD_PTR dwContext) const { - static constexpr DWORD buff_characters = 21; // 18446744073709551615 - wchar_t buff[buff_characters]; - DWORD size = sizeof(buff); - auto succeeded = WinHttpQueryHeaders(m_hRequest.h, - WINHTTP_QUERY_CONTENT_LENGTH, - WINHTTP_HEADER_NAME_BY_INDEX, - buff, - &size, - WINHTTP_NO_HEADER_INDEX); - if (succeeded) + require_created_handle(); + if (WinHttpSendRequest( + h, lpszHeaders, dwHeadersLength, lpOptional, dwOptionalLength, dwTotalLength, dwContext)) { - return Strings::strto(Strings::to_utf8(buff, size >> 1)); + return true; } - const DWORD last_error = GetLastError(); - if (last_error == ERROR_WINHTTP_HEADER_NOT_FOUND) + context.report_error(format_winhttp_last_error_message("WinHttpSendRequest", sanitized_url)); + return false; + } + + bool ReceiveResponse(DiagnosticContext& context, const SanitizedUrl& url) + { + require_created_handle(); + if (WinHttpReceiveResponse(h, NULL)) { - return Optional{nullopt}; + return true; } - return format_winhttp_last_error_message("WinHttpQueryHeaders", m_sanitized_url, last_error); + context.report_error(format_winhttp_last_error_message("WinHttpReceiveResponse", url)); + return false; } - ExpectedL write_response_body(WriteFilePointer& file, MessageSink& progress_sink) + bool SetTimeouts(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + int nResolveTimeout, + int nConnectTimeout, + int nSendTimeout, + int nReceiveTimeout) const { - static constexpr DWORD buff_size = 65535; - std::unique_ptr buff{new char[buff_size]}; - Optional maybe_content_length; - auto last_write = std::chrono::steady_clock::now(); + require_created_handle(); + if (WinHttpSetTimeouts(h, nResolveTimeout, nConnectTimeout, nSendTimeout, nReceiveTimeout)) + { + return true; + } + context.report_error(format_winhttp_last_error_message("WinHttpSetTimeouts", sanitized_url)); + return false; + } + + bool SetOption(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + DWORD dwOption, + LPVOID lpBuffer, + DWORD dwBufferLength) const + { + require_created_handle(); + if (WinHttpSetOption(h, dwOption, lpBuffer, dwBufferLength)) { - auto maybe_maybe_content_length = query_content_length(); - if (const auto p = maybe_maybe_content_length.get()) - { - maybe_content_length = *p; - } - else - { - return std::move(maybe_maybe_content_length).error(); - } + return true; } - unsigned long long total_downloaded_size = 0; - for (;;) + context.report_error(format_winhttp_last_error_message("WinHttpSetOption", sanitized_url)); + return false; + } + + DWORD QueryHeaders(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + DWORD dwInfoLevel, + LPWSTR pwszName, + LPVOID lpBuffer, + LPDWORD lpdwBufferLength, + LPDWORD lpdwIndex) const + { + require_created_handle(); + if (WinHttpQueryHeaders(h, dwInfoLevel, pwszName, lpBuffer, lpdwBufferLength, lpdwIndex)) { - DWORD this_read; - if (!WinHttpReadData(m_hRequest.h, buff.get(), buff_size, &this_read)) - { - return format_winhttp_last_error_message("WinHttpReadData", m_sanitized_url); - } + return 0; + } - if (this_read == 0) - { - return Unit{}; - } + DWORD last_error = GetLastError(); + context.report_error(format_winhttp_last_error_message("WinHttpQueryHeaders", sanitized_url, last_error)); + return last_error; + } - do - { - const auto this_write = static_cast(file.write(buff.get(), 1, this_read)); - if (this_write == 0) - { - return format_winhttp_last_error_message("fwrite", m_sanitized_url); - } + bool ReadData(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + LPVOID buffer, + DWORD dwNumberOfBytesToRead, + DWORD* numberOfBytesRead) + { + require_created_handle(); + if (WinHttpReadData(h, buffer, dwNumberOfBytesToRead, numberOfBytesRead)) + { + return true; + } - maybe_emit_winhttp_progress(maybe_content_length, last_write, total_downloaded_size, progress_sink); - this_read -= this_write; - total_downloaded_size += this_write; - } while (this_read > 0); + context.report_error(format_winhttp_last_error_message("WinHttpReadData", sanitized_url)); + return false; + } + + ~WinHttpHandle() + { + if (h) + { + // intentionally ignore failures + (void)WinHttpCloseHandle(h); } } - WinHttpHandle m_hRequest; - std::string m_sanitized_url; + private: + HINTERNET h{}; + }; + + enum class WinHttpTrialResult + { + failed, + succeeded, + retry }; struct WinHttpSession { - static ExpectedL make(StringView sanitized_url) + bool open(DiagnosticContext& context, const SanitizedUrl& sanitized_url) { - WinHttpSession ret; + if (!m_hSession.Open(context, + sanitized_url, + L"vcpkg/1.0", + WINHTTP_ACCESS_TYPE_NO_PROXY, + WINHTTP_NO_PROXY_NAME, + WINHTTP_NO_PROXY_BYPASS, + 0)) { - auto h = WinHttpOpen( - L"vcpkg/1.0", WINHTTP_ACCESS_TYPE_NO_PROXY, WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0); - if (!h) - { - return format_winhttp_last_error_message("WinHttpOpen", sanitized_url); - } - - ret.m_hSession = WinHttpHandle{h}; + return false; } // Increase default timeouts to help connections behind proxies // WinHttpSetTimeouts(HINTERNET hInternet, int nResolveTimeout, int nConnectTimeout, int nSendTimeout, int // nReceiveTimeout); - WinHttpSetTimeouts(ret.m_hSession.h, 0, 120000, 120000, 120000); + if (!m_hSession.SetTimeouts(context, sanitized_url, 0, 120000, 120000, 120000)) + { + return false; + } // If the environment variable HTTPS_PROXY is set // use that variable as proxy. This situation might exist when user is in a company network @@ -257,8 +368,10 @@ namespace vcpkg proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; proxy.lpszProxy = env_proxy_settings.data(); proxy.lpszProxyBypass = nullptr; - - WinHttpSetOption(ret.m_hSession.h, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + if (!m_hSession.SetOption(context, sanitized_url, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy))) + { + return false; + } } // IE Proxy fallback, this works on Windows 10 else @@ -272,24 +385,36 @@ namespace vcpkg proxy.dwAccessType = WINHTTP_ACCESS_TYPE_NAMED_PROXY; proxy.lpszProxy = ieProxy.get()->server.data(); proxy.lpszProxyBypass = ieProxy.get()->bypass.data(); - WinHttpSetOption(ret.m_hSession.h, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy)); + if (!m_hSession.SetOption(context, sanitized_url, WINHTTP_OPTION_PROXY, &proxy, sizeof(proxy))) + { + return false; + } } } // Use Windows 10 defaults on Windows 7 DWORD secure_protocols(WINHTTP_FLAG_SECURE_PROTOCOL_TLS1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 | WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2); - WinHttpSetOption( - ret.m_hSession.h, WINHTTP_OPTION_SECURE_PROTOCOLS, &secure_protocols, sizeof(secure_protocols)); + if (!m_hSession.SetOption(context, + sanitized_url, + WINHTTP_OPTION_SECURE_PROTOCOLS, + &secure_protocols, + sizeof(secure_protocols))) + { + return false; + } // Many open source mirrors such as https://download.gnome.org/ will redirect to http mirrors. // `curl.exe -L` does follow https -> http redirection. // Additionally, vcpkg hash checks the resulting archive. DWORD redirect_policy(WINHTTP_OPTION_REDIRECT_POLICY_ALWAYS); - WinHttpSetOption( - ret.m_hSession.h, WINHTTP_OPTION_REDIRECT_POLICY, &redirect_policy, sizeof(redirect_policy)); + if (!m_hSession.SetOption( + context, sanitized_url, WINHTTP_OPTION_REDIRECT_POLICY, &redirect_policy, sizeof(redirect_policy))) + { + return false; + } - return ret; + return true; } WinHttpHandle m_hSession; @@ -297,85 +422,223 @@ namespace vcpkg struct WinHttpConnection { - static ExpectedL make(HINTERNET hSession, - StringView hostname, - INTERNET_PORT port, - StringView sanitized_url) + bool connect(DiagnosticContext& context, + const WinHttpSession& hSession, + StringView hostname, + INTERNET_PORT port, + const SanitizedUrl& sanitized_url) { // Specify an HTTP server. - auto h = WinHttpConnect(hSession, Strings::to_utf16(hostname).c_str(), port, 0); - if (!h) + return m_hConnect.Connect(context, hSession.m_hSession, hostname, port, sanitized_url); + } + + WinHttpHandle m_hConnect; + }; + + struct WinHttpRequest + { + bool open(DiagnosticContext& context, + const WinHttpConnection& hConnect, + StringView path_query_fragment, + const SanitizedUrl& sanitized_url, + bool https, + const wchar_t* method = L"GET") + { + if (!m_hRequest.OpenRequest(context, + hConnect.m_hConnect, + sanitized_url, + method, + path_query_fragment, + nullptr, + WINHTTP_NO_REFERER, + WINHTTP_DEFAULT_ACCEPT_TYPES, + https ? WINHTTP_FLAG_SECURE : 0)) + { + return false; + } + + // Send a request. + if (!m_hRequest.SendRequest( + context, sanitized_url, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0)) + { + return false; + } + + // End the request. + if (!m_hRequest.ReceiveResponse(context, sanitized_url)) + { + return false; + } + + return true; + } + + Optional query_status(DiagnosticContext& context, const SanitizedUrl& sanitized_url) const + { + DWORD status_code; + DWORD size = sizeof(status_code); + DWORD last_error = m_hRequest.QueryHeaders(context, + sanitized_url, + WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, + WINHTTP_HEADER_NAME_BY_INDEX, + &status_code, + &size, + WINHTTP_NO_HEADER_INDEX); + if (last_error) { - return format_winhttp_last_error_message("WinHttpConnect", sanitized_url); + return nullopt; } - return WinHttpConnection{WinHttpHandle{h}}; + return status_code; } - WinHttpHandle m_hConnect; + bool query_content_length(DiagnosticContext& context, + const SanitizedUrl& sanitized_url, + Optional& result) const + { + static constexpr DWORD buff_characters = 21; // 18446744073709551615 + wchar_t buff[buff_characters]; + DWORD size = sizeof(buff); + AttemptDiagnosticContext adc{context}; + DWORD last_error = m_hRequest.QueryHeaders(adc, + sanitized_url, + WINHTTP_QUERY_CONTENT_LENGTH, + WINHTTP_HEADER_NAME_BY_INDEX, + buff, + &size, + WINHTTP_NO_HEADER_INDEX); + if (!last_error) + { + adc.commit(); + result = Strings::strto(Strings::to_utf8(buff, size >> 1)); + return true; + } + + if (last_error == ERROR_WINHTTP_HEADER_NOT_FOUND) + { + adc.handle(); + return true; + } + + adc.commit(); + return false; + } + + WinHttpTrialResult write_response_body(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const SanitizedUrl& sanitized_url, + const WriteFilePointer& file) + { + static constexpr DWORD buff_size = 65535; + std::unique_ptr buff{new char[buff_size]}; + Optional maybe_content_length; + auto last_write = std::chrono::steady_clock::now(); + if (!query_content_length(context, sanitized_url, maybe_content_length)) + { + return WinHttpTrialResult::retry; + } + + unsigned long long total_downloaded_size = 0; + for (;;) + { + DWORD this_read; + if (!m_hRequest.ReadData(context, sanitized_url, buff.get(), buff_size, &this_read)) + { + return WinHttpTrialResult::retry; + } + + if (this_read == 0) + { + return WinHttpTrialResult::succeeded; + } + + do + { + const auto this_write = static_cast(file.write(buff.get(), 1, this_read)); + if (this_write == 0) + { + context.report_error(format_filesystem_call_error( + std::error_code{errno, std::generic_category()}, "fwrite", {file.path()})); + return WinHttpTrialResult::failed; + } + + maybe_emit_winhttp_progress( + machine_readable_progress, maybe_content_length, last_write, total_downloaded_size); + this_read -= this_write; + total_downloaded_size += this_write; + } while (this_read > 0); + } + } + + WinHttpHandle m_hRequest; }; #endif - ExpectedL split_uri_view(StringView uri) + Optional parse_split_url_view(StringView raw_url) { - auto sep = std::find(uri.begin(), uri.end(), ':'); - if (sep == uri.end()) return msg::format_error(msgInvalidUri, msg::value = uri); + auto sep = std::find(raw_url.begin(), raw_url.end(), ':'); + if (sep == raw_url.end()) + { + return nullopt; + } - StringView scheme(uri.begin(), sep); - if (Strings::starts_with({sep + 1, uri.end()}, "//")) + StringView scheme(raw_url.begin(), sep); + if (Strings::starts_with({sep + 1, raw_url.end()}, "//")) { - auto path_start = std::find(sep + 3, uri.end(), '/'); - return SplitURIView{scheme, StringView{sep + 1, path_start}, {path_start, uri.end()}}; + auto path_start = std::find(sep + 3, raw_url.end(), '/'); + return SplitUrlView{scheme, StringView{sep + 1, path_start}, StringView{path_start, raw_url.end()}}; } + // no authority - return SplitURIView{scheme, {}, {sep + 1, uri.end()}}; + return SplitUrlView{scheme, {}, StringView{sep + 1, raw_url.end()}}; } - static ExpectedL try_verify_downloaded_file_hash(const ReadOnlyFilesystem& fs, - StringView sanitized_url, - const Path& downloaded_path, - StringView sha512) + static bool check_downloaded_file_hash(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const SanitizedUrl& sanitized_url, + const Path& downloaded_path, + StringView sha512) { - std::string actual_hash = - vcpkg::Hash::get_file_hash(fs, downloaded_path, Hash::Algorithm::Sha512).value_or_exit(VCPKG_LINE_INFO); - if (!Strings::case_insensitive_ascii_equals(sha512, actual_hash)) + auto maybe_actual_hash = + vcpkg::Hash::get_file_hash_required(context, fs, downloaded_path, Hash::Algorithm::Sha512); + if (auto actual_hash = maybe_actual_hash.get()) { - return msg::format_error(msgDownloadFailedHashMismatch, - msg::url = sanitized_url, - msg::path = downloaded_path, - msg::expected = sha512, - msg::actual = actual_hash); - } + if (Strings::case_insensitive_ascii_equals(sha512, *actual_hash)) + { + return true; + } - return Unit{}; - } + context.report(DiagnosticLine{DiagKind::Error, + downloaded_path, + msg::format(msgDownloadFailedHashMismatch, msg::url = sanitized_url)}); + context.report(DiagnosticLine{DiagKind::Note, + msg::format(msgDownloadFailedHashMismatchExpectedHash, msg::sha = sha512)}); + context.report(DiagnosticLine{ + DiagKind::Note, msg::format(msgDownloadFailedHashMismatchActualHash, msg::sha = *actual_hash)}); + } - void verify_downloaded_file_hash(const ReadOnlyFilesystem& fs, - StringView url, - const Path& downloaded_path, - StringView sha512) - { - try_verify_downloaded_file_hash(fs, url, downloaded_path, sha512).value_or_exit(VCPKG_LINE_INFO); + return false; } - static ExpectedL check_downloaded_file_hash(const ReadOnlyFilesystem& fs, - const Optional& hash, - StringView sanitized_url, - const Path& download_part_path) + static bool check_downloaded_file_hash(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const SanitizedUrl& sanitized_url, + const Path& downloaded_path, + Optional maybe_sha512) { - if (auto p = hash.get()) + if (auto sha512 = maybe_sha512.get()) { - return try_verify_downloaded_file_hash(fs, sanitized_url, download_part_path, *p); + return check_downloaded_file_hash(context, fs, sanitized_url, downloaded_path, *sha512); } - Debug::println("Skipping hash check because none was specified."); - return Unit{}; + return true; } - static std::vector> curl_bulk_operation(View operation_args, - StringLiteral prefixArgs, - View headers, - View secrets) + static std::vector curl_bulk_operation(DiagnosticContext& context, + View operation_args, + StringLiteral prefixArgs, + View headers, + View secrets) { #define GUID_MARKER "5ec47b8e-6776-4d70-b9b3-ac2a57bc0a1c" static constexpr StringLiteral guid_marker = GUID_MARKER; @@ -389,14 +652,9 @@ namespace vcpkg GUID_MARKER "%{http_code} %{exitcode} %{errormsg}\\n"); #undef GUID_MARKER - std::vector> ret; + std::vector ret; ret.reserve(operation_args.size()); - - for (auto&& header : headers) - { - prefix_cmd.string_arg("-H").string_arg(header); - } - + add_curl_headers(prefix_cmd, headers); while (ret.size() != operation_args.size()) { // there's an edge case that we aren't handling here where not even one operation fits with the configured @@ -411,47 +669,36 @@ namespace vcpkg } // actually run curl + bool new_curl_seen = false; std::vector debug_lines; - auto maybe_this_batch_exit_code = cmd_execute_and_stream_lines(batch_cmd, [&](StringView line) { + auto maybe_this_batch_exit_code = cmd_execute_and_stream_lines(context, batch_cmd, [&](StringView line) { debug_lines.emplace_back(line.data(), line.size()); - parse_curl_status_line(ret, guid_marker, line); + new_curl_seen |= parse_curl_status_line(context, ret, guid_marker, line); }); if (auto this_batch_exit_code = maybe_this_batch_exit_code.get()) { - if (!ret.empty()) + if (!new_curl_seen) { - if (auto last_http_code = ret.back().get()) - { - if (*last_http_code == 0 && *this_batch_exit_code) - { - // old version of curl, we only have the result code for the last operation - ret.back() = msg::format(msgCurlFailedGeneric, msg::exit_code = *this_batch_exit_code); - } - } + // old version of curl, we only have the result code for the last operation + context.report_error(msgCurlFailedGeneric, msg::exit_code = *this_batch_exit_code); } if (ret.size() != last_try_op) { // curl didn't process everything we asked of it; this usually means curl crashed - auto full_failure = - msg::format_error(msgCurlFailedToReturnExpectedNumberOfExitCodes, - msg::exit_code = *this_batch_exit_code, - msg::command_line = replace_secrets(std::move(batch_cmd).extract(), secrets)); - for (const auto& debug_line : debug_lines) - { - full_failure.append_raw('\n'); - full_failure.append_raw(debug_line); - } - - ret.emplace_back(std::move(full_failure)); + auto command_line = std::move(batch_cmd).extract(); + replace_secrets(command_line, secrets); + context.report_error_with_log(Strings::join("\n", debug_lines), + msgCurlFailedToReturnExpectedNumberOfExitCodes, + msg::exit_code = *this_batch_exit_code, + msg::command_line = command_line); return ret; } } else { // couldn't even launch curl, record this as the last fatal error and give up - ret.emplace_back(std::move(maybe_this_batch_exit_code).error()); return ret; } } @@ -459,20 +706,26 @@ namespace vcpkg return ret; } - std::vector> url_heads(View urls, View headers, View secrets) + std::vector url_heads(DiagnosticContext& context, + View urls, + View headers, + View secrets) { return curl_bulk_operation( + context, Util::fmap(urls, [](const std::string& url) { return Command{}.string_arg(url_encode_spaces(url)); }), "--head", headers, secrets); } - std::vector> download_files(View> url_pairs, - View headers, - View secrets) + std::vector download_files_no_cache(DiagnosticContext& context, + View> url_pairs, + View headers, + View secrets) { - return curl_bulk_operation(Util::fmap(url_pairs, + return curl_bulk_operation(context, + Util::fmap(url_pairs, [](const std::pair& url_pair) { return Command{} .string_arg(url_encode_spaces(url_pair.first)) @@ -484,7 +737,8 @@ namespace vcpkg secrets); } - bool submit_github_dependency_graph_snapshot(const Optional& maybe_github_server_url, + bool submit_github_dependency_graph_snapshot(DiagnosticContext& context, + const Optional& maybe_github_server_url, const std::string& github_token, const std::string& github_repository, const Json::Object& snapshot) @@ -508,26 +762,26 @@ namespace vcpkg auto cmd = Command{"curl"}; cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}"); cmd.string_arg("-X").string_arg("POST"); - cmd.string_arg("-H").string_arg("Accept: application/vnd.github+json"); + { + std::string headers[] = { + "Accept: application/vnd.github+json", + "Authorization: Bearer " + github_token, + "X-GitHub-Api-Version: 2022-11-28", + }; + add_curl_headers(cmd, headers); + } - std::string res = "Authorization: Bearer " + github_token; - cmd.string_arg("-H").string_arg(res); - cmd.string_arg("-H").string_arg("X-GitHub-Api-Version: 2022-11-28"); cmd.string_arg(uri); cmd.string_arg("-d").string_arg("@-"); RedirectedProcessLaunchSettings settings; settings.stdin_content = Json::stringify(snapshot); int code = 0; - auto result = cmd_execute_and_stream_lines(cmd, settings, [&code](StringView line) { + auto result = cmd_execute_and_stream_lines(context, cmd, settings, [&code](StringView line) { if (Strings::starts_with(line, guid_marker)) { code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); } - else - { - Debug::println(line); - } }); auto r = result.get(); @@ -538,93 +792,100 @@ namespace vcpkg return false; } - ExpectedL put_file(const ReadOnlyFilesystem&, - StringView url, - const std::vector& secrets, - View headers, - const Path& file, - StringView method) + static bool store_to_asset_cache_impl(DiagnosticContext& context, + StringView raw_url, + const SanitizedUrl& sanitized_url, + StringLiteral method, + View headers, + const Path& file) { static constexpr StringLiteral guid_marker = "9a1db05f-a65d-419b-aa72-037fb4d0672e"; - if (Strings::starts_with(url, "ftp://")) + if (Strings::starts_with(raw_url, "ftp://")) { // HTTP headers are ignored for FTP clients auto ftp_cmd = Command{"curl"}; - ftp_cmd.string_arg(url_encode_spaces(url)); + ftp_cmd.string_arg(url_encode_spaces(raw_url)); ftp_cmd.string_arg("-T").string_arg(file); - auto maybe_res = cmd_execute_and_capture_output(ftp_cmd); + auto maybe_res = cmd_execute_and_capture_output(context, ftp_cmd); if (auto res = maybe_res.get()) { if (res->exit_code == 0) { - return 0; + return true; } - Debug::print(res->output, '\n'); - return msg::format_error(msgCurlFailedToPut, - msg::exit_code = res->exit_code, - msg::url = replace_secrets(url.to_string(), secrets)); + context.report_error_with_log( + res->output, msgCurlFailedToPut, msg::exit_code = res->exit_code, msg::url = sanitized_url); + return false; } - return std::move(maybe_res).error(); + return false; } auto http_cmd = Command{"curl"}.string_arg("-X").string_arg(method); - for (auto&& header : headers) - { - http_cmd.string_arg("-H").string_arg(header); - } - + add_curl_headers(http_cmd, headers); http_cmd.string_arg("-w").string_arg("\\n" + guid_marker.to_string() + "%{http_code}"); - http_cmd.string_arg(url); + http_cmd.string_arg(raw_url); http_cmd.string_arg("-T").string_arg(file); int code = 0; - auto res = cmd_execute_and_stream_lines(http_cmd, [&code](StringView line) { + auto res = cmd_execute_and_stream_lines(context, http_cmd, [&code](StringView line) { if (Strings::starts_with(line, guid_marker)) { code = std::strtol(line.data() + guid_marker.size(), nullptr, 10); } }); - if (auto pres = res.get()) + auto pres = res.get(); + if (!pres) + { + return false; + } + + if (*pres != 0 || (code >= 100 && code < 200) || code >= 300) + { + context.report_error(msg::format( + msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = sanitized_url, msg::value = code)); + return false; + } + + return true; + } + + bool store_to_asset_cache(DiagnosticContext& context, + StringView raw_url, + const SanitizedUrl& sanitized_url, + StringLiteral method, + View headers, + const Path& file) + { + if (store_to_asset_cache_impl(context, raw_url, sanitized_url, method, headers, file)) { - if (*pres != 0 || (code >= 100 && code < 200) || code >= 300) - { - return msg::format_error( - msgCurlFailedToPutHttp, msg::exit_code = *pres, msg::url = url, msg::value = code); - } + context.statusln(msg::format(msgAssetCacheSuccesfullyStored)); + return true; } - msg::println(msgAssetCacheSuccesfullyStored, - msg::path = file.filename(), - msg::url = replace_secrets(url.to_string(), secrets)); - return 0; + + return false; } std::string format_url_query(StringView base_url, View query_params) { - auto url = base_url.to_string(); if (query_params.empty()) { - return url; + return base_url.to_string(); } - return url + "?" + Strings::join("&", query_params); + return fmt::format(FMT_COMPILE("{}?{}"), base_url, fmt::join(query_params, "&")); } - ExpectedL invoke_http_request(StringView method, - View headers, - StringView url, - StringView data) + Optional invoke_http_request(DiagnosticContext& context, + StringLiteral method, + View headers, + StringView raw_url, + StringView data) { auto cmd = Command{"curl"}.string_arg("-s").string_arg("-L"); - cmd.string_arg("-H").string_arg( - fmt::format("User-Agent: vcpkg/{}-{} (curl)", VCPKG_BASE_VERSION_AS_STRING, VCPKG_VERSION_AS_STRING)); - - for (auto&& header : headers) - { - cmd.string_arg("-H").string_arg(header); - } + add_curl_headers(cmd, headers); cmd.string_arg("-X").string_arg(method); @@ -633,92 +894,78 @@ namespace vcpkg cmd.string_arg("--data-raw").string_arg(data); } - cmd.string_arg(url_encode_spaces(url)); + cmd.string_arg(url_encode_spaces(raw_url)); + + auto maybe_output = cmd_execute_and_capture_output(context, cmd); + if (auto output = check_zero_exit_code(context, maybe_output, "curl")) + { + return *output; + } - return flatten_out(cmd_execute_and_capture_output(cmd), "curl"); + return nullopt; } #if defined(_WIN32) - enum class WinHttpTrialResult - { - failed, - succeeded, - retry - }; - - static WinHttpTrialResult download_winhttp_trial(const Filesystem& fs, - WinHttpSession& s, + static WinHttpTrialResult download_winhttp_trial(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const Filesystem& fs, + const WinHttpSession& s, const Path& download_path_part_path, - SplitURIView split_uri, + SplitUrlView split_uri_view, StringView hostname, INTERNET_PORT port, - StringView sanitized_url, - std::vector& errors, - MessageSink& progress_sink) + const SanitizedUrl& sanitized_url) { - auto maybe_conn = WinHttpConnection::make(s.m_hSession.h, hostname, port, sanitized_url); - const auto conn = maybe_conn.get(); - if (!conn) + WinHttpConnection conn; + if (!conn.connect(context, s, hostname, port, sanitized_url)) { - errors.push_back(std::move(maybe_conn).error()); return WinHttpTrialResult::retry; } - auto maybe_req = WinHttpRequest::make( - conn->m_hConnect.h, split_uri.path_query_fragment, sanitized_url, split_uri.scheme == "https"); - const auto req = maybe_req.get(); - if (!req) + WinHttpRequest req; + if (!req.open( + context, conn, split_uri_view.path_query_fragment, sanitized_url, split_uri_view.scheme == "https")) { - errors.push_back(std::move(maybe_req).error()); return WinHttpTrialResult::retry; } - auto maybe_status = req->query_status(); + auto maybe_status = req.query_status(context, sanitized_url); const auto status = maybe_status.get(); if (!status) { - errors.push_back(std::move(maybe_status).error()); return WinHttpTrialResult::retry; } if (*status < 200 || *status >= 300) { - errors.push_back( - msg::format_error(msgDownloadFailedStatusCode, msg::url = sanitized_url, msg::value = *status)); + context.report_error(msgDownloadFailedStatusCode, msg::url = sanitized_url, msg::value = *status); return WinHttpTrialResult::failed; } - auto f = fs.open_for_write(download_path_part_path, VCPKG_LINE_INFO); - auto maybe_write = req->write_response_body(f, progress_sink); - const auto write = maybe_write.get(); - if (!write) - { - errors.push_back(std::move(maybe_write).error()); - return WinHttpTrialResult::retry; - } - - return WinHttpTrialResult::succeeded; + return req.write_response_body(context, + machine_readable_progress, + sanitized_url, + fs.open_for_write(download_path_part_path, VCPKG_LINE_INFO)); } /// /// Download a file using WinHTTP -- only supports HTTP and HTTPS /// - static bool download_winhttp(const Filesystem& fs, + static bool download_winhttp(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const Filesystem& fs, const Path& download_path_part_path, - SplitURIView split_uri, - const std::string& url, - const std::vector& secrets, - std::vector& errors, - MessageSink& progress_sink) + SplitUrlView split_url_view, + const SanitizedUrl& sanitized_url) { // `download_winhttp` does not support user or port syntax in authorities - auto hostname = split_uri.authority.value_or_exit(VCPKG_LINE_INFO).substr(2); + auto hostname = split_url_view.authority.value_or_exit(VCPKG_LINE_INFO).substr(2); INTERNET_PORT port; - if (split_uri.scheme == "https") + if (split_url_view.scheme == "https") { port = INTERNET_DEFAULT_HTTPS_PORT; } - else if (split_uri.scheme == "http") + else if (split_url_view.scheme == "http") { port = INTERNET_DEFAULT_HTTP_PORT; } @@ -731,39 +978,105 @@ namespace vcpkg const auto dir = download_path_part_path.parent_path(); fs.create_directories(dir, VCPKG_LINE_INFO); - const auto sanitized_url = replace_secrets(url, secrets); - static auto s = WinHttpSession::make(sanitized_url).value_or_exit(VCPKG_LINE_INFO); - for (size_t trials = 0; trials < 4; ++trials) + WinHttpSession s; + if (!s.open(context, sanitized_url)) { - if (trials > 0) - { - // 1s, 2s, 4s - const auto trialMs = 500 << trials; - msg::println_warning(msgDownloadFailedRetrying, msg::value = trialMs); - std::this_thread::sleep_for(std::chrono::milliseconds(trialMs)); - } + return false; + } + + AttemptDiagnosticContext adc{context}; + switch (download_winhttp_trial(adc, + machine_readable_progress, + fs, + s, + download_path_part_path, + split_url_view, + hostname, + port, + sanitized_url)) + { + case WinHttpTrialResult::succeeded: adc.commit(); return true; + case WinHttpTrialResult::failed: adc.commit(); return false; + case WinHttpTrialResult::retry: break; + } - switch (download_winhttp_trial( - fs, s, download_path_part_path, split_uri, hostname, port, sanitized_url, errors, progress_sink)) + for (size_t trials = 1; trials < 4; ++trials) + { + // 1s, 2s, 4s + const auto trialMs = 500 << trials; + adc.handle(); + context.statusln( + DiagnosticLine(DiagKind::Warning, + msg::format(msgDownloadFailedRetrying, msg::value = trialMs, msg::url = sanitized_url)) + .to_message_line()); + std::this_thread::sleep_for(std::chrono::milliseconds(trialMs)); + switch (download_winhttp_trial(adc, + machine_readable_progress, + fs, + s, + download_path_part_path, + split_url_view, + hostname, + port, + sanitized_url)) { - case WinHttpTrialResult::failed: return false; - case WinHttpTrialResult::succeeded: return true; + case WinHttpTrialResult::succeeded: adc.commit(); return true; + case WinHttpTrialResult::failed: adc.commit(); return false; case WinHttpTrialResult::retry: break; } } + adc.commit(); return false; } #endif - static bool try_download_file(const Filesystem& fs, - const std::string& url, - View headers, - const Path& download_path, - const Optional& sha512, - const std::vector& secrets, - std::vector& errors, - MessageSink& progress_sink) + enum class DownloadPrognosis + { + Success, + OtherError, + NetworkErrorProxyMightHelp + }; + + static bool check_combine_download_prognosis(DownloadPrognosis& target, DownloadPrognosis individual_call) + { + switch (individual_call) + { + case DownloadPrognosis::Success: return true; + case DownloadPrognosis::OtherError: + if (target == DownloadPrognosis::Success) + { + target = DownloadPrognosis::OtherError; + } + + return false; + case DownloadPrognosis::NetworkErrorProxyMightHelp: + if (target == DownloadPrognosis::Success || target == DownloadPrognosis::OtherError) + { + target = DownloadPrognosis::NetworkErrorProxyMightHelp; + } + + return false; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + static void maybe_report_proxy_might_help(DiagnosticContext& context, DownloadPrognosis prognosis) + { + if (prognosis == DownloadPrognosis::NetworkErrorProxyMightHelp) + { + context.report(DiagnosticLine{DiagKind::Note, msg::format(msgDownloadFailedProxySettings)}); + } + } + + static DownloadPrognosis try_download_file(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const Filesystem& fs, + StringView raw_url, + const SanitizedUrl& sanitized_url, + View headers, + const Path& download_path, + const Optional& maybe_sha512) { auto download_path_part_path = download_path; download_path_part_path += "."; @@ -777,119 +1090,143 @@ namespace vcpkg #if defined(_WIN32) auto maybe_https_proxy_env = get_environment_variable(EnvironmentVariableHttpsProxy); bool needs_proxy_auth = false; - if (maybe_https_proxy_env) + if (auto proxy_url = maybe_https_proxy_env.get()) { - const auto& proxy_url = maybe_https_proxy_env.value_or_exit(VCPKG_LINE_INFO); - needs_proxy_auth = proxy_url.find('@') != std::string::npos; + needs_proxy_auth = proxy_url->find('@') != std::string::npos; } if (headers.size() == 0 && !needs_proxy_auth) { - auto split_uri = split_uri_view(url).value_or_exit(VCPKG_LINE_INFO); - if (split_uri.scheme == "https" || split_uri.scheme == "http") + auto maybe_split_uri_view = parse_split_url_view(raw_url); + auto split_uri_view = maybe_split_uri_view.get(); + if (!split_uri_view) { - auto maybe_authority = split_uri.authority.get(); + context.report_error(msgInvalidUri, msg::value = sanitized_url); + return DownloadPrognosis::OtherError; + } + + if (split_uri_view->scheme == "https" || split_uri_view->scheme == "http") + { + auto maybe_authority = split_uri_view->authority.get(); if (!maybe_authority) { - Checks::msg_exit_with_error(VCPKG_LINE_INFO, msgInvalidUri, msg::value = url); + context.report_error(msg::format(msgInvalidUri, msg::value = sanitized_url)); + return DownloadPrognosis::OtherError; } - auto authority = maybe_authority->substr(2); - // This check causes complex URLs (non-default port, embedded basic auth) to be passed down to curl.exe + auto authority = StringView{*maybe_authority}.substr(2); + // This check causes complex URLs (non-default port, embedded basic auth) to be passed down to + // curl.exe if (Strings::find_first_of(authority, ":@") == authority.end()) { - if (download_winhttp(fs, download_path_part_path, split_uri, url, secrets, errors, progress_sink)) + if (!download_winhttp(context, + machine_readable_progress, + fs, + download_path_part_path, + *split_uri_view, + sanitized_url)) { - auto maybe_hash_check = check_downloaded_file_hash(fs, sha512, url, download_path_part_path); - if (maybe_hash_check.has_value()) - { - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); - return true; - } - else - { - errors.push_back(std::move(maybe_hash_check).error()); - } + return DownloadPrognosis::NetworkErrorProxyMightHelp; } - return false; + + if (!check_downloaded_file_hash(context, fs, sanitized_url, download_path_part_path, maybe_sha512)) + { + return DownloadPrognosis::OtherError; + } + + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return DownloadPrognosis::Success; } } } #endif auto cmd = Command{"curl"} .string_arg("--fail") + .string_arg("--retry") + .string_arg("3") .string_arg("-L") - .string_arg(url_encode_spaces(url)) + .string_arg(url_encode_spaces(raw_url)) .string_arg("--create-dirs") .string_arg("--output") .string_arg(download_path_part_path); - for (auto&& header : headers) - { - cmd.string_arg("-H").string_arg(header); - } - - std::string non_progress_content; - auto maybe_exit_code = cmd_execute_and_stream_lines(cmd, [&](StringView line) { + add_curl_headers(cmd, headers); + bool seen_any_curl_errors = false; + // if seen_any_curl_errors, contains the curl error lines starting with "curl:" + // otherwise, contains all curl's output unless it is the machine readable output + std::vector likely_curl_errors; + auto maybe_exit_code = cmd_execute_and_stream_lines(context, cmd, [&](StringView line) { const auto maybe_parsed = try_parse_curl_progress_data(line); if (const auto parsed = maybe_parsed.get()) { - progress_sink.print(Color::none, fmt::format("{}%\n", parsed->total_percent)); - } - else - { - non_progress_content.append(line.data(), line.size()); - non_progress_content.push_back('\n'); + machine_readable_progress.println(Color::none, + LocalizedString::from_raw(fmt::format("{}%", parsed->total_percent))); + return; } - }); - const auto sanitized_url = replace_secrets(url, secrets); - if (const auto exit_code = maybe_exit_code.get()) - { - if (*exit_code != 0) + static constexpr StringLiteral WarningColon = "warning: "; + if (Strings::case_insensitive_ascii_starts_with(line, WarningColon)) { - errors.push_back( - msg::format_error(msgDownloadFailedCurl, msg::url = sanitized_url, msg::exit_code = *exit_code) - .append_raw('\n') - .append_raw(non_progress_content)); - return false; + context.statusln( + DiagnosticLine{DiagKind::Warning, LocalizedString::from_raw(line.substr(WarningColon.size()))} + .to_message_line()); + return; } - auto maybe_hash_check = check_downloaded_file_hash(fs, sha512, sanitized_url, download_path_part_path); - if (maybe_hash_check.has_value()) + // clang-format off + // example: + // 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (6) Could not resolve host: nonexistent.example.com + // clang-format on + static constexpr StringLiteral CurlColon = "curl:"; + auto curl_start = std::search(line.begin(), line.end(), CurlColon.begin(), CurlColon.end()); + if (curl_start == line.end()) { - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); - return true; + if (seen_any_curl_errors) + { + return; + } + + curl_start = line.begin(); } else { - errors.push_back(std::move(maybe_hash_check).error()); + if (!seen_any_curl_errors) + { + seen_any_curl_errors = true; + likely_curl_errors.clear(); + } } - } - else + + likely_curl_errors.emplace_back(curl_start, line.end()); + }); + + const auto exit_code = maybe_exit_code.get(); + if (!exit_code) { - errors.push_back(std::move(maybe_exit_code).error()); + return DownloadPrognosis::OtherError; } - return false; - } - - static Optional try_download_file(const Filesystem& fs, - View urls, - View headers, - const Path& download_path, - const Optional& sha512, - const std::vector& secrets, - std::vector& errors, - MessageSink& progress_sink) - { - for (auto&& url : urls) + if (*exit_code != 0) { - if (try_download_file(fs, url, headers, download_path, sha512, secrets, errors, progress_sink)) + std::set seen_errors; + for (StringView likely_curl_error : likely_curl_errors) { - return url; + auto seen_position = seen_errors.lower_bound(likely_curl_error); + if (seen_position == seen_errors.end() || *seen_position != likely_curl_error) + { + seen_errors.emplace_hint(seen_position, likely_curl_error); + context.report(DiagnosticLine{DiagKind::Error, LocalizedString::from_raw(likely_curl_error)}); + } } + + return DownloadPrognosis::NetworkErrorProxyMightHelp; } - return nullopt; + if (!check_downloaded_file_hash(context, fs, sanitized_url, download_path_part_path, maybe_sha512)) + { + return DownloadPrognosis::OtherError; + } + + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return DownloadPrognosis::Success; } View azure_blob_headers() @@ -898,11 +1235,14 @@ namespace vcpkg return s_headers; } - void parse_curl_status_line(std::vector>& http_codes, StringLiteral prefix, StringView this_line) + bool parse_curl_status_line(DiagnosticContext& context, + std::vector& http_codes, + StringLiteral prefix, + StringView this_line) { if (!Strings::starts_with(this_line, prefix)) { - return; + return false; } auto first = this_line.begin(); @@ -917,7 +1257,7 @@ namespace vcpkg { // this output is broken, even if we don't know %{exit_code} or ${errormsg}, the spaces in front // of them should still be printed. - return; + return false; } if (!ParserBase::is_ascii_digit(*first)) @@ -930,20 +1270,20 @@ namespace vcpkg if (*first != ' ' || ++first == last) { // didn't see the space after the http_code - return; + return false; } if (*first == ' ') { // old curl that doesn't understand %{exit_code}, this is the space after it http_codes.emplace_back(http_code); - return; + return false; } if (!ParserBase::is_ascii_digit(*first)) { // not exit_code - return; + return false; } const auto first_exit_code = first; @@ -952,204 +1292,540 @@ namespace vcpkg if (++first == last) { // didn't see the space after %{exit_code} - return; + return false; } if (*first == ' ') { // the space after exit_code, everything after this space is the error message if any + http_codes.emplace_back(http_code); auto exit_code = Strings::strto(StringView{first_exit_code, first}).value_or_exit(VCPKG_LINE_INFO); - if (exit_code == 0) + // note that this gets the space out of the output :) + if (exit_code != 0) { - // success! - http_codes.emplace_back(http_code); - return; + context.report_error(msg::format(msgCurlFailedGeneric, msg::exit_code = exit_code) + .append_raw(StringView{first, last})); } - // note that this gets the space out of the output :) - http_codes.emplace_back( - msg::format(msgCurlFailedGeneric, msg::exit_code = exit_code).append_raw(StringView{first, last})); - return; + return true; } if (!ParserBase::is_ascii_digit(*first)) { // non numeric exit_code? - return; + return false; } } } - bool DownloadManager::get_block_origin() const { return m_config.m_block_origin; } + static DownloadPrognosis download_file_azurl_asset_cache(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const Path& download_path, + StringView target_filename, + const Optional& maybe_sha512) + { + auto read_template = asset_cache_settings.m_read_url_template.get(); + auto sha512 = maybe_sha512.get(); + if (!read_template || !sha512) + { + // can't use http asset caches when none are configured or we don't have a SHA + return DownloadPrognosis::OtherError; + } - bool DownloadManager::asset_cache_configured() const { return m_config.m_read_url_template.has_value(); } + auto raw_read_url = Strings::replace_all(*read_template, "", *sha512); + SanitizedUrl sanitized_read_url{raw_read_url, asset_cache_settings.m_secrets}; + context.statusln(msg::format(msgAssetCacheConsult, msg::path = target_filename, msg::url = sanitized_read_url)); + return try_download_file(context, + machine_readable_progress, + fs, + raw_read_url, + sanitized_read_url, + asset_cache_settings.m_read_headers, + download_path, + maybe_sha512); + } - void DownloadManager::download_file(const Filesystem& fs, - const std::string& url, - View headers, - const Path& download_path, - const Optional& sha512, - MessageSink& progress_sink) const + static void report_script_while_command_line(DiagnosticContext& context, const std::string& raw_command) { - this->download_file(fs, View(&url, 1), headers, download_path, sha512, progress_sink); + context.report(DiagnosticLine{ + DiagKind::Note, + msg::format(msgWhileRunningAssetCacheScriptCommandLine).append_raw(": ").append_raw(raw_command)}); } - std::string DownloadManager::download_file(const Filesystem& fs, - View urls, - View headers, - const Path& download_path, - const Optional& sha512, - MessageSink& progress_sink) const + static void report_script_failed_to_make_file(DiagnosticContext& context, + const std::string& raw_command, + const Path& download_path_part_path) { - std::vector errors; - bool block_origin_enabled = m_config.m_block_origin; + context.report(DiagnosticLine{ + DiagKind::Error, download_path_part_path, msg::format(msgAssetCacheScriptFailedToWriteFile)}); + context.report(DiagnosticLine{ + DiagKind::Note, msg::format(msgAssetCacheScriptCommandLine).append_raw(": ").append_raw(raw_command)}); + } - if (urls.size() == 0) + static void report_asset_cache_authoritative_urls(DiagnosticContext& context, + DiagKind first_message_kind, + msg::MessageT first_message, + const std::vector& sanitized_urls) + { + auto first_sanitized_url = sanitized_urls.begin(); + const auto last_sanitized_url = sanitized_urls.end(); + if (first_sanitized_url != last_sanitized_url) { - if (auto hash = sha512.get()) - { - errors.push_back(msg::format_error(msgNoUrlsAndHashSpecified, msg::sha = *hash)); - } - else + context.report( + DiagnosticLine{first_message_kind, msg::format(first_message, msg::url = *first_sanitized_url)}); + while (++first_sanitized_url != last_sanitized_url) { - errors.push_back(msg::format_error(msgNoUrlsAndNoHashSpecified)); + context.report( + DiagnosticLine{DiagKind::Note, msg::format(msgDownloadOrUrl, msg::url = *first_sanitized_url)}); } } + } + + static DownloadPrognosis download_file_script_asset_cache(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + View raw_urls, + const std::vector& sanitized_urls, + const Path& download_path, + StringView target_filename, + const Optional& maybe_sha512) + { + using Hash::HashPrognosis; + auto script = asset_cache_settings.m_script.get(); + if (!script) + { + return DownloadPrognosis::OtherError; + } - if (auto hash = sha512.get()) + if (raw_urls.empty() && !maybe_sha512.has_value()) { - if (auto read_template = m_config.m_read_url_template.get()) + Checks::unreachable(VCPKG_LINE_INFO); + } + + context.statusln(msg::format(msgAssetCacheConsultScript, msg::path = target_filename)); + const auto download_path_part_path = fmt::format("{}.{}.part", download_path, get_process_id()); + Lazy escaped_url; + const auto escaped_dpath = Command(download_path_part_path).extract(); + auto maybe_raw_command = api_stable_format(context, *script, [&](std::string& out, StringView key) { + if (key == "url") { - auto read_url = Strings::replace_all(*read_template, "", *hash); - if (try_download_file(fs, - read_url, - m_config.m_read_headers, - download_path, - sha512, - m_config.m_secrets, - errors, - progress_sink)) - { - msg::println(msgAssetCacheHit, - msg::path = download_path.filename(), - msg::url = replace_secrets(read_url, m_config.m_secrets)); - return read_url; - } - else if (block_origin_enabled) + if (raw_urls.empty()) { - msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename()); + context.report_error(msg::format(msgAssetCacheScriptNeedsUrl, + msg::value = *script, + msg::sha = maybe_sha512.value_or_exit(VCPKG_LINE_INFO))); + return false; } - else + + Strings::append(out, escaped_url.get_lazy([&] { return Command(raw_urls[0]).extract(); })); + return true; + } + + if (key == "sha512") + { + if (auto sha512 = maybe_sha512.get()) { - msg::println(msgAssetCacheMiss, msg::url = urls[0]); + Strings::append_ascii_lowercase(out, *sha512); + return true; } + + context.report_error( + msg::format(msgAssetCacheScriptNeedsSha, msg::value = *script, msg::url = sanitized_urls[0])); + return false; } - else if (auto script = m_config.m_script.get()) + + if (key == "dst") { - if (urls.size() != 0) - { - const auto download_path_part_path = download_path + fmt::format(".{}.part", get_process_id()); - const auto escaped_url = Command(urls[0]).extract(); - auto escaped_sha512 = Command(*hash).extract(); - Strings::inplace_ascii_to_lowercase(escaped_sha512); - const auto escaped_dpath = Command(download_path_part_path).extract(); - Command cmd; - cmd.raw_arg(api_stable_format(*script, [&](std::string& out, StringView key) { - if (key == "url") - { - Strings::append(out, escaped_url); - } - else if (key == "sha512") - { - Strings::append(out, escaped_sha512); - } - else if (key == "dst") - { - Strings::append(out, escaped_dpath); - } - }).value_or_exit(VCPKG_LINE_INFO)); - - RedirectedProcessLaunchSettings settings; - settings.environment = get_clean_environment(); - settings.echo_in_debug = EchoInDebug::Show; - - auto maybe_res = flatten(cmd_execute_and_capture_output(cmd, settings), ""); - if (maybe_res) - { - auto maybe_success = - try_verify_downloaded_file_hash(fs, "", download_path_part_path, *hash); - if (maybe_success) - { - fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); - msg::println(msgDownloadSuccesful, msg::path = download_path.filename()); - return urls[0]; - } - msg::println_error(maybe_success.error()); - } - else + Strings::append(out, escaped_dpath); + return true; + } + + context.report_error(msg::format(msgAssetCacheScriptBadVariable, msg::value = *script, msg::list = key)); + context.report( + DiagnosticLine{DiagKind::Note, msg::format(msgAssetCacheScriptBadVariableHint, msg::list = key)}); + return false; + }); + + auto raw_command = maybe_raw_command.get(); + if (!raw_command) + { + return DownloadPrognosis::OtherError; + } + + Command cmd; + cmd.raw_arg(*raw_command); + RedirectedProcessLaunchSettings settings; + settings.environment = get_clean_environment(); + auto maybe_res = cmd_execute_and_stream_lines( + context, cmd, settings, [&](StringView line) { context.statusln(LocalizedString::from_raw(line)); }); + auto res = maybe_res.get(); + if (!res) + { + report_script_while_command_line(context, *raw_command); + return DownloadPrognosis::OtherError; + } + + if (*res != 0) + { + context.report_error(msg::format(msgAssetCacheScriptFailed, msg::exit_code = *res)); + context.report(DiagnosticLine{ + DiagKind::Note, msg::format(msgAssetCacheScriptCommandLine).append_raw(": ").append_raw(*raw_command)}); + return DownloadPrognosis::OtherError; + } + + auto sha512 = maybe_sha512.get(); + if (sha512) + { + auto hash_result = Hash::get_file_hash(context, fs, download_path_part_path, Hash::Algorithm::Sha512); + switch (hash_result.prognosis) + { + case HashPrognosis::Success: + if (Strings::case_insensitive_ascii_equals(*sha512, hash_result.hash)) { - msg::println_error(maybe_res.error()); + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return DownloadPrognosis::Success; } - } + + context.report(DiagnosticLine{DiagKind::Error, + download_path_part_path, + msg::format(msgAssetCacheScriptFailedToWriteCorrectHash)}); + context.report(DiagnosticLine{ + DiagKind::Note, + msg::format(msgAssetCacheScriptCommandLine).append_raw(": ").append_raw(*raw_command)}); + context.report(DiagnosticLine{ + DiagKind::Note, msg::format(msgDownloadFailedHashMismatchExpectedHash, msg::sha = *sha512)}); + context.report(DiagnosticLine{ + DiagKind::Note, + msg::format(msgDownloadFailedHashMismatchActualHash, msg::sha = hash_result.hash)}); + return DownloadPrognosis::OtherError; + case HashPrognosis::FileNotFound: + report_script_failed_to_make_file(context, *raw_command, download_path_part_path); + return DownloadPrognosis::OtherError; + case HashPrognosis::OtherError: + report_script_while_command_line(context, *raw_command); + return DownloadPrognosis::OtherError; + default: Checks::unreachable(VCPKG_LINE_INFO); } } - if (block_origin_enabled) + if (fs.exists(download_path_part_path, VCPKG_LINE_INFO)) { - msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename()); + fs.rename(download_path_part_path, download_path, VCPKG_LINE_INFO); + return DownloadPrognosis::Success; } - else + + report_script_failed_to_make_file(context, *raw_command, download_path_part_path); + return DownloadPrognosis::OtherError; + } + + static DownloadPrognosis download_file_asset_cache(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + View raw_urls, + const std::vector& sanitized_urls, + const Path& download_path, + StringView target_filename, + const Optional& maybe_sha512) + { + switch (download_file_azurl_asset_cache( + context, machine_readable_progress, asset_cache_settings, fs, download_path, target_filename, maybe_sha512)) + { + case DownloadPrognosis::Success: return DownloadPrognosis::Success; + case DownloadPrognosis::OtherError: + return download_file_script_asset_cache(context, + asset_cache_settings, + fs, + raw_urls, + sanitized_urls, + download_path, + target_filename, + maybe_sha512); + case DownloadPrognosis::NetworkErrorProxyMightHelp: return DownloadPrognosis::NetworkErrorProxyMightHelp; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + static void report_download_success_and_maybe_upload(DiagnosticContext& context, + const Path& download_path, + StringView target_filename, + const AssetCachingSettings& asset_cache_settings, + const Optional& maybe_sha512) + { + auto hash = maybe_sha512.get(); + auto url_template = asset_cache_settings.m_write_url_template.get(); + if (hash && url_template && !url_template->empty()) { - if (urls.size() != 0) + auto raw_upload_url = Strings::replace_all(*url_template, "", *hash); + SanitizedUrl sanitized_upload_url{raw_upload_url, asset_cache_settings.m_secrets}; + context.statusln(msg::format( + msgDownloadSuccesfulUploading, msg::path = target_filename, msg::url = sanitized_upload_url)); + WarningDiagnosticContext wdc{context}; + if (!store_to_asset_cache(wdc, + raw_upload_url, + sanitized_upload_url, + "PUT", + asset_cache_settings.m_write_headers, + download_path)) { - msg::println(msgDownloadingUrl, msg::url = download_path.filename()); - auto maybe_url = try_download_file( - fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink); - if (auto url = maybe_url.get()) - { - msg::println(msgDownloadSuccesful, msg::path = download_path.filename()); + context.report(DiagnosticLine{DiagKind::Warning, + msg::format(msgFailedToStoreBackToMirror, + msg::path = target_filename, + msg::url = sanitized_upload_url)}); + } + } + else + { + context.statusln(msg::format(msgDownloadSuccesful, msg::path = target_filename)); + } + } - if (auto hash = sha512.get()) - { - auto maybe_push = put_file_to_mirror(fs, download_path, *hash); - if (!maybe_push) - { - msg::println_warning(msgFailedToStoreBackToMirror, - msg::path = download_path.filename(), - msg::url = replace_secrets(download_path.c_str(), m_config.m_secrets)); - msg::println(maybe_push.error()); - } - } + bool download_file_asset_cached(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const std::string& url, + View headers, + const Path& download_path, + const Optional& maybe_sha512) + { + return download_file_asset_cached(context, + machine_readable_progress, + asset_cache_settings, + fs, + View(&url, 1), + headers, + download_path, + maybe_sha512); + } + + bool download_file_asset_cached(DiagnosticContext& context, + MessageSink& machine_readable_progress, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + View raw_urls, + View headers, + const Path& download_path, + const Optional& maybe_sha512) + { + // Design goals: + // * We want it to be clear when asset cache(s) are used. This means not printing the authoritative URL in a + // 'downloading' message when we aren't looking at it. + // * We don't want to say that something is an error / failure unless it actually is. This means asset cache + // failures followed by authoritative success must print only success. This also means that we can't print + // asset cache errors immediately, since they might be 'eaten' by a subsequent authoritative success. + // * We want to print something before 'going to sleep' for network access ever, so if the machine where that + // network access is is being slow or whatever the user understands. + // * We want to print errors and warnings as close to when they happen as possible notwithstanding other goals. + // * We want to print the proxy warning if and only if a failure looks like it might be something a proxy could + // fix. For example, successful network access with the wrong SHA is not proxy-fixable. + // * If we are printing the proxy message, we want to take some effort to only print it once, and put it on the + // *last* HTTP failure we print. This avoids a ton of console spew and makes it likely to be near the end of + // failure output and thus not scrolled off the top of the console buffer. + // * We consider hash check failure the same as a network I/O failure, and let other sources 'fix' the problem. + // + // See examples of console output in asset-caching.ps1 + + // Note: no secrets for the input URLs + std::vector sanitized_urls = + Util::fmap(raw_urls, [&](const std::string& url) { return SanitizedUrl{url, {}}; }); + const auto last_sanitized_url = sanitized_urls.end(); + const auto target_filename = download_path.filename(); + bool can_read_asset_cache = false; + if (asset_cache_settings.m_read_url_template.has_value() && maybe_sha512.has_value()) + { + // url asset cache reads need a hash + can_read_asset_cache = true; + } + + if (asset_cache_settings.m_script.has_value() && (maybe_sha512.has_value() || !raw_urls.empty())) + { + // script asset cache reads need either a hash or a URL + can_read_asset_cache = true; + } - return *url; + if (raw_urls.empty()) + { + // try to fetch from asset cache only without a known URL + if (auto sha512 = maybe_sha512.get()) + { + if (can_read_asset_cache) + { + context.statusln( + msg::format(msgDownloadingAssetShaToFile, msg::sha = *sha512, msg::path = download_path)); } else { - msg::println(msgDownloadFailedProxySettings, - msg::path = download_path.filename(), - msg::url = "https://github.com/microsoft/vcpkg-tool/pull/77"); + context.report_error(msg::format( + msgDownloadingAssetShaWithoutAssetCache, msg::sha = *sha512, msg::path = download_path)); + return false; } } + else + { + context.report_error(msgNoUrlsAndNoHashSpecified); + return false; + } + } + + if (asset_cache_settings.m_block_origin && !can_read_asset_cache) + { + // this will emit msgAssetCacheMissBlockOrigin below, this message just ensures the filename is mentioned in + // the output at all + context.statusln(msg::format(msgDownloadingFile, msg::path = target_filename)); + } + + DownloadPrognosis asset_cache_prognosis = DownloadPrognosis::Success; + // the asset cache downloads might fail, but that's OK if we can download the file from an authoritative source + AttemptDiagnosticContext asset_cache_attempt_context{context}; + if (check_combine_download_prognosis(asset_cache_prognosis, + download_file_asset_cache(asset_cache_attempt_context, + machine_readable_progress, + asset_cache_settings, + fs, + raw_urls, + sanitized_urls, + download_path, + target_filename, + maybe_sha512))) + { + asset_cache_attempt_context.commit(); + if (raw_urls.empty()) + { + context.statusln(msg::format(msgAssetCacheHit)); + return true; + } + + auto first_sanitized_url = sanitized_urls.begin(); + LocalizedString overall_url; + overall_url.append_raw(first_sanitized_url->to_string()); + while (++first_sanitized_url != last_sanitized_url) + { + overall_url.append_raw(", ").append(msgDownloadOrUrl, msg::url = *first_sanitized_url); + } + + context.statusln(msg::format(msgAssetCacheHitUrl, msg::url = overall_url)); + return true; + } + + if (raw_urls.empty()) + { + asset_cache_attempt_context.commit(); + context.report_error( + msg::format(msgAssetCacheMissNoUrls, msg::sha = maybe_sha512.value_or_exit(VCPKG_LINE_INFO))); + maybe_report_proxy_might_help(context, asset_cache_prognosis); + return false; + } + + if (asset_cache_settings.m_block_origin) + { + asset_cache_attempt_context.commit(); + report_asset_cache_authoritative_urls( + context, DiagKind::Error, msgAssetCacheMissBlockOrigin, sanitized_urls); + maybe_report_proxy_might_help(context, asset_cache_prognosis); + return false; + } + + auto first_raw_url = raw_urls.begin(); + const auto last_raw_url = raw_urls.end(); + auto first_sanitized_url = sanitized_urls.begin(); + AttemptDiagnosticContext authoritative_attempt_context{context}; + DownloadPrognosis authoritative_prognosis = DownloadPrognosis::Success; + if (can_read_asset_cache) + { + context.statusln(msg::format(msgAssetCacheMiss, msg::url = *first_sanitized_url)); + } + else if (raw_urls.size() == 1) + { + context.statusln( + msg::format(msgDownloadingUrlToFile, msg::url = *first_sanitized_url, msg::path = target_filename)); + } + else + { + context.statusln(msg::format(msgDownloadingFileFirstAuthoritativeSource, + msg::path = target_filename, + msg::url = *first_sanitized_url)); + } + + if (check_combine_download_prognosis(authoritative_prognosis, + try_download_file(authoritative_attempt_context, + machine_readable_progress, + fs, + *first_raw_url, + *first_sanitized_url, + headers, + download_path, + maybe_sha512))) + { + asset_cache_attempt_context.handle(); + authoritative_attempt_context.handle(); + report_download_success_and_maybe_upload( + context, download_path, target_filename, asset_cache_settings, maybe_sha512); + return true; + } + + while (++first_sanitized_url, ++first_raw_url != last_raw_url) + { + context.statusln(msg::format(msgDownloadTryingAuthoritativeSource, msg::url = *first_sanitized_url)); + if (check_combine_download_prognosis(authoritative_prognosis, + try_download_file(authoritative_attempt_context, + machine_readable_progress, + fs, + *first_raw_url, + *first_sanitized_url, + headers, + download_path, + maybe_sha512))) + { + asset_cache_attempt_context.handle(); + authoritative_attempt_context.handle(); + report_download_success_and_maybe_upload( + context, download_path, target_filename, asset_cache_settings, maybe_sha512); + return true; + } } - for (LocalizedString& error : errors) + if (asset_cache_prognosis == DownloadPrognosis::NetworkErrorProxyMightHelp && + authoritative_prognosis != DownloadPrognosis::NetworkErrorProxyMightHelp) { - msg::println(error); + // reorder the proxy warning up to the asset cache prognosis if that's where it comes from + asset_cache_attempt_context.commit(); + maybe_report_proxy_might_help(context, asset_cache_prognosis); + authoritative_attempt_context.commit(); + return false; } - Checks::exit_fail(VCPKG_LINE_INFO); + check_combine_download_prognosis(authoritative_prognosis, asset_cache_prognosis); + asset_cache_attempt_context.commit(); + authoritative_attempt_context.commit(); + maybe_report_proxy_might_help(context, authoritative_prognosis); + return false; } - ExpectedL DownloadManager::put_file_to_mirror(const ReadOnlyFilesystem& fs, - const Path& file_to_put, - StringView sha512) const + bool store_to_asset_cache(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Path& file_to_put, + StringView sha512) { - auto maybe_mirror_url = Strings::replace_all(m_config.m_write_url_template.value_or(""), "", sha512); - if (!maybe_mirror_url.empty()) + if (auto url_template = asset_cache_settings.m_write_url_template.get()) { - return put_file(fs, maybe_mirror_url, m_config.m_secrets, m_config.m_write_headers, file_to_put); + if (url_template->empty()) + { + return true; + } + + auto raw_upload_url = Strings::replace_all(*url_template, "", sha512); + SanitizedUrl sanitized_upload_url{raw_upload_url, asset_cache_settings.m_secrets}; + return store_to_asset_cache(context, + raw_upload_url, + sanitized_upload_url, + "PUT", + asset_cache_settings.m_write_headers, + file_to_put); } - return 0; + + return true; } Optional try_parse_curl_max5_size(StringView sv) @@ -1267,8 +1943,8 @@ namespace vcpkg const auto last = curl_progress_line.end(); if (parse_curl_uint_impl(result.total_percent, first, last) || parse_curl_max5_impl(result.total_size, first, last) || - parse_curl_uint_impl(result.recieved_percent, first, last) || - parse_curl_max5_impl(result.recieved_size, first, last) || + parse_curl_uint_impl(result.received_percent, first, last) || + parse_curl_max5_impl(result.received_size, first, last) || parse_curl_uint_impl(result.transfer_percent, first, last) || parse_curl_max5_impl(result.transfer_size, first, last) || parse_curl_max5_impl(result.average_download_speed, first, last) || diff --git a/src/vcpkg/base/files.cpp b/src/vcpkg/base/files.cpp index e466e97163..73139ad88c 100644 --- a/src/vcpkg/base/files.cpp +++ b/src/vcpkg/base/files.cpp @@ -1374,7 +1374,7 @@ namespace vcpkg FilePointer::FilePointer(const Path& path) : m_fs(nullptr), m_path(path) { } FilePointer::FilePointer() noexcept : m_fs(nullptr), m_path{} { } - FilePointer::FilePointer(FilePointer&& other) noexcept : m_fs(other.m_fs), m_path(std ::move(other.m_path)) + FilePointer::FilePointer(FilePointer&& other) noexcept : m_fs(other.m_fs), m_path(std::move(other.m_path)) { other.m_fs = nullptr; other.m_path = {}; @@ -3912,8 +3912,7 @@ namespace vcpkg ls.append_indent().append_raw(as_preferred).append_raw('\n'); } - ls.append_raw('\n'); - msg_sink.print(ls); + msg_sink.println(ls); } uint64_t get_filesystem_stats() { return g_us_filesystem_stats.load(); } diff --git a/src/vcpkg/base/hash.cpp b/src/vcpkg/base/hash.cpp index c87bc6a1ec..c895108541 100644 --- a/src/vcpkg/base/hash.cpp +++ b/src/vcpkg/base/hash.cpp @@ -142,10 +142,14 @@ namespace vcpkg::Hash virtual std::string get_hash() override { + static constexpr unsigned long static_max_hash_size = 64; // SHA512, 512/8 = 64 const auto hash_size = get_hash_buffer_size(); - const auto buffer = std::make_unique(hash_size); - const auto hash = buffer.get(); + if (hash_size > static_max_hash_size) + { + Checks::unreachable(VCPKG_LINE_INFO, "hash buffer too small"); + } + uchar hash[static_max_hash_size]; const NTSTATUS error_code = BCryptFinishHash(hash_handle, hash, hash_size, 0); Checks::check_exit(VCPKG_LINE_INFO, NT_SUCCESS(error_code), "Failed to finalize the hash"); return to_hex(hash, hash + hash_size); @@ -518,20 +522,17 @@ namespace vcpkg::Hash static ReturnType do_hash(Algorithm algo, const F& f) { #if defined(_WIN32) - auto hasher = BCryptHasher(algo); - return f(hasher); + return f(BCryptHasher(algo)); #else switch (algo) { case Algorithm::Sha256: { - auto hasher = ShaHasher(); - return f(hasher); + return f(ShaHasher()); } case Algorithm::Sha512: { - auto hasher = ShaHasher(); - return f(hasher); + return f(ShaHasher()); } default: Checks::unreachable(VCPKG_LINE_INFO); } @@ -540,7 +541,7 @@ namespace vcpkg::Hash std::string get_bytes_hash(const void* first, const void* last, Algorithm algo) { - return do_hash(algo, [first, last](Hasher& hasher) { + return do_hash(algo, [first, last](Hasher&& hasher) { hasher.add_bytes(first, last); return hasher.get_hash(); }); @@ -553,17 +554,25 @@ namespace vcpkg::Hash std::string get_string_sha256(StringView s) { return get_string_hash(s, Hash::Algorithm::Sha256); } - ExpectedL get_file_hash(const ReadOnlyFilesystem& fs, const Path& path, Algorithm algo) + HashResult get_file_hash(DiagnosticContext& context, const ReadOnlyFilesystem& fs, const Path& path, Algorithm algo) { - Debug::println("Trying to hash ", path); + HashResult result; std::error_code ec; auto file = fs.open_for_read(path, ec); if (ec) { - return error_prefix().append(msgHashFileFailureToRead, msg::path = path).append_raw(ec.message()); + if (ec == std::errc::no_such_file_or_directory || ec == std::errc::not_a_directory) + { + result.prognosis = HashPrognosis::FileNotFound; + return result; + } + + context.report_error(format_filesystem_call_error(ec, "open_for_read", {path})); + result.prognosis = HashPrognosis::OtherError; + return result; } - return do_hash>(algo, [&](Hasher& hasher) -> ExpectedL { + result.hash = do_hash(algo, [&](Hasher&& hasher) { constexpr std::size_t buffer_size = 1024 * 32; char buffer[buffer_size]; do @@ -575,13 +584,37 @@ namespace vcpkg::Hash } else if ((ec = file.error())) { - return error_prefix().append(msgHashFileFailureToRead, msg::path = path).append_raw(ec.message()); + result.prognosis = HashPrognosis::OtherError; + context.report_error(format_filesystem_call_error(ec, "read", {path})); + return std::string(); } } while (!file.eof()); - auto result_hash = hasher.get_hash(); - Debug::print(fmt::format("{} has hash {}\n", path, result_hash)); - return result_hash; + return std::move(hasher).get_hash(); }); + + return result; + } + + Optional get_file_hash_required(DiagnosticContext& context, + const ReadOnlyFilesystem& fs, + const Path& path, + Algorithm algo) + { + auto result = get_file_hash(context, fs, path, algo); + switch (result.prognosis) + { + case HashPrognosis::Success: return result.hash; + case HashPrognosis::FileNotFound: + context.report(DiagnosticLine{DiagKind::Error, path, msg::format(msgFileNotFound)}); + return nullopt; + case HashPrognosis::OtherError: return nullopt; + default: Checks::unreachable(VCPKG_LINE_INFO); + } + } + + ExpectedL get_file_hash(const ReadOnlyFilesystem& fs, const Path& path, Algorithm algo) + { + return adapt_context_to_expected(get_file_hash_required, fs, path, algo); } } diff --git a/src/vcpkg/base/message_sinks.cpp b/src/vcpkg/base/message_sinks.cpp index f28f109550..c73c0c851c 100644 --- a/src/vcpkg/base/message_sinks.cpp +++ b/src/vcpkg/base/message_sinks.cpp @@ -6,28 +6,165 @@ namespace using namespace vcpkg; struct NullMessageSink : MessageSink { - virtual void print(Color, StringView) override { } + virtual void println(const MessageLine&) override { } + virtual void println(MessageLine&&) override { } + virtual void println(const LocalizedString&) override { } + virtual void println(LocalizedString&&) override { } + virtual void println(Color, const LocalizedString&) override { } + virtual void println(Color, LocalizedString&&) override { } }; NullMessageSink null_sink_instance; - struct OutMessageSink : MessageSink + struct OutMessageSink final : MessageSink { - virtual void print(Color c, StringView sv) override { msg::write_unlocalized_text(c, sv); } + virtual void println(const MessageLine& line) override + { + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text(segment.color, segment.text); + } + + msg::write_unlocalized_text(Color::none, "\n"); + } + + virtual void println(MessageLine&& line) override + { + auto& segments = line.get_segments(); + if (segments.empty()) + { + msg::write_unlocalized_text(Color::none, "\n"); + return; + } + + line.print(segments.back().color, "\n"); + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text(segment.color, segment.text); + } + } + virtual void println(const LocalizedString& text) override + { + msg::write_unlocalized_text(Color::none, text); + msg::write_unlocalized_text(Color::none, "\n"); + } + virtual void println(LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text(Color::none, text); + } + virtual void println(Color color, const LocalizedString& text) override + { + msg::write_unlocalized_text(color, text); + msg::write_unlocalized_text(Color::none, "\n"); + } + virtual void println(Color color, LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text(color, text); + } }; OutMessageSink out_sink_instance; - struct StdOutMessageSink : MessageSink + struct StdOutMessageSink final : MessageSink { - virtual void print(Color c, StringView sv) override { msg::write_unlocalized_text_to_stdout(c, sv); } + virtual void println(const MessageLine& line) override + { + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text_to_stdout(segment.color, segment.text); + } + + msg::write_unlocalized_text_to_stdout(Color::none, "\n"); + } + + virtual void println(MessageLine&& line) override + { + auto& segments = line.get_segments(); + if (segments.empty()) + { + msg::write_unlocalized_text_to_stdout(Color::none, "\n"); + return; + } + + line.print(segments.back().color, "\n"); + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text_to_stdout(segment.color, segment.text); + } + } + virtual void println(const LocalizedString& text) override + { + msg::write_unlocalized_text_to_stdout(Color::none, text); + msg::write_unlocalized_text_to_stdout(Color::none, "\n"); + } + virtual void println(LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text_to_stdout(Color::none, text); + } + virtual void println(Color color, const LocalizedString& text) override + { + msg::write_unlocalized_text_to_stdout(color, text); + msg::write_unlocalized_text_to_stdout(color, "\n"); + } + virtual void println(Color color, LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text_to_stdout(color, text); + } }; StdOutMessageSink stdout_sink_instance; - struct StdErrMessageSink : MessageSink + struct StdErrMessageSink final : MessageSink { - virtual void print(Color c, StringView sv) override { msg::write_unlocalized_text_to_stderr(c, sv); } + virtual void println(const MessageLine& line) override + { + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text_to_stderr(segment.color, segment.text); + } + + msg::write_unlocalized_text_to_stderr(Color::none, "\n"); + } + + virtual void println(MessageLine&& line) override + { + auto& segments = line.get_segments(); + if (segments.empty()) + { + msg::write_unlocalized_text_to_stderr(Color::none, "\n"); + return; + } + + line.print(segments.back().color, "\n"); + for (auto&& segment : line.get_segments()) + { + msg::write_unlocalized_text_to_stderr(segment.color, segment.text); + } + } + virtual void println(const LocalizedString& text) override + { + msg::write_unlocalized_text_to_stderr(Color::none, text); + msg::write_unlocalized_text_to_stderr(Color::none, "\n"); + } + virtual void println(LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text_to_stderr(Color::none, text); + } + virtual void println(Color color, const LocalizedString& text) override + { + msg::write_unlocalized_text_to_stderr(color, text); + msg::write_unlocalized_text_to_stderr(color, "\n"); + } + virtual void println(Color color, LocalizedString&& text) override + { + text.append_raw('\n'); + msg::write_unlocalized_text_to_stderr(color, text); + } }; StdErrMessageSink stderr_sink_instance; @@ -36,26 +173,110 @@ namespace namespace vcpkg { - void MessageSink::println_warning(const LocalizedString& s) { println(Color::warning, warning_prefix().append(s)); } - void MessageSink::println_error(const LocalizedString& s) { println(Color::error, error_prefix().append(s)); } + MessageLine::MessageLine(const LocalizedString& ls) { segments.push_back({Color::none, ls.data()}); } + MessageLine::MessageLine(LocalizedString&& ls) { segments.push_back({Color::none, std::move(ls).extract_data()}); } + void MessageLine::print(Color color, StringView text) + { + if (!segments.empty() && segments.back().color == color) + { + segments.back().text.append(text.data(), text.size()); + } + else + { + segments.push_back({color, std::string(text)}); + } + } + void MessageLine::print(StringView text) { print(Color::none, text); } + const std::vector& MessageLine::get_segments() const noexcept { return segments; } + + std::string MessageLine::to_string() const { return adapt_to_string(*this); } + void MessageLine::to_string(std::string& target) const + { + for (const auto& segment : segments) + { + target.append(segment.text); + } + } + + void MessageSink::println(const LocalizedString& s) + { + MessageLine line; + line.print(Color::none, s); + this->println(line); + } + + void MessageSink::println(LocalizedString&& s) { this->println(s); } + + void MessageSink::println(Color c, const LocalizedString& s) + { + MessageLine line; + line.print(c, s); + this->println(line); + } + + void MessageSink::println(Color c, LocalizedString&& s) { this->println(c, s); } MessageSink& null_sink = null_sink_instance; MessageSink& out_sink = out_sink_instance; MessageSink& stderr_sink = stderr_sink_instance; MessageSink& stdout_sink = stdout_sink_instance; - void FileSink::print(Color, StringView sv) + void FileSink::println(const MessageLine& line) { + std::string whole_line; + line.to_string(whole_line); + whole_line.push_back('\n'); Checks::msg_check_exit(VCPKG_LINE_INFO, - m_out_file.write(sv.data(), 1, sv.size()) == sv.size(), + m_out_file.write(whole_line.data(), 1, whole_line.size()) == whole_line.size(), msgErrorWhileWriting, msg::path = m_log_file); } - void CombiningSink::print(Color c, StringView sv) + void FileSink::println(MessageLine&& line) + { + line.print("\n"); + for (auto&& segment : line.get_segments()) + { + Checks::msg_check_exit(VCPKG_LINE_INFO, + m_out_file.write(segment.text.data(), 1, segment.text.size()) == segment.text.size(), + msgErrorWhileWriting, + msg::path = m_log_file); + } + } + + void TeeSink::println(const MessageLine& line) { - m_first.print(c, sv); - m_second.print(c, sv); + m_first.println(line); + m_second.println(line); } + void TeeSink::println(MessageLine&& line) + { + m_first.println(line); + m_second.println(std::move(line)); + } + + void TeeSink::println(const LocalizedString& line) + { + m_first.println(line); + m_second.println(line); + } + + void TeeSink::println(LocalizedString&& line) + { + m_first.println(line); + m_second.println(std::move(line)); + } + + void TeeSink::println(Color color, const LocalizedString& line) + { + m_first.println(color, line); + m_second.println(color, line); + } + + void TeeSink::println(Color color, LocalizedString&& line) + { + m_first.println(color, line); + m_second.println(color, std::move(line)); + } } diff --git a/src/vcpkg/base/strings.cpp b/src/vcpkg/base/strings.cpp index 16d6a686b3..8f4f5870b6 100644 --- a/src/vcpkg/base/strings.cpp +++ b/src/vcpkg/base/strings.cpp @@ -24,9 +24,10 @@ namespace vcpkg::Strings::details void append_internal(std::string& into, StringView s) { into.append(s.begin(), s.end()); } } -vcpkg::ExpectedL vcpkg::details::api_stable_format_impl(StringView sv, - void (*cb)(void*, std::string&, StringView), - void* user) +Optional vcpkg::details::api_stable_format_impl(DiagnosticContext& context, + StringView sv, + bool (*cb)(void*, std::string&, StringView), + void* user) { // Transforms similarly to std::format -- "{xyz}" -> f(xyz), "{{" -> "{", "}}" -> "}" @@ -46,7 +47,8 @@ vcpkg::ExpectedL vcpkg::details::api_stable_format_impl(StringView { if (p == last) { - return msg::format(msgInvalidFormatString, msg::actual = sv); + context.report_error(msg::format(msgInvalidFormatString, msg::actual = sv)); + return nullopt; } else if (*p == '{') { @@ -60,10 +62,15 @@ vcpkg::ExpectedL vcpkg::details::api_stable_format_impl(StringView p = std::find_first_of(p, last, s_brackets, s_brackets + 2); if (p == last || p[0] != '}') { - return msg::format(msgInvalidFormatString, msg::actual = sv); + context.report_error(msg::format(msgInvalidFormatString, msg::actual = sv)); + return nullopt; } // p[0] == '}' - cb(user, out, {seq_start, p}); + if (!cb(user, out, {seq_start, p})) + { + return nullopt; + } + prev = ++p; } } @@ -71,14 +78,16 @@ vcpkg::ExpectedL vcpkg::details::api_stable_format_impl(StringView { if (p == last || p[0] != '}') { - return msg::format(msgInvalidFormatString, msg::actual = sv); + context.report_error(msg::format(msgInvalidFormatString, msg::actual = sv)); + return nullopt; } out.push_back('}'); prev = ++p; } } + out.append(prev, last); - return {std::move(out), expected_left_tag}; + return out; } namespace @@ -193,7 +202,7 @@ void Strings::inplace_ascii_to_lowercase(std::string& s) std::string Strings::ascii_to_lowercase(StringView s) { std::string result; - std::transform(s.begin(), s.end(), std::back_inserter(result), tolower_char); + append_ascii_lowercase(result, s); return result; } @@ -204,6 +213,11 @@ std::string Strings::ascii_to_uppercase(StringView s) return result; } +void Strings::append_ascii_lowercase(std::string& target, StringView s) +{ + std::transform(s.begin(), s.end(), std::back_inserter(target), tolower_char); +} + bool Strings::case_insensitive_ascii_starts_with(StringView s, StringView pattern) { if (s.size() < pattern.size()) return false; diff --git a/src/vcpkg/base/system.process.cpp b/src/vcpkg/base/system.process.cpp index ffff5d17d6..799542c8e8 100644 --- a/src/vcpkg/base/system.process.cpp +++ b/src/vcpkg/base/system.process.cpp @@ -43,14 +43,6 @@ namespace { using namespace vcpkg; - LocalizedString format_system_error_message(StringLiteral api_name, ExitCodeIntegral error_value) - { - return msg::format_error(msgSystemApiErrorMessage, - msg::system_api = api_name, - msg::exit_code = error_value, - msg::error_msg = std::system_category().message(static_cast(error_value))); - } - static std::atomic_int32_t debug_id_counter{1000}; #if defined(_WIN32) struct CtrlCStateMachine @@ -790,14 +782,15 @@ namespace } }; - ExpectedL windows_create_process(std::int32_t debug_id, - ProcessInfo& process_info, - StringView command_line, - const Optional& working_directory, - const Optional& environment, - BOOL bInheritHandles, - DWORD dwCreationFlags, - STARTUPINFOEXW& startup_info) noexcept + bool windows_create_process(DiagnosticContext& context, + std::int32_t debug_id, + ProcessInfo& process_info, + StringView command_line, + const Optional& working_directory, + const Optional& environment, + BOOL bInheritHandles, + DWORD dwCreationFlags, + STARTUPINFOEXW& startup_info) noexcept { Debug::print(fmt::format("{}: CreateProcessW({})\n", debug_id, command_line)); @@ -840,24 +833,26 @@ namespace &startup_info.StartupInfo, &process_info)) { - return format_system_error_message("CreateProcessW", GetLastError()); + context.report_system_error("CreateProcessW", GetLastError()); + return false; } - return Unit{}; + return true; } // Used to, among other things, control which handles are inherited by child processes. // from https://devblogs.microsoft.com/oldnewthing/20111216-00/?p=8873 struct ProcAttributeList { - ExpectedL create(DWORD dwAttributeCount) + bool create(DiagnosticContext& context, DWORD dwAttributeCount) { Checks::check_exit(VCPKG_LINE_INFO, buffer.empty()); SIZE_T size = 0; if (InitializeProcThreadAttributeList(nullptr, dwAttributeCount, 0, &size) || GetLastError() != ERROR_INSUFFICIENT_BUFFER) { - return format_system_error_message("InitializeProcThreadAttributeList nullptr", GetLastError()); + context.report_system_error("InitializeProcThreadAttributeList nullptr", GetLastError()); + return false; } Checks::check_exit(VCPKG_LINE_INFO, size > 0); ASSUME(size > 0); @@ -865,18 +860,21 @@ namespace if (!InitializeProcThreadAttributeList( reinterpret_cast(buffer.data()), dwAttributeCount, 0, &size)) { - return format_system_error_message("InitializeProcThreadAttributeList attribute_list", GetLastError()); + context.report_system_error("InitializeProcThreadAttributeList attribute_list", GetLastError()); + return false; } - return Unit{}; + return true; } - ExpectedL update_attribute(DWORD_PTR Attribute, PVOID lpValue, SIZE_T cbSize) + bool update_attribute(DiagnosticContext& context, DWORD_PTR Attribute, PVOID lpValue, SIZE_T cbSize) { if (!UpdateProcThreadAttribute(get(), 0, Attribute, lpValue, cbSize, nullptr, nullptr)) { - return format_system_error_message("InitializeProcThreadAttributeList attribute_list", GetLastError()); + context.report_system_error("InitializeProcThreadAttributeList attribute_list", GetLastError()); + return false; } - return Unit{}; + + return true; } LPPROC_THREAD_ATTRIBUTE_LIST get() noexcept { @@ -912,17 +910,18 @@ namespace close_handle_mark_invalid(write_pipe); } - ExpectedL create() + bool create(DiagnosticContext& context) { Checks::check_exit(VCPKG_LINE_INFO, read_pipe == INVALID_HANDLE_VALUE); Checks::check_exit(VCPKG_LINE_INFO, write_pipe == INVALID_HANDLE_VALUE); SECURITY_ATTRIBUTES anonymousSa{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE}; if (!CreatePipe(&read_pipe, &write_pipe, &anonymousSa, 0)) { - return format_system_error_message("CreatePipe", GetLastError()); + context.report_system_error("CreatePipe", GetLastError()); + return false; } - return Unit{}; + return true; } }; @@ -960,7 +959,7 @@ namespace close_handle_mark_invalid(write_pipe); } - ExpectedL create(std::int32_t debug_id) + bool create(DiagnosticContext& context, std::int32_t debug_id) { Checks::check_exit(VCPKG_LINE_INFO, read_pipe == INVALID_HANDLE_VALUE); Checks::check_exit(VCPKG_LINE_INFO, write_pipe == INVALID_HANDLE_VALUE); @@ -981,17 +980,19 @@ namespace &namedPipeSa); if (write_pipe == INVALID_HANDLE_VALUE) { - return format_system_error_message("CreateNamedPipeW stdin", GetLastError()); + context.report_system_error("CreateNamedPipeW stdin", GetLastError()); + return false; } SECURITY_ATTRIBUTES openSa{sizeof(SECURITY_ATTRIBUTES), nullptr, TRUE}; read_pipe = CreateFileW(pipe_name.c_str(), FILE_GENERIC_READ, 0, &openSa, OPEN_EXISTING, 0, 0); if (read_pipe == INVALID_HANDLE_VALUE) { - return format_system_error_message("CreateFileW stdin", GetLastError()); + context.report_system_error("CreateFileW stdin", GetLastError()); + return false; } - return Unit{}; + return true; } }; @@ -1156,31 +1157,34 @@ namespace } } - ExpectedL create() + bool create(DiagnosticContext& context) { #if defined(__APPLE__) static std::mutex pipe_creation_lock; std::lock_guard lck{pipe_creation_lock}; if (pipe(pipefd)) { - return format_system_error_message("pipe", errno); + context.report_system_error("pipe", errno); + return false; } for (size_t idx = 0; idx < 2; ++idx) { if (fcntl(pipefd[idx], F_SETFD, FD_CLOEXEC)) { - return format_system_error_message("fcntl", errno); + context.report_system_error("fcntl", errno); + return false; } } #else // ^^^ Apple // !Apple vvv if (pipe2(pipefd, O_CLOEXEC)) { - return format_system_error_message("pipe2", errno); + context.report_system_error("pipe2", errno); + return false; } #endif // ^^^ !Apple - return Unit{}; + return true; } }; @@ -1198,15 +1202,16 @@ namespace PosixSpawnFileActions(const PosixSpawnFileActions&) = delete; PosixSpawnFileActions& operator=(const PosixSpawnFileActions&) = delete; - ExpectedL adddup2(int fd, int newfd) + bool adddup2(DiagnosticContext& context, int fd, int newfd) { const int error = posix_spawn_file_actions_adddup2(&actions, fd, newfd); if (error) { - return format_system_error_message("posix_spawn_file_actions_adddup2", error); + context.report_system_error("posix_spawn_file_actions_adddup2", error); + return false; } - return Unit{}; + return true; } }; @@ -1216,7 +1221,7 @@ namespace PosixPid() : pid{-1} { } - ExpectedL wait_for_termination() + Optional wait_for_termination(DiagnosticContext& context) { int exit_code = -1; if (pid != -1) @@ -1225,7 +1230,8 @@ namespace const auto child = waitpid(pid, &status, 0); if (child != pid) { - return format_system_error_message("waitpid", errno); + context.report_system_error("waitpid", errno); + return nullopt; } if (WIFEXITED(status)) @@ -1311,18 +1317,7 @@ namespace vcpkg return new_env; } #endif -} // namespace vcpkg - -namespace -{ - void debug_print_cmd_execute_background_failure(int32_t debug_id, const LocalizedString& error) - { - Debug::print(fmt::format("{}: cmd_execute_background() failed: {}\n", debug_id, error)); - } -} -namespace vcpkg -{ void cmd_execute_background(const Command& cmd_line) { const auto debug_id = debug_id_counter.fetch_add(1, std::memory_order_relaxed); @@ -1334,18 +1329,15 @@ namespace vcpkg startup_info_ex.StartupInfo.cb = sizeof(STARTUPINFOEXW); startup_info_ex.StartupInfo.dwFlags = STARTF_USESHOWWINDOW; startup_info_ex.StartupInfo.wShowWindow = SW_HIDE; - auto process_create = windows_create_process(debug_id, - process_info, - cmd_line.command_line(), - nullopt, - nullopt, - FALSE, - CREATE_NEW_CONSOLE | CREATE_BREAKAWAY_FROM_JOB, - startup_info_ex); - if (!process_create) - { - debug_print_cmd_execute_background_failure(debug_id, process_create.error()); - } + (void)windows_create_process(Debug::g_debugging ? console_diagnostic_context : null_diagnostic_context, + debug_id, + process_info, + cmd_line.command_line(), + nullopt, + nullopt, + FALSE, + CREATE_NEW_CONSOLE | CREATE_BREAKAWAY_FROM_JOB, + startup_info_ex); #else // ^^^ _WIN32 // !_WIN32 pid_t pid; @@ -1366,17 +1358,17 @@ namespace vcpkg argv.emplace_back(nullptr); int error = posix_spawn(&pid, "/bin/sh", nullptr /*file_actions*/, nullptr /*attrp*/, argv.data(), environ); - if (error) + if (error && Debug::g_debugging) { - debug_print_cmd_execute_background_failure(debug_id, format_system_error_message("posix_spawn", errno)); - return; + console_diagnostic_context.report_system_error("posix_spawn", errno); } #endif // ^^^ !_WIN32 } - static ExpectedL cmd_execute_impl(const Command& cmd, - const ProcessLaunchSettings& settings, - const int32_t debug_id) + static Optional cmd_execute_impl(DiagnosticContext& context, + const Command& cmd, + const ProcessLaunchSettings& settings, + const int32_t debug_id) { #if defined(_WIN32) STARTUPINFOEXW startup_info_ex; @@ -1386,10 +1378,10 @@ namespace vcpkg startup_info_ex.StartupInfo.wShowWindow = SW_HIDE; ProcAttributeList proc_attribute_list; - auto proc_attribute_list_create = proc_attribute_list.create(1); + auto proc_attribute_list_create = proc_attribute_list.create(context, 1); if (!proc_attribute_list_create) { - return std::move(proc_attribute_list_create).error(); + return nullopt; } constexpr size_t number_of_candidate_handles = 3; @@ -1399,17 +1391,18 @@ namespace vcpkg size_t number_of_handles = std::unique(handles_to_inherit, handles_to_inherit + number_of_candidate_handles) - handles_to_inherit; - auto maybe_error = proc_attribute_list.update_attribute( - PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles_to_inherit, number_of_handles * sizeof(HANDLE)); - if (!maybe_error.has_value()) + if (!proc_attribute_list.update_attribute( + context, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles_to_inherit, number_of_handles * sizeof(HANDLE))) { - return std::move(maybe_error).error(); + return nullopt; } + startup_info_ex.lpAttributeList = proc_attribute_list.get(); SpawnProcessGuard spawn_process_guard; ProcessInfo process_info; - auto process_create = windows_create_process(debug_id, + auto process_create = windows_create_process(context, + debug_id, process_info, cmd.command_line(), settings.working_directory, @@ -1419,11 +1412,12 @@ namespace vcpkg startup_info_ex); if (!process_create) { - return std::move(process_create).error(); + return nullopt; } return process_info.wait(); #else + (void)context; Command real_command_line_builder; if (const auto wd = settings.working_directory.get()) { @@ -1448,46 +1442,50 @@ namespace vcpkg #endif } - ExpectedL cmd_execute(const Command& cmd) + Optional cmd_execute(DiagnosticContext& context, const Command& cmd) { ProcessLaunchSettings default_process_launch_settings; - return cmd_execute(cmd, default_process_launch_settings); + return cmd_execute(context, cmd, default_process_launch_settings); } - ExpectedL cmd_execute(const Command& cmd, const ProcessLaunchSettings& settings) + Optional cmd_execute(DiagnosticContext& context, + const Command& cmd, + const ProcessLaunchSettings& settings) { const ElapsedTimer timer; const auto debug_id = debug_id_counter.fetch_add(1, std::memory_order_relaxed); - auto maybe_result = cmd_execute_impl(cmd, settings, debug_id); + auto maybe_exit_code = cmd_execute_impl(context, cmd, settings, debug_id); const auto elapsed = timer.us_64(); g_subprocess_stats += elapsed; - if (auto result = maybe_result.get()) + if (auto exit_code = maybe_exit_code.get()) { - Debug::print(fmt::format("{}: cmd_execute() returned {} after {} us\n", debug_id, *result, elapsed)); + Debug::print(fmt::format("{}: child process returned {} after {} us\n", debug_id, *exit_code, elapsed)); } else { Debug::print( - fmt::format("{}: cmd_execute() returned ({}) after {} us\n", debug_id, maybe_result.error(), elapsed)); + fmt::format("{}: cmd_execute() failed to launch child process after {} us\n", debug_id, elapsed)); } - return maybe_result; + return maybe_exit_code; } - ExpectedL cmd_execute_and_stream_lines(const Command& cmd, - const std::function& per_line_cb) + Optional cmd_execute_and_stream_lines(DiagnosticContext& context, + const Command& cmd, + const std::function& per_line_cb) { RedirectedProcessLaunchSettings default_redirected_process_launch_settings; - return cmd_execute_and_stream_lines(cmd, default_redirected_process_launch_settings, per_line_cb); + return cmd_execute_and_stream_lines(context, cmd, default_redirected_process_launch_settings, per_line_cb); } - ExpectedL cmd_execute_and_stream_lines(const Command& cmd, - const RedirectedProcessLaunchSettings& settings, - const std::function& per_line_cb) + Optional cmd_execute_and_stream_lines(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& per_line_cb) { Strings::LinesStream lines; - auto rc = - cmd_execute_and_stream_data(cmd, settings, [&](const StringView sv) { lines.on_data(sv, per_line_cb); }); + auto rc = cmd_execute_and_stream_data( + context, cmd, settings, [&](const StringView sv) { lines.on_data(sv, per_line_cb); }); lines.on_end(per_line_cb); return rc; } @@ -1503,7 +1501,7 @@ namespace std::size_t offset; // Write a hunk of data to `target`. If there is no more input to write, returns `true`. - ExpectedL do_write(int target) + Optional do_write(DiagnosticContext& context, int target) { const auto this_write = input.size() - offset; // Big enough to be big, small enough to avoid implementation limits @@ -1515,7 +1513,8 @@ namespace write(target, static_cast(input.data() + offset), this_write_clamped); if (actually_written < 0) { - return format_system_error_message("write", errno); + context.report_system_error("write", errno); + return nullopt; } offset += actually_written; @@ -1526,10 +1525,11 @@ namespace }; #endif // ^^^ !_WIN32 - ExpectedL cmd_execute_and_stream_data_impl(const Command& cmd, - const RedirectedProcessLaunchSettings& settings, - const std::function& data_cb, - uint32_t debug_id) + Optional cmd_execute_and_stream_data_impl(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& data_cb, + uint32_t debug_id) { #if defined(_WIN32) std::wstring as_utf16; @@ -1544,7 +1544,8 @@ namespace auto stdin_content_size_raw = stdin_content.size(); if (stdin_content_size_raw > MAXDWORD) { - return format_system_error_message("WriteFileEx", ERROR_INSUFFICIENT_BUFFER); + context.report_system_error("WriteFileEx", ERROR_INSUFFICIENT_BUFFER); + return nullopt; } auto stdin_content_size = static_cast(stdin_content_size_raw); @@ -1567,52 +1568,48 @@ namespace } // Create a pipe for the child process's STDIN. - auto stdin_create = process_info.stdin_pipe.create(debug_id); - if (!stdin_create) + if (!process_info.stdin_pipe.create(context, debug_id)) { - return std::move(stdin_create).error(); + return nullopt; } startup_info_ex.StartupInfo.hStdInput = process_info.stdin_pipe.read_pipe; // Create a pipe for the child process's STDOUT/STDERR. - auto stdout_create = process_info.stdout_pipe.create(); - if (!stdout_create) + if (!process_info.stdout_pipe.create(context)) { - return std::move(stdout_create).error(); + return nullopt; } startup_info_ex.StartupInfo.hStdOutput = process_info.stdout_pipe.write_pipe; startup_info_ex.StartupInfo.hStdError = process_info.stdout_pipe.write_pipe; ProcAttributeList proc_attribute_list; - auto proc_attribute_list_create = proc_attribute_list.create(1); - if (!proc_attribute_list_create) + if (!proc_attribute_list.create(context, 1)) { - return std::move(proc_attribute_list_create).error(); + return nullopt; } HANDLE handles_to_inherit[2] = {startup_info_ex.StartupInfo.hStdOutput, startup_info_ex.StartupInfo.hStdInput}; - auto maybe_error = proc_attribute_list.update_attribute( - PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles_to_inherit, 2 * sizeof(HANDLE)); - if (!maybe_error.has_value()) + if (!proc_attribute_list.update_attribute( + context, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, handles_to_inherit, 2 * sizeof(HANDLE))) { - return std::move(maybe_error).error(); + return nullopt; } - startup_info_ex.lpAttributeList = proc_attribute_list.get(); - auto process_create = windows_create_process(debug_id, - process_info.proc_info, - cmd.command_line(), - settings.working_directory, - settings.environment, - TRUE, - dwCreationFlags, - startup_info_ex); + startup_info_ex.lpAttributeList = proc_attribute_list.get(); - if (!process_create) + if (!windows_create_process(context, + debug_id, + process_info.proc_info, + cmd.command_line(), + settings.working_directory, + settings.environment, + TRUE, + dwCreationFlags, + startup_info_ex)) { - return std::move(process_create).error(); + return nullopt; } close_handle_mark_invalid(process_info.stdin_pipe.read_pipe); @@ -1675,27 +1672,32 @@ namespace fflush(stdout); AnonymousPipe child_input; + if (!child_input.create(context)) { - auto err = child_input.create(); - if (!err) - { - return std::move(err).error(); - } + return nullopt; } AnonymousPipe child_output; + if (!child_output.create(context)) { - auto err = child_output.create(); - if (!err) - { - return std::move(err).error(); - } + return nullopt; } PosixSpawnFileActions actions; - actions.adddup2(child_input.pipefd[0], 0); - actions.adddup2(child_output.pipefd[1], 1); - actions.adddup2(child_output.pipefd[1], 2); + if (!actions.adddup2(context, child_input.pipefd[0], 0)) + { + return nullopt; + } + + if (!actions.adddup2(context, child_output.pipefd[1], 1)) + { + return nullopt; + } + + if (!actions.adddup2(context, child_output.pipefd[1], 2)) + { + return nullopt; + } std::vector argv_builder; argv_builder.reserve(3); @@ -1716,7 +1718,8 @@ namespace int error = posix_spawn(&pid.pid, "/bin/sh", &actions.actions, nullptr, argv.data(), environ); if (error) { - return format_system_error_message("posix_spawn", error); + context.report_system_error("posix_spawn", error); + return nullopt; } close_mark_invalid(child_input.pipefd[0]); @@ -1732,10 +1735,11 @@ namespace { if (fcntl(child_input.pipefd[1], F_SETFL, O_NONBLOCK)) { - return format_system_error_message("fcntl", errno); + context.report_system_error("fcntl", errno); + return nullopt; } - auto maybe_done = stdin_tracker.do_write(child_input.pipefd[1]); + auto maybe_done = stdin_tracker.do_write(context, child_input.pipefd[1]); bool done = false; if (const auto done_first = maybe_done.get()) { @@ -1747,7 +1751,7 @@ namespace } else { - return std::move(maybe_done).error(); + return nullopt; } if (!done) @@ -1761,7 +1765,8 @@ namespace polls[1].events = POLLIN; if (poll(polls, 2, -1) < 0) { - return format_system_error_message("poll", errno); + context.report_system_error("poll", errno); + return nullopt; } if (polls[0].revents & POLLERR) @@ -1771,7 +1776,7 @@ namespace } else if (polls[0].revents & POLLOUT) { - auto maybe_next_done = stdin_tracker.do_write(child_input.pipefd[1]); + auto maybe_next_done = stdin_tracker.do_write(context, child_input.pipefd[1]); if (const auto next_done = maybe_next_done.get()) { if (*next_done) @@ -1782,7 +1787,7 @@ namespace } else { - return std::move(maybe_next_done).error(); + return nullopt; } } @@ -1791,7 +1796,8 @@ namespace auto read_amount = read(child_output.pipefd[0], buf, sizeof(buf)); if (read_amount < 0) { - return format_system_error_message("read", errno); + context.report_system_error("read", errno); + return nullopt; } // can't be 0 because poll told us otherwise @@ -1823,7 +1829,8 @@ namespace break; } - return format_system_error_message("read", error); + context.report_system_error("read", errno); + return nullopt; } if (read_amount == 0) @@ -1840,27 +1847,29 @@ namespace } } - return pid.wait_for_termination(); + return pid.wait_for_termination(context); #endif /// ^^^ !_WIN32 } } // unnamed namespace namespace vcpkg { - ExpectedL cmd_execute_and_stream_data(const Command& cmd, - const std::function& data_cb) + Optional cmd_execute_and_stream_data(DiagnosticContext& context, + const Command& cmd, + const std::function& data_cb) { RedirectedProcessLaunchSettings default_redirected_process_launch_settings; - return cmd_execute_and_stream_data(cmd, default_redirected_process_launch_settings, data_cb); + return cmd_execute_and_stream_data(context, cmd, default_redirected_process_launch_settings, data_cb); } - ExpectedL cmd_execute_and_stream_data(const Command& cmd, - const RedirectedProcessLaunchSettings& settings, - const std::function& data_cb) + Optional cmd_execute_and_stream_data(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings, + const std::function& data_cb) { const ElapsedTimer timer; const auto debug_id = debug_id_counter.fetch_add(1, std::memory_order_relaxed); - auto maybe_exit_code = cmd_execute_and_stream_data_impl(cmd, settings, data_cb, debug_id); + auto maybe_exit_code = cmd_execute_and_stream_data_impl(context, cmd, settings, data_cb, debug_id); const auto elapsed = timer.us_64(); g_subprocess_stats += elapsed; if (const auto exit_code = maybe_exit_code.get()) @@ -1874,17 +1883,18 @@ namespace vcpkg return maybe_exit_code; } - ExpectedL cmd_execute_and_capture_output(const Command& cmd) + Optional cmd_execute_and_capture_output(DiagnosticContext& context, const Command& cmd) { RedirectedProcessLaunchSettings default_redirected_process_launch_settings; - return cmd_execute_and_capture_output(cmd, default_redirected_process_launch_settings); + return cmd_execute_and_capture_output(context, cmd, default_redirected_process_launch_settings); } - ExpectedL cmd_execute_and_capture_output(const Command& cmd, - const RedirectedProcessLaunchSettings& settings) + Optional cmd_execute_and_capture_output(DiagnosticContext& context, + const Command& cmd, + const RedirectedProcessLaunchSettings& settings) { std::string output; - return cmd_execute_and_stream_data(cmd, settings, [&](StringView sv) { Strings::append(output, sv); }) + return cmd_execute_and_stream_data(context, cmd, settings, [&](StringView sv) { Strings::append(output, sv); }) .map([&](ExitCodeIntegral exit_code) { return ExitCodeAndOutput{exit_code, std::move(output)}; }); } @@ -1960,4 +1970,23 @@ namespace vcpkg expected_right_tag}; } + std::string* check_zero_exit_code(DiagnosticContext& context, + Optional& maybe_exit, + StringView exe_path) + { + if (auto exit = maybe_exit.get()) + { + if (exit->exit_code == 0) + { + return &exit->output; + } + + context.report( + DiagnosticLine{DiagKind::Error, + exe_path, + msg::format(msgProgramPathReturnedNonzeroExitCode, msg::exit_code = exit->exit_code)}); + } + + return nullptr; + } } // namespace vcpkg diff --git a/src/vcpkg/binarycaching.cpp b/src/vcpkg/binarycaching.cpp index 50f98f0830..e4b3eb595e 100644 --- a/src/vcpkg/binarycaching.cpp +++ b/src/vcpkg/binarycaching.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -381,10 +382,8 @@ namespace struct HTTPPutBinaryProvider : IWriteBinaryProvider { - HTTPPutBinaryProvider(const Filesystem& fs, - std::vector&& urls, - const std::vector& secrets) - : m_fs(fs), m_urls(std::move(urls)), m_secrets(secrets) + HTTPPutBinaryProvider(std::vector&& urls, const std::vector& secrets) + : m_urls(std::move(urls)), m_secrets(secrets) { } @@ -396,11 +395,14 @@ namespace for (auto&& templ : m_urls) { auto url = templ.instantiate_variables(request); - auto maybe_success = put_file(m_fs, url, m_secrets, templ.headers, zip_path); + PrintingDiagnosticContext pdc{msg_sink}; + WarningDiagnosticContext wdc{pdc}; + auto maybe_success = + store_to_asset_cache(wdc, url, SanitizedUrl{url, m_secrets}, "PUT", templ.headers, zip_path); if (maybe_success) + { count_stored++; - else - msg_sink.println(Color::warning, maybe_success.error()); + } } return count_stored; } @@ -409,7 +411,6 @@ namespace bool needs_zip_file() const override { return true; } private: - const Filesystem& m_fs; std::vector m_urls; std::vector m_secrets; }; @@ -439,20 +440,13 @@ namespace make_temp_archive_path(m_buildtrees, action.spec)); } - auto codes_maybe = download_files(url_paths, m_url_template.headers, m_secrets); - for (size_t i = 0; i < codes_maybe.size(); ++i) + WarningDiagnosticContext wdc{console_diagnostic_context}; + auto codes = download_files_no_cache(wdc, url_paths, m_url_template.headers, m_secrets); + for (size_t i = 0; i < codes.size(); ++i) { - auto const& maybe_code = codes_maybe[i]; - if (auto code = maybe_code.get()) - { - if (*code == 200) - { - out_zip_paths[i].emplace(std::move(url_paths[i].second), RemoveWhen::always); - } - } - else + if (codes[i] == 200) { - msg::println_warning(maybe_code.error()); + out_zip_paths[i].emplace(std::move(url_paths[i].second), RemoveWhen::always); } } } @@ -465,22 +459,14 @@ namespace urls.push_back(m_url_template.instantiate_variables(BinaryPackageReadInfo{*actions[idx]})); } - auto codes_maybe = url_heads(urls, {}, m_secrets); - for (size_t i = 0; i < codes_maybe.size(); ++i) + WarningDiagnosticContext wdc{console_diagnostic_context}; + auto codes = url_heads(wdc, urls, {}, m_secrets); + for (size_t i = 0; i < codes.size(); ++i) { - auto result_availability = CacheAvailability::unavailable; - if (const auto code = codes_maybe[i].get()) - { - if (*code == 200) - { - result_availability = CacheAvailability::available; - } - } - - out_status[i] = result_availability; + out_status[i] = codes[i] == 200 ? CacheAvailability::available : CacheAvailability::unavailable; } - for (size_t i = codes_maybe.size(); i < out_status.size(); ++i) + for (size_t i = codes.size(); i < out_status.size(); ++i) { out_status[i] = CacheAvailability::unavailable; } @@ -827,7 +813,9 @@ namespace m_token_header, m_accept_header.to_string(), }; - auto res = invoke_http_request("GET", headers, url); + + WarningDiagnosticContext wdc{console_diagnostic_context}; + auto res = invoke_http_request(wdc, "GET", headers, url); if (auto p = res.get()) { auto maybe_json = Json::parse_object(*p, m_url); @@ -861,21 +849,13 @@ namespace url_indices.push_back(idx); } - const auto codes_maybe = download_files(url_paths, {}, m_secrets); - for (size_t i = 0; i < codes_maybe.size(); ++i) + WarningDiagnosticContext wdc{console_diagnostic_context}; + const auto codes = download_files_no_cache(wdc, url_paths, {}, m_secrets); + for (size_t i = 0; i < codes.size(); ++i) { - auto const& maybe_code = codes_maybe[i]; - if (auto code = maybe_code.get()) + if (codes[i] == 200) { - if (*code == 200) - { - out_zip_paths[url_indices[i]].emplace(std::move(url_paths[i].second), RemoveWhen::always); - } - } - else - { - msg::println(maybe_code.error()); - continue; + out_zip_paths[url_indices[i]].emplace(std::move(url_paths[i].second), RemoveWhen::always); } } } @@ -916,7 +896,8 @@ namespace m_token_header, }; - auto res = invoke_http_request("POST", headers, m_url, stringify(payload)); + WarningDiagnosticContext wdc{console_diagnostic_context}; + auto res = invoke_http_request(wdc, "POST", headers, m_url, stringify(payload)); if (auto p = res.get()) { auto maybe_json = Json::parse_object(*p, m_url); @@ -932,7 +913,7 @@ namespace return {}; } - size_t push_success(const BinaryPackageWriteInfo& request, MessageSink&) override + size_t push_success(const BinaryPackageWriteInfo& request, MessageSink& msg_sink) override { if (!request.zip_path) return 0; @@ -952,8 +933,10 @@ namespace "Content-Range: bytes 0-" + std::to_string(cache_size) + "/*", }; - const auto url = m_url + "/" + std::to_string(*cacheId.get()); - if (put_file(m_fs, url, {}, custom_headers, zip_path, "PATCH")) + PrintingDiagnosticContext pdc{msg_sink}; + WarningDiagnosticContext wdc{pdc}; + const auto raw_url = m_url + "/" + std::to_string(*cacheId.get()); + if (store_to_asset_cache(wdc, raw_url, SanitizedUrl{raw_url, {}}, "PATCH", custom_headers, zip_path)) { Json::Object commit; commit.insert("size", std::to_string(cache_size)); @@ -963,15 +946,10 @@ namespace m_token_header, }; - auto res = invoke_http_request("POST", headers, url, stringify(commit)); - if (res) + if (invoke_http_request(wdc, "POST", headers, raw_url, stringify(commit))) { ++upload_count; } - else - { - msg::println(res.error()); - } } } return upload_count; @@ -1058,7 +1036,7 @@ namespace } else { - out_sink.println_warning(res.error()); + msg::println_warning(res.error()); } } } @@ -1117,7 +1095,7 @@ namespace } else { - msg_sink.println_warning(res.error()); + msg_sink.println(warning_prefix().append(std::move(res).error())); } } return upload_count; @@ -1857,16 +1835,19 @@ namespace } bool has_sha = false; bool has_other = false; - api_stable_format(url_template.url_template, [&](std::string&, StringView key) { - if (key == "sha") - { - has_sha = true; - } - else - { - has_other = true; - } - }); + api_stable_format( + null_diagnostic_context, url_template.url_template, [&](std::string&, StringView key) { + if (key == "sha") + { + has_sha = true; + } + else + { + has_other = true; + } + + return true; + }); if (!has_sha) { if (has_other) @@ -2037,30 +2018,38 @@ namespace vcpkg { LocalizedString UrlTemplate::valid() const { + BufferedDiagnosticContext bdc{out_sink}; std::vector invalid_keys; - auto result = api_stable_format(url_template, [&](std::string&, StringView key) { + auto result = api_stable_format(bdc, url_template, [&](std::string&, StringView key) { static constexpr StringLiteral valid_keys[] = {"name", "version", "sha", "triplet"}; if (!Util::Vectors::contains(valid_keys, key)) { invalid_keys.push_back(key.to_string()); } + + return true; }); - if (!result) + + if (!invalid_keys.empty()) { - return std::move(result).error(); + bdc.report_error(msg::format(msgUnknownVariablesInTemplate, + msg::value = url_template, + msg::list = Strings::join(", ", invalid_keys))); + result.clear(); } - if (!invalid_keys.empty()) + + if (result.has_value()) { - return msg::format(msgUnknownVariablesInTemplate, - msg::value = url_template, - msg::list = Strings::join(", ", invalid_keys)); + return {}; } - return {}; + + return LocalizedString::from_raw(std::move(bdc).to_string()); } std::string UrlTemplate::instantiate_variables(const BinaryPackageReadInfo& info) const { - return api_stable_format(url_template, + return api_stable_format(console_diagnostic_context, + url_template, [&](std::string& out, StringView key) { if (key == "version") { @@ -2080,10 +2069,12 @@ namespace vcpkg } else { - Debug::println("Unknown key: ", key); - // We do a input validation while parsing the config - Checks::unreachable(VCPKG_LINE_INFO); + Checks::unreachable( + VCPKG_LINE_INFO, + "used instantiate_variables without checking valid() first"); }; + + return true; }) .value_or_exit(VCPKG_LINE_INFO); } @@ -2263,7 +2254,7 @@ namespace vcpkg if (!s.url_templates_to_put.empty()) { ret.write.push_back( - std::make_unique(fs, std::move(s.url_templates_to_put), s.secrets)); + std::make_unique(std::move(s.url_templates_to_put), s.secrets)); } if (!s.gcs_write_prefixes.empty()) { @@ -2601,14 +2592,15 @@ namespace vcpkg } } -ExpectedL vcpkg::parse_download_configuration(const Optional& arg) +ExpectedL vcpkg::parse_download_configuration(const Optional& arg) { - if (!arg || arg.get()->empty()) return DownloadManagerConfig{}; + AssetCachingSettings result; + if (!arg || arg.get()->empty()) return result; get_global_metrics_collector().track_define(DefineMetric::AssetSource); AssetSourcesState s; - const auto source = Strings::concat("$", EnvironmentVariableXVcpkgAssetSources); + const auto source = format_environment_variable(EnvironmentVariableXVcpkgAssetSources); AssetSourcesParser parser(*arg.get(), source, &s); parser.parse(); if (auto err = parser.get_error()) @@ -2634,27 +2626,22 @@ ExpectedL vcpkg::parse_download_configuration(const Optio .append(msgSeeURL, msg::url = docs::assetcaching_url); } - Optional get_url; if (!s.url_templates_to_get.empty()) { - get_url = std::move(s.url_templates_to_get.back()); + result.m_read_url_template = std::move(s.url_templates_to_get.back()); } - Optional put_url; - std::vector put_headers; + if (!s.azblob_templates_to_put.empty()) { - put_url = std::move(s.azblob_templates_to_put.back()); + result.m_write_url_template = std::move(s.azblob_templates_to_put.back()); auto v = azure_blob_headers(); - put_headers.assign(v.begin(), v.end()); + result.m_write_headers.assign(v.begin(), v.end()); } - return DownloadManagerConfig{std::move(get_url), - std::vector{}, - std::move(put_url), - std::move(put_headers), - std::move(s.secrets), - s.block_origin, - s.script}; + result.m_secrets = std::move(s.secrets); + result.m_block_origin = s.block_origin; + result.m_script = std::move(s.script); + return result; } ExpectedL vcpkg::parse_binary_provider_configs(const std::string& env_string, @@ -2669,7 +2656,9 @@ ExpectedL vcpkg::parse_binary_provider_configs(const st return *err; } - BinaryConfigParser env_parser(env_string, "VCPKG_BINARY_SOURCES", &s); + // must live until the end of the function due to StringView in BinaryConfigParser + const auto source = format_environment_variable("VCPKG_BINARY_SOURCES"); + BinaryConfigParser env_parser(env_string, source, &s); env_parser.parse(); if (auto err = env_parser.get_error()) { diff --git a/src/vcpkg/commands.bootstrap-standalone.cpp b/src/vcpkg/commands.bootstrap-standalone.cpp index 6663836108..0c0a398273 100644 --- a/src/vcpkg/commands.bootstrap-standalone.cpp +++ b/src/vcpkg/commands.bootstrap-standalone.cpp @@ -27,7 +27,7 @@ namespace vcpkg { (void)args.parse_arguments(CommandBootstrapStandaloneMetadata); - DownloadManager download_manager{{}}; + AssetCachingSettings asset_cache_settings; const auto maybe_vcpkg_root_env = args.vcpkg_root_dir_env.get(); if (!maybe_vcpkg_root_env) { @@ -36,11 +36,17 @@ namespace vcpkg const auto vcpkg_root = fs.almost_canonical(*maybe_vcpkg_root_env, VCPKG_LINE_INFO); fs.create_directories(vcpkg_root, VCPKG_LINE_INFO); - auto tarball = - download_vcpkg_standalone_bundle(download_manager, fs, vcpkg_root).value_or_exit(VCPKG_LINE_INFO); + auto maybe_tarball = + download_vcpkg_standalone_bundle(console_diagnostic_context, asset_cache_settings, fs, vcpkg_root); + auto tarball = maybe_tarball.get(); + if (!tarball) + { + Checks::exit_fail(VCPKG_LINE_INFO); + } + fs.remove_all(vcpkg_root / "vcpkg-artifacts", VCPKG_LINE_INFO); - extract_tar(find_system_tar(fs).value_or_exit(VCPKG_LINE_INFO), tarball, vcpkg_root); - fs.remove(tarball, VCPKG_LINE_INFO); + extract_tar(find_system_tar(fs).value_or_exit(VCPKG_LINE_INFO), *tarball, vcpkg_root); + fs.remove(*tarball, VCPKG_LINE_INFO); Checks::exit_success(VCPKG_LINE_INFO); } } diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index 63b344e610..e9e80f5686 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -1062,7 +1062,7 @@ namespace vcpkg size_t error_count = 0; { FileSink file_sink{fs, stdoutlog, Append::YES}; - CombiningSink combo_sink{out_sink, file_sink}; + TeeSink combo_sink{out_sink, file_sink}; error_count = perform_post_build_lint_checks( action.spec, paths, pre_build_info, build_info, scfl.port_directory(), combo_sink); }; diff --git a/src/vcpkg/commands.ci-verify-versions.cpp b/src/vcpkg/commands.ci-verify-versions.cpp index 2fd5191cc4..fc4b4400f1 100644 --- a/src/vcpkg/commands.ci-verify-versions.cpp +++ b/src/vcpkg/commands.ci-verify-versions.cpp @@ -43,14 +43,13 @@ namespace if (!extracted_tree) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append(maybe_extracted_tree.error()) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgWhileValidatingVersion, msg::version = version_entry.version.version) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append(maybe_extracted_tree.error()) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgWhileValidatingVersion, msg::version = version_entry.version.version)); return success; } @@ -70,14 +69,13 @@ namespace // // However including both paths likely helps investigation and there isn't an easy way to replace only that // file path right now - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append(load_result.maybe_scfl.error()) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgWhileValidatingVersion, msg::version = version_entry.version.version) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append(load_result.maybe_scfl.error()) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgWhileValidatingVersion, msg::version = version_entry.version.version)); return success; } @@ -87,50 +85,47 @@ namespace if (version_entry_spec != scfl_spec) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionInDeclarationDoesNotMatch, - msg::git_tree_sha = version_entry.git_tree, - msg::expected = version_entry_spec, - msg::actual = scfl_spec) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionInDeclarationDoesNotMatch, + msg::git_tree_sha = version_entry.git_tree, + msg::expected = version_entry_spec, + msg::actual = scfl_spec)); } if (version_entry.version.scheme != git_tree_version.scheme) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionSchemeMismatch1Old, - msg::version = version_entry.version.version, - msg::expected = get_scheme_name(version_entry.version.scheme), - msg::actual = get_scheme_name(git_tree_version.scheme), - msg::package_name = port_name, - msg::git_tree_sha = version_entry.git_tree) - .append_raw('\n') - .append_raw(scfl->control_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortDeclaredHere, msg::package_name = port_name) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgVersionSchemeMismatch2) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionSchemeMismatch1Old, + msg::version = version_entry.version.version, + msg::expected = get_scheme_name(version_entry.version.scheme), + msg::actual = get_scheme_name(git_tree_version.scheme), + msg::package_name = port_name, + msg::git_tree_sha = version_entry.git_tree) + .append_raw('\n') + .append_raw(scfl->control_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortDeclaredHere, msg::package_name = port_name) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgVersionSchemeMismatch2)); } if (success) { - success_sink.print(LocalizedString::from_raw(versions_file_path) - .append_raw(": ") - .append_raw(MessagePrefix) - .append(msgVersionVerifiedOK, - msg::version_spec = VersionSpec{port_name, version_entry.version.version}, - msg::git_tree_sha = version_entry.git_tree) - .append_raw('\n')); + success_sink.println(LocalizedString::from_raw(versions_file_path) + .append_raw(": ") + .append_raw(MessagePrefix) + .append(msgVersionVerifiedOK, + msg::version_spec = VersionSpec{port_name, version_entry.version.version}, + msg::git_tree_sha = version_entry.git_tree)); } return success; @@ -156,21 +151,20 @@ namespace if (!entries) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(scfl.control_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionDatabaseFileMissing) - .append_raw('\n') - .append_raw(versions_database_entry.versions_file_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionDatabaseFileMissing2) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgVersionDatabaseFileMissing3, - msg::command_line = fmt::format("vcpkg x-add-version {}", port_name)) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(scfl.control_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionDatabaseFileMissing) + .append_raw('\n') + .append_raw(versions_database_entry.versions_file_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionDatabaseFileMissing2) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgVersionDatabaseFileMissing3, + msg::command_line = fmt::format("vcpkg x-add-version {}", port_name))); return success; } @@ -185,22 +179,21 @@ namespace if (it == versions_end) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(scfl.control_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionNotFoundInVersionsFile2, - msg::version_spec = VersionSpec{port_name, local_port_version.version}) - .append_raw('\n') - .append_raw(versions_database_entry.versions_file_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionNotFoundInVersionsFile3) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgVersionNotFoundInVersionsFile4, - msg::command_line = fmt::format("vcpkg x-add-version {}", port_name)) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(scfl.control_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionNotFoundInVersionsFile2, + msg::version_spec = VersionSpec{port_name, local_port_version.version}) + .append_raw('\n') + .append_raw(versions_database_entry.versions_file_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionNotFoundInVersionsFile3) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgVersionNotFoundInVersionsFile4, + msg::command_line = fmt::format("vcpkg x-add-version {}", port_name))); return success; } @@ -209,7 +202,7 @@ namespace { success = false; // assume the port is correct, so report the error on the version database file - errors_sink.print( + errors_sink.println( Color::error, LocalizedString::from_raw(versions_database_entry.versions_file_path) .append_raw(": ") @@ -230,58 +223,56 @@ namespace .append_raw('\n') .append_raw(NotePrefix) .append(msgVersionOverwriteVersion, msg::version_spec = local_version_spec) - .append_raw(fmt::format("\nvcpkg x-add-version {} --overwrite-version\n", port_name))); + .append_raw(fmt::format("\nvcpkg x-add-version {} --overwrite-version", port_name))); } if (local_git_tree != version_entry.git_tree) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(versions_database_entry.versions_file_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionShaMismatch1, - msg::version_spec = local_version_spec, - msg::git_tree_sha = version_entry.git_tree) - .append_raw('\n') - .append_raw(scfl.port_directory()) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionShaMismatch2, msg::git_tree_sha = local_git_tree) - .append_raw('\n') - .append_raw(scfl.control_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionShaMismatch3, msg::version_spec = local_version_spec) - .append_raw('\n') - .append_indent() - .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) - .append_indent() - .append_raw("git add versions\n") - .append_indent() - .append(msgGitCommitUpdateVersionDatabase) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgVersionShaMismatch4, msg::version_spec = local_version_spec) - .append_raw('\n') - .append_indent() - .append_raw(fmt::format("vcpkg x-add-version {} --overwrite-version\n", port_name)) - .append_indent() - .append_raw("git add versions\n") - .append_indent() - .append(msgGitCommitUpdateVersionDatabase) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(versions_database_entry.versions_file_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionShaMismatch1, + msg::version_spec = local_version_spec, + msg::git_tree_sha = version_entry.git_tree) + .append_raw('\n') + .append_raw(scfl.port_directory()) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionShaMismatch2, msg::git_tree_sha = local_git_tree) + .append_raw('\n') + .append_raw(scfl.control_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionShaMismatch3, msg::version_spec = local_version_spec) + .append_raw('\n') + .append_indent() + .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) + .append_indent() + .append_raw("git add versions\n") + .append_indent() + .append(msgGitCommitUpdateVersionDatabase) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgVersionShaMismatch4, msg::version_spec = local_version_spec) + .append_raw('\n') + .append_indent() + .append_raw(fmt::format("vcpkg x-add-version {} --overwrite-version\n", port_name)) + .append_indent() + .append_raw("git add versions\n") + .append_indent() + .append(msgGitCommitUpdateVersionDatabase)); } if (success) { - success_sink.print(LocalizedString::from_raw(scfl.port_directory()) - .append_raw(": ") - .append_raw(MessagePrefix) - .append(msgVersionVerifiedOK, - msg::version_spec = local_version_spec, - msg::git_tree_sha = version_entry.git_tree) - .append_raw('\n')); + success_sink.println(LocalizedString::from_raw(scfl.port_directory()) + .append_raw(": ") + .append_raw(MessagePrefix) + .append(msgVersionVerifiedOK, + msg::version_spec = local_version_spec, + msg::git_tree_sha = version_entry.git_tree)); } return success; @@ -298,67 +289,64 @@ namespace auto maybe_baseline = baseline.find(port_name); if (maybe_baseline == baseline.end()) { - errors_sink.print(Color::error, - LocalizedString::from_raw(baseline_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgBaselineMissing, msg::package_name = port_name) - .append_raw('\n') - .append_raw(scfl.control_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortDeclaredHere, msg::package_name = port_name) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgAddVersionInstructions, msg::package_name = port_name) - .append_raw('\n') - .append_indent() - .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) - .append_indent() - .append_raw("git add versions\n") - .append_indent() - .append(msgGitCommitUpdateVersionDatabase) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(baseline_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgBaselineMissing, msg::package_name = port_name) + .append_raw('\n') + .append_raw(scfl.control_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortDeclaredHere, msg::package_name = port_name) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgAddVersionInstructions, msg::package_name = port_name) + .append_raw('\n') + .append_indent() + .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) + .append_indent() + .append_raw("git add versions\n") + .append_indent() + .append(msgGitCommitUpdateVersionDatabase)); return false; } auto&& baseline_version = maybe_baseline->second; if (baseline_version == local_port_version.version) { - success_sink.print(LocalizedString::from_raw(baseline_path) - .append_raw(": ") - .append_raw(MessagePrefix) - .append(msgVersionBaselineMatch, - msg::version_spec = VersionSpec{port_name, local_port_version.version}) - .append_raw('\n')); + success_sink.println(LocalizedString::from_raw(baseline_path) + .append_raw(": ") + .append_raw(MessagePrefix) + .append(msgVersionBaselineMatch, + msg::version_spec = VersionSpec{port_name, local_port_version.version})); return true; } // assume the port is correct, so report the error on the baseline.json file - errors_sink.print(Color::error, - LocalizedString::from_raw(baseline_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionBaselineMismatch, - msg::expected = local_port_version.version, - msg::actual = baseline_version, - msg::package_name = port_name) - .append_raw('\n') - .append_raw(scfl.control_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortDeclaredHere, msg::package_name = port_name) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgAddVersionInstructions, msg::package_name = port_name) - .append_raw('\n') - .append_indent() - .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) - .append_indent() - .append_raw("git add versions\n") - .append_indent() - .append(msgGitCommitUpdateVersionDatabase) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(baseline_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionBaselineMismatch, + msg::expected = local_port_version.version, + msg::actual = baseline_version, + msg::package_name = port_name) + .append_raw('\n') + .append_raw(scfl.control_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortDeclaredHere, msg::package_name = port_name) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgAddVersionInstructions, msg::package_name = port_name) + .append_raw('\n') + .append_indent() + .append_raw(fmt::format("vcpkg x-add-version {}\n", port_name)) + .append_indent() + .append_raw("git add versions\n") + .append_indent() + .append(msgGitCommitUpdateVersionDatabase)); return false; } @@ -382,16 +370,15 @@ namespace auto this_error = LocalizedString::from_raw(scfl.control_path) .append_raw(": ") .append_raw(ErrorPrefix) - .append(msgDependencyNotInVersionDatabase, msg::package_name = dependency.name) - .append_raw('\n'); + .append(msgDependencyNotInVersionDatabase, msg::package_name = dependency.name); if (feature_name) { - this_error.append_raw(NotePrefix) - .append(msgDependencyInFeature, msg::feature = *feature_name) - .append_raw('\n'); + this_error.append_raw('\n') + .append_raw(NotePrefix) + .append(msgDependencyInFeature, msg::feature = *feature_name); } - errors_sink.print(Color::error, std::move(this_error)); + errors_sink.println(Color::error, std::move(this_error)); return false; } @@ -411,16 +398,15 @@ namespace .append_raw(dependent_versions_db_entry.versions_file_path) .append_raw(": ") .append_raw(NotePrefix) - .append(msgVersionConstraintNotInDatabase2) - .append_raw('\n'); + .append(msgVersionConstraintNotInDatabase2); if (feature_name) { - this_error.append_raw(NotePrefix) - .append(msgDependencyInFeature, msg::feature = *feature_name) - .append_raw('\n'); + this_error.append_raw('\n') + .append_raw(NotePrefix) + .append(msgDependencyInFeature, msg::feature = *feature_name); } - errors_sink.print(Color::error, std::move(this_error)); + errors_sink.println(Color::error, std::move(this_error)); return false; } @@ -463,13 +449,12 @@ namespace if (!override_entries) { success = false; - errors_sink.print( + errors_sink.println( Color::error, LocalizedString::from_raw(scfl.control_path) .append_raw(": ") .append_raw(ErrorPrefix) - .append(msgVersionOverrideNotInVersionDatabase, msg::package_name = override_.name) - .append_raw('\n')); + .append(msgVersionOverrideNotInVersionDatabase, msg::package_name = override_.name)); continue; } @@ -478,29 +463,27 @@ namespace })) { success = false; - errors_sink.print(Color::error, - LocalizedString::from_raw(scfl.control_path) - .append_raw(": ") - .append_raw(ErrorPrefix) - .append(msgVersionOverrideVersionNotInVersionDatabase1, - msg::package_name = override_.name, - msg::version = override_.version) - .append_raw('\n') - .append_raw(override_versions_db_entry.versions_file_path) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgVersionOverrideVersionNotInVersionDatabase2) - .append_raw('\n')); + errors_sink.println(Color::error, + LocalizedString::from_raw(scfl.control_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgVersionOverrideVersionNotInVersionDatabase1, + msg::package_name = override_.name, + msg::version = override_.version) + .append_raw('\n') + .append_raw(override_versions_db_entry.versions_file_path) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgVersionOverrideVersionNotInVersionDatabase2)); } } if (success) { - success_sink.print(LocalizedString::from_raw(scfl.control_path) - .append_raw(": ") - .append_raw(MessagePrefix) - .append(msgVersionConstraintOk) - .append_raw('\n')); + success_sink.println(LocalizedString::from_raw(scfl.control_path) + .append_raw(": ") + .append_raw(MessagePrefix) + .append(msgVersionConstraintOk)); } return success; @@ -568,7 +551,7 @@ namespace vcpkg auto git_tree_it = port_git_tree_map.find(port_name); if (git_tree_it == port_git_tree_map.end()) { - errors_sink.print( + errors_sink.println( Color::error, LocalizedString::from_raw(scfl.control_path) .append_raw(": ") @@ -587,7 +570,7 @@ namespace vcpkg .append_indent() .append_raw("git add versions\n") .append_indent() - .append_raw(fmt::format("git commit --amend -m \"{}\"\n", + .append_raw(fmt::format("git commit --amend -m \"{}\"", msg::format(msgVersionShaMissing4, msg::package_name = port_name)))); } else diff --git a/src/vcpkg/commands.download.cpp b/src/vcpkg/commands.download.cpp index 101f14880b..8e2ee8f44c 100644 --- a/src/vcpkg/commands.download.cpp +++ b/src/vcpkg/commands.download.cpp @@ -90,8 +90,8 @@ namespace vcpkg void command_download_and_exit(const VcpkgCmdArguments& args, const Filesystem& fs) { auto parsed = args.parse_arguments(CommandDownloadMetadata); - DownloadManager download_manager{ - parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO)}; + auto asset_cache_settings = + parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO); auto file = fs.absolute(parsed.command_arguments[0], VCPKG_LINE_INFO); auto sha = get_sha512_check(parsed); @@ -117,7 +117,12 @@ namespace vcpkg msg::println_error(msgMismatchedFiles); Checks::unreachable(VCPKG_LINE_INFO); } - download_manager.put_file_to_mirror(fs, file, actual_hash).value_or_exit(VCPKG_LINE_INFO); + + if (!store_to_asset_cache(console_diagnostic_context, asset_cache_settings, file, actual_hash)) + { + Checks::exit_fail(VCPKG_LINE_INFO); + } + Checks::exit_success(VCPKG_LINE_INFO); } else @@ -137,14 +142,20 @@ namespace vcpkg urls = it_urls->second; } - download_manager.download_file( - fs, - urls, - headers, - file, - sha, - Util::Sets::contains(parsed.switches, SwitchZMachineReadableProgress) ? out_sink : null_sink); - Checks::exit_success(VCPKG_LINE_INFO); + if (download_file_asset_cached( + console_diagnostic_context, + Util::Sets::contains(parsed.switches, SwitchZMachineReadableProgress) ? out_sink : null_sink, + asset_cache_settings, + fs, + urls, + headers, + file, + sha)) + { + Checks::exit_success(VCPKG_LINE_INFO); + } + + Checks::exit_fail(VCPKG_LINE_INFO); } } } // namespace vcpkg diff --git a/src/vcpkg/commands.format-manifest.cpp b/src/vcpkg/commands.format-manifest.cpp index d1199f167f..3bfae50bc4 100644 --- a/src/vcpkg/commands.format-manifest.cpp +++ b/src/vcpkg/commands.format-manifest.cpp @@ -148,7 +148,7 @@ namespace vcpkg else { auto maybe_manifest = - Paragraphs::try_load_project_manifest_text(contents->content, contents->origin, stdout_sink); + Paragraphs::try_load_project_manifest_text(contents->content, contents->origin, out_sink); if (auto manifest = maybe_manifest.get()) { to_write.push_back(ToWrite{contents->content, std::move(*manifest), path, path}); diff --git a/src/vcpkg/commands.help.cpp b/src/vcpkg/commands.help.cpp index 6c29d18ef1..b2206064cf 100644 --- a/src/vcpkg/commands.help.cpp +++ b/src/vcpkg/commands.help.cpp @@ -78,7 +78,7 @@ namespace {"assetcaching", [](const VcpkgPaths&) { msg::println(format_help_topic_asset_caching()); }}, {"binarycaching", [](const VcpkgPaths&) { msg::println(format_help_topic_binary_caching()); }}, {"commands", [](const VcpkgPaths&) { print_full_command_list(); }}, - {"topics", [](const VcpkgPaths&) { msg::print(help_topics()); }}, + {"topics", [](const VcpkgPaths&) { msg::println(help_topics()); }}, {"triplet", [](const VcpkgPaths& paths) { help_topic_valid_triplet(paths.get_triplet_db()); }}, {"versioning", help_topic_versioning}, }; @@ -101,7 +101,6 @@ namespace LocalizedString result; result.append(msgAvailableHelpTopics); result.append_floating_list(1, all_topic_names); - result.append_raw('\n'); return result; } @@ -193,8 +192,8 @@ namespace vcpkg } } - stderr_sink.println_error(msgUnknownTopic, msg::value = topic); - stderr_sink.print(help_topics()); + stderr_sink.println(msg::format_error(msgUnknownTopic, msg::value = topic)); + stderr_sink.println(help_topics()); get_global_metrics_collector().track_string(StringMetric::CommandContext, "unknown"); Checks::exit_fail(VCPKG_LINE_INFO); } diff --git a/src/vcpkg/commands.remove.cpp b/src/vcpkg/commands.remove.cpp index 1b6d43965e..ea54319828 100644 --- a/src/vcpkg/commands.remove.cpp +++ b/src/vcpkg/commands.remove.cpp @@ -68,7 +68,11 @@ namespace vcpkg } else { - msg::println_warning(msgFileNotFound, msg::path = target); + msg::println(Color::warning, + LocalizedString::from_raw(target) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgFileNotFound)); } } diff --git a/src/vcpkg/commands.set-installed.cpp b/src/vcpkg/commands.set-installed.cpp index 9fbbe44ce5..ed101d6b81 100644 --- a/src/vcpkg/commands.set-installed.cpp +++ b/src/vcpkg/commands.set-installed.cpp @@ -201,8 +201,9 @@ namespace vcpkg bool dependency_graph_success = false; if (snapshot && github_token && github_repository) { + WarningDiagnosticContext wdc{console_diagnostic_context}; dependency_graph_success = submit_github_dependency_graph_snapshot( - args.github_server_url, *github_token, *github_repository, *snapshot); + wdc, args.github_server_url, *github_token, *github_repository, *snapshot); if (dependency_graph_success) { msg::println(msgDependencyGraphSuccess); diff --git a/src/vcpkg/configure-environment.cpp b/src/vcpkg/configure-environment.cpp index f36a3d2e6f..97df6be9fc 100644 --- a/src/vcpkg/configure-environment.cpp +++ b/src/vcpkg/configure-environment.cpp @@ -105,25 +105,46 @@ namespace namespace vcpkg { - ExpectedL download_vcpkg_standalone_bundle(const DownloadManager& download_manager, - const Filesystem& fs, - const Path& download_root) + Optional download_vcpkg_standalone_bundle(DiagnosticContext& context, + const AssetCachingSettings& asset_cache_settings, + const Filesystem& fs, + const Path& download_root) { #if defined(VCPKG_STANDALONE_BUNDLE_SHA) const auto bundle_tarball = download_root / "vcpkg-standalone-bundle-" VCPKG_BASE_VERSION_AS_STRING ".tar.gz"; - msg::println(msgDownloadingVcpkgStandaloneBundle, msg::version = VCPKG_BASE_VERSION_AS_STRING); + context.statusln(msg::format(msgDownloadingVcpkgStandaloneBundle, msg::version = VCPKG_BASE_VERSION_AS_STRING)); const auto bundle_uri = "https://github.com/microsoft/vcpkg-tool/releases/download/" VCPKG_BASE_VERSION_AS_STRING "/vcpkg-standalone-bundle.tar.gz"; - download_manager.download_file( - fs, bundle_uri, {}, bundle_tarball, MACRO_TO_STRING(VCPKG_STANDALONE_BUNDLE_SHA), null_sink); + if (!download_file_asset_cached(context, + null_sink, + asset_cache_settings, + fs, + bundle_uri, + {}, + bundle_tarball, + MACRO_TO_STRING(VCPKG_STANDALONE_BUNDLE_SHA))) + { + return nullopt; + } #else // ^^^ VCPKG_STANDALONE_BUNDLE_SHA / !VCPKG_STANDALONE_BUNDLE_SHA vvv const auto bundle_tarball = download_root / "vcpkg-standalone-bundle-latest.tar.gz"; - msg::println(Color::warning, msgDownloadingVcpkgStandaloneBundleLatest); - fs.remove(bundle_tarball, VCPKG_LINE_INFO); + context.report(DiagnosticLine{DiagKind::Warning, msg::format(msgDownloadingVcpkgStandaloneBundleLatest)}); + std::error_code ec; + fs.remove(bundle_tarball, ec); + if (ec) + { + context.report_error(format_filesystem_call_error(ec, "remove", {bundle_tarball})); + return nullopt; + } + const auto bundle_uri = "https://github.com/microsoft/vcpkg-tool/releases/latest/download/vcpkg-standalone-bundle.tar.gz"; - download_manager.download_file(fs, bundle_uri, {}, bundle_tarball, nullopt, null_sink); + if (!download_file_asset_cached( + context, null_sink, asset_cache_settings, fs, bundle_uri, {}, bundle_tarball, nullopt)) + { + return nullopt; + } #endif // ^^^ !VCPKG_STANDALONE_BUNDLE_SHA return bundle_tarball; } @@ -159,11 +180,17 @@ namespace vcpkg fs.remove_all(vcpkg_artifacts_path, VCPKG_LINE_INFO); auto temp = get_exe_path_of_current_process(); temp.replace_filename("vcpkg-artifacts-temp"); - auto tarball = download_vcpkg_standalone_bundle(paths.get_download_manager(), fs, paths.downloads) - .value_or_exit(VCPKG_LINE_INFO); - set_directory_to_archive_contents(fs, paths.get_tool_cache(), null_sink, tarball, temp); + auto maybe_tarball = download_vcpkg_standalone_bundle( + console_diagnostic_context, paths.get_asset_cache_settings(), fs, paths.downloads); + auto tarball = maybe_tarball.get(); + if (!tarball) + { + Checks::exit_fail(VCPKG_LINE_INFO); + } + + set_directory_to_archive_contents(fs, paths.get_tool_cache(), null_sink, *tarball, temp); fs.rename_with_retry(temp / "vcpkg-artifacts", vcpkg_artifacts_path, VCPKG_LINE_INFO); - fs.remove(tarball, VCPKG_LINE_INFO); + fs.remove(*tarball, VCPKG_LINE_INFO); fs.remove_all(temp, VCPKG_LINE_INFO); #if defined(VCPKG_STANDALONE_BUNDLE_SHA) fs.write_contents(vcpkg_artifacts_version_path, VCPKG_BASE_VERSION_AS_STRING, VCPKG_LINE_INFO); diff --git a/src/vcpkg/paragraphs.cpp b/src/vcpkg/paragraphs.cpp index 8ea3853dc4..bcd868544b 100644 --- a/src/vcpkg/paragraphs.cpp +++ b/src/vcpkg/paragraphs.cpp @@ -409,7 +409,7 @@ namespace vcpkg::Paragraphs std::string{}}; } - return PortLoadResult{try_load_port_manifest_text(manifest_contents, manifest_path, stdout_sink) + return PortLoadResult{try_load_port_manifest_text(manifest_contents, manifest_path, out_sink) .map([&](std::unique_ptr&& scf) { return SourceControlFileAndLocation{ std::move(scf), std::move(manifest_path), port_location.spdx_location}; diff --git a/src/vcpkg/postbuildlint.cpp b/src/vcpkg/postbuildlint.cpp index a11e6131e7..a7b2405729 100644 --- a/src/vcpkg/postbuildlint.cpp +++ b/src/vcpkg/postbuildlint.cpp @@ -42,21 +42,16 @@ namespace vcpkg const Path& relative_dir, const std::vector& relative_paths) { - auto ls = LocalizedString() - .append_raw('\n') - .append_raw(relative_dir) - .append_raw(": ") - .append_raw(NotePrefix) - .append(kind_prefix) - .append_raw('\n'); + auto ls = + LocalizedString().append_raw(relative_dir).append_raw(": ").append_raw(NotePrefix).append(kind_prefix); for (const Path& package_relative_path : relative_paths) { auto as_generic = package_relative_path; as_generic.make_generic(); - ls.append_raw(NotePrefix).append_raw(as_generic).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append_raw(as_generic); } - msg_sink.print(ls); + msg_sink.println(ls); } // clang-format off @@ -113,12 +108,11 @@ namespace vcpkg const auto include_dir = package_dir / "include"; if (!fs.exists(include_dir, IgnoreErrors{}) || fs.is_empty(include_dir, IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingIncludeDir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingIncludeDir)); return LintStatus::PROBLEM_DETECTED; } @@ -134,12 +128,11 @@ namespace vcpkg const auto include_dir = package_dir / "include"; if (fs.exists(include_dir, IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugIncludeDirInCMakeHelperPort) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugIncludeDirInCMakeHelperPort)); return LintStatus::PROBLEM_DETECTED; } @@ -228,20 +221,18 @@ namespace vcpkg if (!violations.empty()) { Util::sort(violations); - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugRestrictedHeaderPaths) - .append_raw('\n')); - msg_sink.print(LocalizedString::from_raw(include_dir) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortBugRestrictedHeaderPathsNote) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugRestrictedHeaderPaths)); + msg_sink.println(LocalizedString::from_raw(include_dir) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortBugRestrictedHeaderPathsNote)); for (auto&& violation : violations) { - msg_sink.print(LocalizedString::from_raw(NotePrefix).append_raw(violation).append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix).append_raw(violation)); } return LintStatus::PROBLEM_DETECTED; @@ -263,14 +254,12 @@ namespace vcpkg if (!files_found.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugDuplicateIncludeFiles) - .append_raw('\n')); - msg_sink.print( - LocalizedString::from_raw(NotePrefix).append(msgPortBugDuplicateIncludeFilesFixIt).append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugDuplicateIncludeFiles)); + msg_sink.println(LocalizedString::from_raw(NotePrefix).append(msgPortBugDuplicateIncludeFilesFixIt)); return LintStatus::PROBLEM_DETECTED; } @@ -285,12 +274,11 @@ namespace vcpkg const auto debug_share = package_dir / FileDebug / FileShare; if (fs.exists(debug_share, IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugDebugShareDir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugDebugShareDir)); return LintStatus::PROBLEM_DETECTED; } @@ -305,12 +293,11 @@ namespace vcpkg { if (!fs.exists(package_dir / FileShare / package_name / FileVcpkgPortConfig, IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingCMakeHelperPortFile) - .append_raw("\n")); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingCMakeHelperPortFile)); return LintStatus::PROBLEM_DETECTED; } @@ -332,23 +319,21 @@ namespace vcpkg if (fs.is_regular_file(usage_path_from) && !fs.is_regular_file(usage_path_to)) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingProvidedUsage) - .append_raw('\n')); - msg_sink.print(LocalizedString::from_raw(usage_path_from) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgUsageTextHere) - .append_raw('\n') - .append_raw(NotePrefix) - .append(msgUsageInstallInstructions) - .append_raw('\n') - .append_raw(NotePrefix) - .append_raw(STANDARD_INSTALL_USAGE) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingProvidedUsage)); + msg_sink.println(LocalizedString::from_raw(usage_path_from) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgUsageTextHere) + .append_raw('\n') + .append_raw(NotePrefix) + .append(msgUsageInstallInstructions) + .append_raw('\n') + .append_raw(NotePrefix) + .append_raw(STANDARD_INSTALL_USAGE)); return LintStatus::PROBLEM_DETECTED; } @@ -382,11 +367,11 @@ namespace vcpkg if (!misplaced_cmake_files.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMisplacedCMakeFiles)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMisplacedCMakeFiles)); print_relative_paths( msg_sink, msgFilesRelativeToThePackageDirectoryHere, package_dir, misplaced_cmake_files); return LintStatus::PROBLEM_DETECTED; @@ -404,12 +389,11 @@ namespace vcpkg fs.exists(package_dir / "debug" VCPKG_PREFERRED_SEPARATOR "lib" VCPKG_PREFERRED_SEPARATOR "cmake", IgnoreErrors{})) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMergeLibCMakeDir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMergeLibCMakeDir)); return LintStatus::PROBLEM_DETECTED; } @@ -448,11 +432,11 @@ namespace vcpkg if (!bad_dlls.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugDllInLibDir)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugDllInLibDir)); print_relative_paths(msg_sink, msgDllsRelativeToThePackageDirectoryHere, package_dir, bad_dlls); return LintStatus::PROBLEM_DETECTED; } @@ -474,22 +458,20 @@ namespace vcpkg { case FileType::regular: return LintStatus::SUCCESS; break; case FileType::directory: - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgCopyrightIsDir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgCopyrightIsDir)); return LintStatus::PROBLEM_DETECTED; default: break; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingLicense) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingLicense)); // We only search in the root of each unpacked source archive to reduce false positives auto src_relative_dirs = Util::fmap(fs.get_directories_non_recursive(build_dir / "src", IgnoreErrors{}), @@ -512,14 +494,13 @@ namespace vcpkg auto args = Util::fmap(src_dir_relative_copyright_files, [](StringLiteral copyright_file) { return fmt::format(FMT_COMPILE("\"${{SOURCE_PATH}}/{}\""), copyright_file); }); - msg_sink.print( + msg_sink.println( LocalizedString::from_raw(portfile_cmake) .append_raw(": ") .append_raw(NotePrefix) .append(msgPortBugMissingLicenseFixIt, msg::value = fmt::format(FMT_COMPILE("vcpkg_install_copyright(FILE_LIST {})"), - fmt::join(args, " "))) - .append_raw('\n')); + fmt::join(args, " ")))); } return LintStatus::PROBLEM_DETECTED; @@ -542,10 +523,10 @@ namespace vcpkg if (!build_dir_relative_copyright_files.empty()) { - msg_sink.print(LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgPortBugFoundCopyrightFiles)); + msg_sink.println(LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgPortBugFoundCopyrightFiles)); print_relative_paths( msg_sink, msgFilesRelativeToTheBuildDirectoryHere, build_dir, build_dir_relative_copyright_files); } @@ -570,11 +551,11 @@ namespace vcpkg if (!exes.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugFoundExeInBinDir)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugFoundExeInBinDir)); print_relative_paths(msg_sink, msgExecutablesRelativeToThePackageDirectoryHere, package_dir, exes); return LintStatus::PROBLEM_DETECTED; @@ -662,11 +643,11 @@ namespace vcpkg if (!dlls_with_no_exports.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugSetDllsWithoutExports)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugSetDllsWithoutExports)); print_relative_paths(msg_sink, msgDllsRelativeToThePackageDirectoryHere, package_dir, dlls_with_no_exports); return LintStatus::PROBLEM_DETECTED; } @@ -696,11 +677,11 @@ namespace vcpkg if (!dlls_with_improper_uwp_bit.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugDllAppContainerBitNotSet)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugDllAppContainerBitNotSet)); print_relative_paths( msg_sink, msgDllsRelativeToThePackageDirectoryHere, package_dir, dlls_with_improper_uwp_bit); return LintStatus::PROBLEM_DETECTED; @@ -759,28 +740,26 @@ namespace vcpkg std::vector binaries_with_invalid_architecture, MessageSink& msg_sink) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgBuiltWithIncorrectArchitecture, msg::arch = expected_architecture) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgBuiltWithIncorrectArchitecture, msg::arch = expected_architecture)); auto msg = LocalizedString::from_raw(package_dir) .append_raw(": ") .append_raw(NotePrefix) - .append(msgBinariesRelativeToThePackageDirectoryHere) - .append_raw('\n'); + .append(msgBinariesRelativeToThePackageDirectoryHere); for (const FileAndArch& b : binaries_with_invalid_architecture) { - msg.append_raw(NotePrefix) + msg.append_raw('\n') + .append_raw(NotePrefix) .append(msgBinaryWithInvalidArchitecture, msg::path = b.relative_file.generic_u8string(), - msg::arch = b.actual_arch) - .append_raw('\n'); + msg::arch = b.actual_arch); } - msg_sink.print(msg); + msg_sink.println(msg); } static void check_dll_architecture(const std::string& expected_architecture, @@ -879,11 +858,11 @@ namespace vcpkg return LintStatus::SUCCESS; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugFoundDllInStaticBuild)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugFoundDllInStaticBuild)); print_relative_paths(msg_sink, msgDllsRelativeToThePackageDirectoryHere, package_dir, relative_dlls); return LintStatus::PROBLEM_DETECTED; } @@ -906,7 +885,7 @@ namespace vcpkg { auto as_generic = binary; as_generic.make_generic(); - ls.append_raw(NotePrefix).append_raw(as_generic).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append_raw(as_generic); } } } @@ -924,38 +903,36 @@ namespace vcpkg return LintStatus::SUCCESS; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMismatchingNumberOfBinaries) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMismatchingNumberOfBinaries)); LocalizedString ls = LocalizedString::from_raw(package_dir) .append_raw(": ") .append_raw(NotePrefix) - .append(msgBinariesRelativeToThePackageDirectoryHere) - .append_raw('\n'); + .append(msgBinariesRelativeToThePackageDirectoryHere); if (debug_count == 0) { - ls.append_raw(NotePrefix).append(msgPortBugMissingDebugBinaries).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append(msgPortBugMissingDebugBinaries); } else { - ls.append_raw(NotePrefix).append(msgPortBugFoundDebugBinaries).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append(msgPortBugFoundDebugBinaries); append_binary_set(ls, relative_debug_binary_sets); } if (release_count == 0) { - ls.append_raw(NotePrefix).append(msgPortBugMissingReleaseBinaries).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append(msgPortBugMissingReleaseBinaries); } else { - ls.append_raw(NotePrefix).append(msgPortBugFoundReleaseBinaries).append_raw('\n'); + ls.append_raw('\n').append_raw(NotePrefix).append(msgPortBugFoundReleaseBinaries); append_binary_set(ls, relative_release_binary_sets); } - msg_sink.print(ls); + msg_sink.println(ls); return LintStatus::PROBLEM_DETECTED; } @@ -966,11 +943,10 @@ namespace vcpkg { if (lib_count == 0 && dll_count != 0) { - msg_sink.print(LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMissingImportedLibs) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMissingImportedLibs)); return LintStatus::PROBLEM_DETECTED; } @@ -998,24 +974,23 @@ namespace vcpkg for (auto&& bad_dir : bad_dirs) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugBinDirExists, msg::path = bad_dir) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugBinDirExists, msg::path = bad_dir)); } auto args = Util::fmap(bad_dirs, [](const Path& bad_dir) { return fmt::format(FMT_COMPILE("\"${{CURRENT_PACKAGES_DIR}}/{}\""), bad_dir); }); - msg_sink.print( + msg_sink.println( LocalizedString::from_raw(NotePrefix) .append(msgPortBugRemoveBinDir) .append_raw('\n') .append_raw("if(VCPKG_LIBRARY_LINKAGE STREQUAL \"static\")\n") .append_indent() - .append_raw(fmt::format(FMT_COMPILE("file(REMOVE_RECURSE {})\nendif()\n"), fmt::join(args, " ")))); + .append_raw(fmt::format(FMT_COMPILE("file(REMOVE_RECURSE {})\nendif()"), fmt::join(args, " ")))); return bad_dirs.size(); } @@ -1031,24 +1006,23 @@ namespace vcpkg if (!relative_empty_directories.empty()) { Util::sort(relative_empty_directories); - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugFoundEmptyDirectories) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugFoundEmptyDirectories)); auto args = Util::fmap(relative_empty_directories, [](const Path& empty_dir) { return fmt::format(FMT_COMPILE("\"${{CURRENT_PACKAGES_DIR}}/{}\""), empty_dir.generic_u8string()); }); - msg_sink.print( + msg_sink.println( LocalizedString::from_raw(package_dir) .append_raw(": ") .append_raw(NotePrefix) .append(msgDirectoriesRelativeToThePackageDirectoryHere) .append_raw('\n') .append_raw(NotePrefix) - .append_raw(fmt::format(FMT_COMPILE("file(REMOVE_RECURSE {})\n"), fmt::join(args, " ")))); + .append_raw(fmt::format(FMT_COMPILE("file(REMOVE_RECURSE {})"), fmt::join(args, " ")))); return LintStatus::PROBLEM_DETECTED; } @@ -1126,18 +1100,18 @@ namespace vcpkg if (!misplaced_pkgconfig_files.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMisplacedPkgConfigFiles)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMisplacedPkgConfigFiles)); print_relative_paths(msg_sink, msgFilesRelativeToThePackageDirectoryHere, package_dir, Util::fmap(misplaced_pkgconfig_files, [](const MisplacedFile& mf) -> Path { return mf.relative_path; })); - msg_sink.print(LocalizedString::from_raw(NotePrefix).append(msgPortBugMovePkgConfigFiles).append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix).append(msgPortBugMovePkgConfigFiles)); { auto create_directory_line = LocalizedString::from_raw("file(MAKE_DIRECTORY"); std::vector directories; @@ -1162,8 +1136,8 @@ namespace vcpkg fmt::format(FMT_COMPILE(R"###( "${{CURRENT_PACKAGES_DIR}}/{}")###"), directory)); } - create_directory_line.append_raw(")\n"); - msg_sink.print(create_directory_line); + create_directory_line.append_raw(")"); + msg_sink.println(create_directory_line); } // destroy create_directory_line for (const auto& item : misplaced_pkgconfig_files) @@ -1177,15 +1151,15 @@ namespace vcpkg default: Checks::unreachable(VCPKG_LINE_INFO); } - msg_sink.print(LocalizedString::from_raw(fmt::format( - FMT_COMPILE(R"###(file(RENAME "${{CURRENT_PACKAGES_DIR}}/{}" "${{CURRENT_PACKAGES_DIR}}/{}/{}"))###" - "\n"), + msg_sink.println(LocalizedString::from_raw(fmt::format( + FMT_COMPILE( + R"###(file(RENAME "${{CURRENT_PACKAGES_DIR}}/{}" "${{CURRENT_PACKAGES_DIR}}/{}/{}"))###"), item.relative_path.generic_u8string(), *dir, item.relative_path.filename()))); } - msg_sink.print(LocalizedString::from_raw("vcpkg_fixup_pkgconfig()\n")); + msg_sink.println(LocalizedString::from_raw("vcpkg_fixup_pkgconfig()")); msg_sink.println(msgPortBugRemoveEmptyDirs); return LintStatus::PROBLEM_DETECTED; @@ -1327,32 +1301,28 @@ namespace vcpkg return LintStatus::SUCCESS; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugInvalidCrtLinkageHeader) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugInvalidCrtLinkageHeader)); - msg_sink.print(LocalizedString::from_raw(package_dir) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgBinariesRelativeToThePackageDirectoryHere) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(package_dir) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgBinariesRelativeToThePackageDirectoryHere)); for (auto&& group : groups_of_invalid_crt) { - msg_sink.print(LocalizedString::from_raw(NotePrefix) - .append(msgPortBugInvalidCrtLinkageCrtGroup, msg::expected = to_string(group.first)) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix) + .append(msgPortBugInvalidCrtLinkageCrtGroup, msg::expected = to_string(group.first))); for (auto&& file : group.second) { for (auto&& linkage : file.linkages) { - msg_sink.print(LocalizedString::from_raw(NotePrefix) - .append(msgPortBugInvalidCrtLinkageEntry, - msg::path = file.relative_file.generic_u8string(), - msg::actual = to_string(linkage)) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix) + .append(msgPortBugInvalidCrtLinkageEntry, + msg::path = file.relative_file.generic_u8string(), + msg::actual = to_string(linkage))); } } } @@ -1390,11 +1360,11 @@ namespace vcpkg if (!dlls_with_outdated_crt.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugOutdatedCRT)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugOutdatedCRT)); print_relative_paths( msg_sink, @@ -1437,11 +1407,11 @@ namespace vcpkg return LintStatus::SUCCESS; } - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugKernel32FromXbox)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugKernel32FromXbox)); print_relative_paths( msg_sink, msgDllsRelativeToThePackageDirectoryHere, @@ -1487,11 +1457,11 @@ namespace vcpkg if (!misplaced_files.empty()) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgPortBugMisplacedFiles)); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgPortBugMisplacedFiles)); print_relative_paths(msg_sink, msgFilesRelativeToThePackageDirectoryHere, package_dir, misplaced_files); return LintStatus::PROBLEM_DETECTED; } @@ -1586,34 +1556,31 @@ namespace vcpkg } Util::sort(failing_files); - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgFilesContainAbsolutePath1) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgFilesContainAbsolutePath1)); for (auto&& absolute_path : prohibited_absolute_paths) { - msg_sink.print(LocalizedString::from_raw(NotePrefix).append_raw(absolute_path).append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(NotePrefix).append_raw(absolute_path)); } if (any_pc_file_fails) { - msg_sink.print(LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgFilesContainAbsolutePathPkgconfigNote) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgFilesContainAbsolutePathPkgconfigNote)); } for (auto&& failing_file : failing_files) { failing_file.make_preferred(); - msg_sink.print(LocalizedString::from_raw(package_dir / failing_file) - .append_raw(": ") - .append_raw(NotePrefix) - .append(msgFilesContainAbsolutePath2) - .append_raw('\n')); + msg_sink.println(LocalizedString::from_raw(package_dir / failing_file) + .append_raw(": ") + .append_raw(NotePrefix) + .append(msgFilesContainAbsolutePath2)); } return LintStatus::PROBLEM_DETECTED; @@ -1937,12 +1904,11 @@ namespace vcpkg spec, paths, pre_build_info, build_info, port_dir, portfile_cmake, msg_sink); if (error_count != 0) { - msg_sink.print(Color::warning, - LocalizedString::from_raw(portfile_cmake) - .append_raw(": ") - .append_raw(WarningPrefix) - .append(msgFailedPostBuildChecks, msg::count = error_count) - .append_raw('\n')); + msg_sink.println(Color::warning, + LocalizedString::from_raw(portfile_cmake) + .append_raw(": ") + .append_raw(WarningPrefix) + .append(msgFailedPostBuildChecks, msg::count = error_count)); } return error_count; diff --git a/src/vcpkg/tools.cpp b/src/vcpkg/tools.cpp index 20c19af394..3ec3bd61f6 100644 --- a/src/vcpkg/tools.cpp +++ b/src/vcpkg/tools.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -741,7 +742,7 @@ namespace vcpkg struct ToolCacheImpl final : ToolCache { const Filesystem& fs; - const std::shared_ptr downloader; + AssetCachingSettings asset_cache_settings; const Path downloads; const Path config_path; const Path tools; @@ -751,13 +752,13 @@ namespace vcpkg vcpkg::Lazy> m_tool_data_cache; ToolCacheImpl(const Filesystem& fs, - const std::shared_ptr& downloader, + const AssetCachingSettings& asset_cache_settings, Path downloads, Path config_path, Path tools, RequireExactVersions abiToolVersionHandling) : fs(fs) - , downloader(downloader) + , asset_cache_settings(asset_cache_settings) , downloads(std::move(downloads)) , config_path(std::move(config_path)) , tools(std::move(tools)) @@ -799,6 +800,8 @@ namespace vcpkg Path download_tool(const ToolData& tool_data, MessageSink& status_sink) const { + using namespace Hash; + const std::array& version = tool_data.version; const std::string version_as_string = fmt::format("{}.{}.{}", version[0], version[1], version[2]); Checks::msg_check_maybe_upgrade(VCPKG_LINE_INFO, @@ -812,13 +815,38 @@ namespace vcpkg msg::version = version_as_string); const auto download_path = downloads / tool_data.download_subpath; - if (!fs.exists(download_path, IgnoreErrors{})) + const auto hash_result = get_file_hash(console_diagnostic_context, fs, download_path, Algorithm::Sha512); + switch (hash_result.prognosis) { - downloader->download_file(fs, tool_data.url, {}, download_path, tool_data.sha512, null_sink); - } - else - { - verify_downloaded_file_hash(fs, tool_data.url, download_path, tool_data.sha512); + case HashPrognosis::Success: + if (!Strings::case_insensitive_ascii_equals(tool_data.sha512, hash_result.hash)) + { + Checks::msg_exit_with_message(VCPKG_LINE_INFO, + LocalizedString::from_raw(download_path) + .append_raw(": ") + .append_raw(ErrorPrefix) + .append(msgToolHashMismatch, + msg::tool_name = tool_data.name, + msg::expected = tool_data.sha512, + msg::actual = hash_result.hash)); + } + + break; + case HashPrognosis::FileNotFound: + if (!download_file_asset_cached(console_diagnostic_context, + null_sink, + asset_cache_settings, + fs, + tool_data.url, + {}, + download_path, + tool_data.sha512)) + { + Checks::exit_fail(VCPKG_LINE_INFO); + } + break; + case HashPrognosis::OtherError: Checks::exit_fail(VCPKG_LINE_INFO); + default: Checks::unreachable(VCPKG_LINE_INFO); } const auto tool_dir_path = tools / tool_data.tool_dir_subpath; @@ -1095,14 +1123,14 @@ namespace vcpkg } std::unique_ptr get_tool_cache(const Filesystem& fs, - std::shared_ptr downloader, + const AssetCachingSettings& asset_cache_settings, Path downloads, Path config_path, Path tools, RequireExactVersions abiToolVersionHandling) { return std::make_unique( - fs, std::move(downloader), downloads, config_path, tools, abiToolVersionHandling); + fs, asset_cache_settings, downloads, config_path, tools, abiToolVersionHandling); } struct ToolDataEntryDeserializer final : Json::IDeserializer diff --git a/src/vcpkg/vcpkgpaths.cpp b/src/vcpkg/vcpkgpaths.cpp index a2b9af75a9..1c44715cff 100644 --- a/src/vcpkg/vcpkgpaths.cpp +++ b/src/vcpkg/vcpkgpaths.cpp @@ -341,8 +341,8 @@ namespace , m_ff_settings(args.feature_flag_settings()) , m_manifest_dir(compute_manifest_dir(fs, args, original_cwd)) , m_bundle(bundle) - , m_download_manager(std::make_shared( - parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO))) + , m_asset_cache_settings( + parse_download_configuration(args.asset_sources_template()).value_or_exit(VCPKG_LINE_INFO)) , m_builtin_ports(process_output_directory(fs, args.builtin_ports_root_dir.get(), root / "ports")) , m_default_vs_path(args.default_visual_studio_path .map([&fs](const std::string& default_visual_studio_path) { @@ -359,7 +359,7 @@ namespace const FeatureFlagSettings m_ff_settings; const Path m_manifest_dir; const BundleSettings m_bundle; - const std::shared_ptr m_download_manager; + const AssetCachingSettings m_asset_cache_settings; const Path m_builtin_ports; const Path m_default_vs_path; const Path scripts; @@ -589,7 +589,7 @@ namespace vcpkg VCPKG_LINE_INFO)) , m_tool_cache(get_tool_cache( fs, - m_download_manager, + m_asset_cache_settings, downloads, args.tools_data_file.has_value() ? Path{*args.tools_data_file.get()} : scripts / "vcpkg-tools.json", tools, @@ -687,11 +687,6 @@ namespace vcpkg Debug::print("Using vcpkg-root: ", root, '\n'); Debug::print("Using builtin-registry: ", builtin_registry_versions, '\n'); Debug::print("Using downloads-root: ", downloads, '\n'); - m_pimpl->m_download_manager->get_block_origin() - ? Debug::println("External asset downloads are blocked (x-block-origin is enabled)..") - : Debug::println("External asset downloads are allowed (x-block-origin is disabled)..."); - m_pimpl->m_download_manager->asset_cache_configured() ? Debug::println("Asset caching is enabled.") - : Debug::println("Asset cache is not configured."); const auto config_path = m_pimpl->m_config_dir / "vcpkg-configuration.json"; auto maybe_manifest_config = config_from_manifest(m_pimpl->m_manifest_doc); @@ -905,7 +900,7 @@ namespace vcpkg } const Filesystem& VcpkgPaths::get_filesystem() const { return m_pimpl->m_fs; } - const DownloadManager& VcpkgPaths::get_download_manager() const { return *m_pimpl->m_download_manager; } + const AssetCachingSettings& VcpkgPaths::get_asset_cache_settings() const { return m_pimpl->m_asset_cache_settings; } const ToolCache& VcpkgPaths::get_tool_cache() const { return *m_pimpl->m_tool_cache; } const Path& VcpkgPaths::get_tool_exe(StringView tool, MessageSink& status_messages) const {