Fix all download failures claiming that the download was disabled by x-block-origin.

azure-pipelines/end-to-end-tests-dir/asset-caching.ps1

@@ -57,6 +57,8 @@ $actual = $actual -replace "`r`n", "`n"
 $expected = @(
 "A suitable version of .* was not found \(required v[0-9\.]+\)."
 "Asset cache miss; downloading from .*"
+"Downloading .*"
+"Successfully downloaded .*."
 "Successfully stored .* to .*."
 ) -join "`n"
 
@@ -82,26 +84,113 @@ if (-not ($actual -match $expected)) {
 # Testing asset caching && x-block-orgin promises when --debug is passed (enabled)
 Refresh-TestRoot
 $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin", "--downloads-root=$DownloadsRoot", "--debug"))
-$actual = $actual -replace "`r`n", "`n"
-
-# Define the regex pattern that accounts for multiline input
-$expectedPattern = "(?s)" +
-                   ".*\[DEBUG\] External asset downloads are blocked \(x-block-origin is enabled\)\.\.\.?" +
-                   ".*\[DEBUG\] Asset caching is enabled\..*"
-
-if (-not ($actual -match $expectedPattern)) {
+if (-not ($actual.Contains("[DEBUG] External asset downloads are blocked (x-block-origin is enabled)") -and $actual.Contains("[DEBUG] Asset caching is enabled."))) {
     throw "Failure: couldn't find expected debug promises (asset caching enabled + x-block-origin enabled)"
 }
 
 # Testing asset caching && x-block-orgin promises when --debug is passed (disabled)
 Refresh-TestRoot
 $actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("install", "vcpkg-internal-e2e-test-port", "--overlay-ports=$PSScriptRoot/../e2e-ports", "--x-asset-sources=clear", "--downloads-root=$DownloadsRoot", "--debug"))
-$actual = $actual -replace "`r`n", "`n"
+if (-not ($actual.Contains("[DEBUG] External asset downloads are allowed (x-block-origin is disabled)") -and $actual.Contains("[DEBUG] Asset cache is not configured"))) {
+    throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)"
+}
 
-$expectedPattern = "(?s)" +
-                   ".*\[DEBUG\] External asset downloads are allowed \(x-block-origin is disabled\)\.\.\.?" +
-                   ".*\[DEBUG\] Asset cache is not configured.*"
+# azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)
+# Expected: Download failure message, nothing about asset caching
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html"))
+if (-not ($actual.Contains("error: https://localhost:1234/foobar.html: curl failed to download with exit code 7"))) {
+    throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (fail)"
+}
 
-if (-not ($actual -match $expectedPattern)) {
-    throw "Failure: couldn't find expected debug promises (asset caching disabled + x-block-origin disabled)"
+#azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)
+#Expected: Download message with "you might need to configure a proxy" and expected/actual sha
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com"))
+if (-not ($actual.Contains("error: File does not have the expected hash:") -and
+          $actual.Contains("url: https://example.com") -and
+          $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and
+          $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) {
+    throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (sha-mismatch)"
+}
+
+# azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)
+# Expected: Download success message, nothing about asset caching
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com"))
+if (-not ($actual.Contains("Downloading example3.html") -and
+          $actual.Contains("Successfully downloaded example3.html."))) {
+    throw "Failure: azurl (no), x-block-origin (no), asset-cache (n/a), download (succeed)"
+}
+
+# azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)
+# Expected: Download failure message, nothing about asset caching, x-block-origin complaint
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=clear;x-block-origin"))
+if (-not ($actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) {
+    throw "Failure: azurl (no), x-block-origin (yes), asset-cache (n/a), download (n/a)"
+}
+
+# azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)
+# Expected: Download failure message, asset cache named, nothing about x-block-origin
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://localhost:1234/foobar.html", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+if (-not ($actual.Contains("Asset cache miss; downloading from https://localhost:1234/foobar.html") -and
+          $actual.Contains("Downloading example3.html") -and
+          $actual.Contains("error: file://$AssetCache") -and
+          $actual.Contains("curl failed to download with exit code 37"))) {
+    throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (fail)"
+}
+
+# azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)
+# Expected: Download success message, asset cache named, nothing about x-block-origin
+Refresh-TestRoot
+Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) {
+    throw "Failure: azurl (yes), x-block-origin (no), asset-cache (hit), download (n/a)"
+}
+
+# azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)
+# Expected: Download message with "you might need to configure a proxy" and expected/actual sha
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and
+          $actual.Contains("Downloading example3.html") -and
+          $actual.Contains("error: file://$AssetCache") -and
+          $actual.Contains("curl failed to download with exit code 37") -and
+          $actual.Contains("error: File does not have the expected hash:") -and
+          $actual.Contains("url: https://example.com") -and
+          $actual.Contains("Expected hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73b") -and
+          $actual.Contains("Actual hash: d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a"))) {
+    throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (sha-mismatch)"
+}
+
+# azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)
+# Expected: Download success message, asset cache upload, nothing about x-block-origin
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+if (-not ($actual.Contains("Asset cache miss; downloading from https://example.com") -and
+          $actual.Contains("Downloading example3.html") -and
+          $actual.Contains("Successfully downloaded example3.html.") -and
+          $actual.Contains("Successfully stored example3.html to file://$AssetCache"))) {
+    throw "Failure: azurl (yes), x-block-origin (no), asset-cache (miss), download (succeed)"
+}
+
+# azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)
+# Expected: Download failure message, which asset cache was tried, x-block-origin complaint
+Refresh-TestRoot
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin"))
+if (-not ($actual.Contains("Asset cache miss for example3.html and downloads are blocked by x-block-origin.") -and
+          $actual.Contains("error: Missing example3.html and downloads are blocked by x-block-origin."))) {
+    throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (miss), download (n/a)"
+}
+
+# azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)
+# Expected: Download success message, asset cache named, nothing about x-block-origin
+Refresh-TestRoot
+Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite"))
+$actual = Run-VcpkgAndCaptureOutput -TestArgs ($commonArgs + @("x-download", "$downloadsRoot/example3.html", "--sha512", "d06b93c883f8126a04589937a884032df031b05518eed9d433efb6447834df2596aebd500d69b8283e5702d988ed49655ae654c1683c7a4ae58bfa6b92f2b73a", "--url", "https://example.com", "--x-asset-sources=x-azurl,file://$AssetCache,,readwrite;x-block-origin"))
+if (-not ($actual.Contains("Asset cache hit for example3.html; downloaded from: file://$AssetCache"))) {
+    throw "Failure: azurl (yes), x-block-origin (yes), asset-cache (hit), download (n/a)"
 }

include/vcpkg/base/message-data.inc.h

@@ -248,6 +248,11 @@ DECLARE_MESSAGE(ArtifactsSwitchX86, (), "", "Forces host detection to x86 when a
 DECLARE_MESSAGE(ArtifactsSwitchWindows, (), "", "Forces host detection to Windows when acquiring artifacts")
 DECLARE_MESSAGE(AssetCacheHit, (msg::path, msg::url), "", "Asset cache hit for {path}; downloaded from: {url}")
 DECLARE_MESSAGE(AssetCacheMiss, (msg::url), "", "Asset cache miss; downloading from {url}")
+DECLARE_MESSAGE(AssetCacheMissBlockOrigin,
+                (msg::path),
+                "x-block-origin is a vcpkg term. Do not translate",
+                "Asset cache miss for {path} and downloads are blocked by x-block-origin.")
+DECLARE_MESSAGE(DownloadSuccesful, (msg::path), "", "Successfully downloaded {path}.")
 DECLARE_MESSAGE(DownloadingUrl, (msg::url), "", "Downloading {url}")
 DECLARE_MESSAGE(AssetCacheProviderAcceptsNoArguments,
                 (msg::value),
@@ -1228,10 +1233,6 @@ DECLARE_MESSAGE(MissingShaVariable,
                 (),
                 "{{sha}} should not be translated",
                 "The {{sha}} variable must be used in the template if other variables are used.")
-DECLARE_MESSAGE(AssetCacheMissBlockOrigin,
-                (msg::path),
-                "x-block-origin is a vcpkg term. Do not translate",
-                "Asset cache miss for {path} and downloads are blocked by x-block-origin.")
 DECLARE_MESSAGE(FailedToExtract, (msg::path), "", "Failed to extract \"{path}\":")
 DECLARE_MESSAGE(FailedToFetchRepo, (msg::url), "", "Failed to fetch {url}.")
 DECLARE_MESSAGE(FailedToFindPortFeature,

locales/messages.json

@@ -601,6 +601,8 @@
   "_DownloadFailedStatusCode.comment": "{value} is an HTTP status code An example of {url} is https://github.com/microsoft/vcpkg.",
   "DownloadRootsDir": "Downloads directory (default: {env_var})",
   "_DownloadRootsDir.comment": "An example of {env_var} is VCPKG_DEFAULT_TRIPLET.",
+  "DownloadSuccesful": "Successfully downloaded {path}.",
+  "_DownloadSuccesful.comment": "An example of {path} is /foo/bar.",
   "DownloadWinHttpError": "{url}: {system_api} failed with exit code {exit_code}",
   "_DownloadWinHttpError.comment": "An example of {system_api} is CreateProcessW. An example of {exit_code} is 127. An example of {url} is https://github.com/microsoft/vcpkg.",
   "DownloadedSources": "Downloaded sources for {spec}",

src/vcpkg/base/downloads.cpp

@@ -905,6 +905,8 @@ namespace vcpkg
                                                MessageSink& progress_sink) const
     {
         std::vector<LocalizedString> errors;
+        bool block_origin_enabled = m_config.m_block_origin;
+
         if (urls.size() == 0)
         {
             if (auto hash = sha512.get())
@@ -936,6 +938,14 @@ namespace vcpkg
                                  msg::url = replace_secrets(read_url, m_config.m_secrets));
                     return read_url;
                 }
+                else if (block_origin_enabled)
+                {
+                    msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename());
+                }
+                else
+                {
+                    msg::println(msgAssetCacheMiss, msg::url = urls[0]);
+                }
             }
             else if (auto script = m_config.m_script.get())
             {
@@ -985,16 +995,16 @@ namespace vcpkg
             }
         }
 
-        if (!m_config.m_block_origin)
+        if (!block_origin_enabled)
         {
             if (urls.size() != 0)
             {
+                msg::println(msgDownloadingUrl, msg::url = download_path.filename());
                 auto maybe_url = try_download_file(
                     fs, urls, headers, download_path, sha512, m_config.m_secrets, errors, progress_sink);
                 if (auto url = maybe_url.get())
                 {
-                    m_config.m_read_url_template.has_value() ? msg::println(msgAssetCacheMiss, msg::url = urls[0])
-                                                             : msg::println(msgDownloadingUrl, msg::url = urls[0]);
+                    msg::println(msgDownloadSuccesful, msg::path = download_path.filename());
 
                     if (auto hash = sha512.get())
                     {
@@ -1012,11 +1022,6 @@ namespace vcpkg
                 }
             }
         }
-        // Asset cache is not configured and x-block-origin enabled
-        if (m_config.m_read_url_template.has_value())
-        {
-            msg::println(msgAssetCacheMissBlockOrigin, msg::path = download_path.filename());
-        }
         else
         {
             msg::println_error(msgMissingAssetBlockOrigin, msg::path = download_path.filename());