Adding support for AWS S3 binary caching

include/vcpkg/base/messages.h

@@ -155,8 +155,10 @@ namespace vcpkg::msg
     DECLARE_MSG_ARG(triplet);
     DECLARE_MSG_ARG(url);
     DECLARE_MSG_ARG(value);
+    DECLARE_MSG_ARG(elapsed);
     DECLARE_MSG_ARG(version);
     DECLARE_MSG_ARG(list);
+    DECLARE_MSG_ARG(output);
 #undef DECLARE_MSG_ARG
 
 // These are `...` instead of

include/vcpkg/tools.h

@@ -18,6 +18,7 @@ namespace vcpkg
         static const std::string CMAKE = "cmake";
         static const std::string GIT = "git";
         static const std::string GSUTIL = "gsutil";
+        static const std::string AWSCLI = "aws";
         static const std::string MONO = "mono";
         static const std::string NINJA = "ninja";
         static const std::string POWERSHELL_CORE = "powershell-core";

locales/en.json

@@ -1,6 +1,10 @@
 {
   "AddPortSucceded": "Succeeded in adding ports to vcpkg.json file.",
   "AddTripletExpressionNotAllowed": "Error: triplet expressions are not allowed here. You may want to change `{name}:{triplet}` to `{name}` instead.",
+  "AwsAttemptingToFetchPackages": "Attempting to fetch {value} packages from AWS",
+  "AwsFailedToDownload": "aws failed to download with exit code: {value}\n{output}",
+  "AwsRestoredPackages": "Restored {value} packages from AWS servers in {elapsed}s",
+  "AwsUploadedPackages": "Uploaded binaries to {value} AWS servers",
   "ErrorIndividualPackagesUnsupported": "Error: In manifest mode, `vcpkg install` does not support individual package arguments.\nTo install additional packages, edit vcpkg.json and then run `vcpkg install` without any package arguments.",
   "ErrorInvalidClassicModeOption": "Error: The option {value} is not supported in classic mode and no manifest was found.",
   "ErrorInvalidManifestModeOption": "Error: The option {value} is not supported in manifest mode.",

src/vcpkg/binarycaching.cpp

@@ -1,6 +1,7 @@
 #include <vcpkg/base/checks.h>
 #include <vcpkg/base/downloads.h>
 #include <vcpkg/base/files.h>
+#include <vcpkg/base/messages.h>
 #include <vcpkg/base/parse.h>
 #include <vcpkg/base/strings.h>
 #include <vcpkg/base/system.debug.h>
@@ -23,6 +24,20 @@ using namespace vcpkg;
 
 namespace
 {
+    DECLARE_AND_REGISTER_MESSAGE(AwsFailedToDownload,
+                                 (msg::value, msg::output),
+                                 "",
+                                 "aws failed to download with exit code: {value}\n{output}");
+    DECLARE_AND_REGISTER_MESSAGE(AwsAttemptingToFetchPackages,
+                                 (msg::value),
+                                 "",
+                                 "Attempting to fetch {value} packages from AWS");
+    DECLARE_AND_REGISTER_MESSAGE(AwsRestoredPackages,
+                                 (msg::value, msg::elapsed),
+                                 "",
+                                 "Restored {value} packages from AWS servers in {elapsed}s");
+    DECLARE_AND_REGISTER_MESSAGE(AwsUploadedPackages, (msg::value), "", "Uploaded binaries to {value} AWS servers");
+
     struct ConfigSegmentsParser : Parse::ParserBase
     {
         using Parse::ParserBase::ParserBase;
@@ -1121,6 +1136,187 @@ namespace
         std::vector<std::string> m_read_prefixes;
         std::vector<std::string> m_write_prefixes;
     };
+
+    bool awscli_stat(const VcpkgPaths& paths, const std::string& url)
+    {
+        const auto cmd = Command{paths.get_tool_exe(Tools::AWSCLI)}.string_arg("s3").string_arg("ls").string_arg(url);
+        return cmd_execute(cmd) == 0;
+    }
+
+    bool awscli_upload_file(const VcpkgPaths& paths, const std::string& aws_object, const Path& archive)
+    {
+        const auto cmd =
+            Command{paths.get_tool_exe(Tools::AWSCLI)}.string_arg("s3").string_arg("cp").path_arg(archive).string_arg(
+                aws_object);
+        const auto out = cmd_execute_and_capture_output(cmd);
+        if (out.exit_code == 0)
+        {
+            return true;
+        }
+
+        msg::println(Color::warning, msgAwsFailedToDownload, msg::value = out.exit_code, msg::output = out.output);
+        return false;
+    }
+
+    bool awscli_download_file(const VcpkgPaths& paths, const std::string& aws_object, const Path& archive)
+    {
+        const auto cmd = Command{paths.get_tool_exe(Tools::AWSCLI)}
+                             .string_arg("s3")
+                             .string_arg("cp")
+                             .string_arg(aws_object)
+                             .path_arg(archive);
+        const auto out = cmd_execute_and_capture_output(cmd);
+        if (out.exit_code == 0)
+        {
+            return true;
+        }
+
+        msg::println(Color::warning, msgAwsFailedToDownload, msg::value = out.exit_code, msg::output = out.output);
+        return false;
+    }
+
+    struct AwsBinaryProvider : IBinaryProvider
+    {
+        AwsBinaryProvider(std::vector<std::string>&& read_prefixes, std::vector<std::string>&& write_prefixes)
+            : m_read_prefixes(std::move(read_prefixes)), m_write_prefixes(std::move(write_prefixes))
+        {
+        }
+
+        static std::string make_aws_path(const std::string& prefix, const std::string& abi)
+        {
+            return Strings::concat(prefix, abi, ".zip");
+        }
+
+        void prefetch(const VcpkgPaths& paths,
+                      View<Dependencies::InstallPlanAction> actions,
+                      View<CacheStatus*> cache_status) const override
+        {
+            auto& fs = paths.get_filesystem();
+
+            const auto timer = ElapsedTimer::create_started();
+
+            size_t restored_count = 0;
+            for (const auto& prefix : m_read_prefixes)
+            {
+                std::vector<std::pair<std::string, Path>> url_paths;
+                std::vector<size_t> url_indices;
+
+                for (size_t idx = 0; idx < actions.size(); ++idx)
+                {
+                    auto&& action = actions[idx];
+                    auto abi = action.package_abi().get();
+                    if (!abi || !cache_status[idx]->should_attempt_restore(this))
+                    {
+                        continue;
+                    }
+
+                    clean_prepare_dir(fs, paths.package_dir(action.spec));
+                    url_paths.emplace_back(make_aws_path(prefix, *abi),
+                                           make_temp_archive_path(paths.buildtrees(), action.spec));
+                    url_indices.push_back(idx);
+                }
+
+                if (url_paths.empty()) break;
+
+                msg::println(msgAwsAttemptingToFetchPackages, msg::value = url_paths.size());
+
+                std::vector<Command> jobs;
+                std::vector<size_t> idxs;
+                for (size_t idx = 0; idx < url_paths.size(); ++idx)
+                {
+                    auto&& action = actions[url_indices[idx]];
+                    auto&& url_path = url_paths[idx];
+                    if (!awscli_download_file(paths, url_path.first, url_path.second)) continue;
+                    jobs.push_back(decompress_archive_cmd(paths, paths.package_dir(action.spec), url_path.second));
+                    idxs.push_back(idx);
+                }
+
+                const auto job_results = cmd_execute_and_capture_output_parallel(jobs, get_clean_environment());
+
+                for (size_t j = 0; j < jobs.size(); ++j)
+                {
+                    const auto idx = idxs[j];
+                    if (job_results[j].exit_code != 0)
+                    {
+                        Debug::print("Failed to decompress ", url_paths[idx].second, '\n');
+                        continue;
+                    }
+
+                    // decompression success
+                    ++restored_count;
+                    fs.remove(url_paths[idx].second, VCPKG_LINE_INFO);
+                    cache_status[url_indices[idx]]->mark_restored();
+                }
+            }
+
+            msg::println(msgAwsRestoredPackages,
+                         msg::value = restored_count,
+                         msg::elapsed = timer.elapsed().as<std::chrono::seconds>().count());
+        }
+
+        RestoreResult try_restore(const VcpkgPaths&, const Dependencies::InstallPlanAction&) const override
+        {
+            return RestoreResult::unavailable;
+        }
+
+        void push_success(const VcpkgPaths& paths, const Dependencies::InstallPlanAction& action) const override
+        {
+            if (m_write_prefixes.empty()) return;
+            const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO);
+            auto& spec = action.spec;
+            const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec);
+            compress_directory(paths, paths.package_dir(spec), tmp_archive_path);
+
+            size_t upload_count = 0;
+            for (const auto& prefix : m_write_prefixes)
+            {
+                if (awscli_upload_file(paths, make_aws_path(prefix, abi), tmp_archive_path))
+                {
+                    ++upload_count;
+                }
+            }
+
+            msg::println(msgAwsUploadedPackages, msg::value = upload_count);
+        }
+
+        void precheck(const VcpkgPaths& paths,
+                      View<Dependencies::InstallPlanAction> actions,
+                      View<CacheStatus*> cache_status) const override
+        {
+            std::vector<CacheAvailability> actions_availability{actions.size()};
+            for (const auto& prefix : m_read_prefixes)
+            {
+                for (size_t idx = 0; idx < actions.size(); ++idx)
+                {
+                    auto&& action = actions[idx];
+                    const auto abi = action.package_abi().get();
+                    if (!abi || !cache_status[idx]->should_attempt_precheck(this))
+                    {
+                        continue;
+                    }
+
+                    if (awscli_stat(paths, make_aws_path(prefix, *abi)))
+                    {
+                        actions_availability[idx] = CacheAvailability::available;
+                        cache_status[idx]->mark_available(this);
+                    }
+                }
+            }
+
+            for (size_t idx = 0; idx < actions.size(); ++idx)
+            {
+                const auto this_cache_status = cache_status[idx];
+                if (this_cache_status && actions_availability[idx] == CacheAvailability::unavailable)
+                {
+                    this_cache_status->mark_unavailable(this);
+                }
+            }
+        }
+
+    private:
+        std::vector<std::string> m_read_prefixes;
+        std::vector<std::string> m_write_prefixes;
+    };
 }
 
 namespace vcpkg
@@ -1412,6 +1608,9 @@ namespace
         std::vector<std::string> gcs_read_prefixes;
         std::vector<std::string> gcs_write_prefixes;
 
+        std::vector<std::string> aws_read_prefixes;
+        std::vector<std::string> aws_write_prefixes;
+
         std::vector<std::string> sources_to_read;
         std::vector<std::string> sources_to_write;
 
@@ -1431,6 +1630,8 @@ namespace
             azblob_templates_to_put.clear();
             gcs_read_prefixes.clear();
             gcs_write_prefixes.clear();
+            aws_read_prefixes.clear();
+            aws_write_prefixes.clear();
             sources_to_read.clear();
             sources_to_write.clear();
             configs_to_read.clear();
@@ -1667,6 +1868,36 @@ namespace
 
                 handle_readwrite(state->gcs_read_prefixes, state->gcs_write_prefixes, std::move(p), segments, 2);
             }
+            else if (segments[0].second == "x-aws")
+            {
+                // Scheme: x-aws,<prefix>[,<readwrite>]
+                if (segments.size() < 2)
+                {
+                    return add_error("expected arguments: binary config 'aws' requires at least a prefix",
+                                     segments[0].first);
+                }
+
+                if (!Strings::starts_with(segments[1].second, "s3://"))
+                {
+                    return add_error(
+                        "invalid argument: binary config 'aws' requires a s3:// base url as the first argument",
+                        segments[1].first);
+                }
+
+                if (segments.size() > 3)
+                {
+                    return add_error("unexpected arguments: binary config 'aws' requires 1 or 2 arguments",
+                                     segments[3].first);
+                }
+
+                auto p = segments[1].second;
+                if (p.back() != '/')
+                {
+                    p.push_back('/');
+                }
+
+                handle_readwrite(state->aws_read_prefixes, state->aws_write_prefixes, std::move(p), segments, 2);
+            }
             else
             {
                 return add_error(
@@ -1927,6 +2158,12 @@ ExpectedS<std::vector<std::unique_ptr<IBinaryProvider>>> vcpkg::create_binary_pr
             std::make_unique<GcsBinaryProvider>(std::move(s.gcs_read_prefixes), std::move(s.gcs_write_prefixes)));
     }
 
+    if (!s.aws_read_prefixes.empty() || !s.aws_write_prefixes.empty())
+    {
+        providers.push_back(
+            std::make_unique<AwsBinaryProvider>(std::move(s.aws_read_prefixes), std::move(s.aws_write_prefixes)));
+    }
+
     if (!s.archives_to_read.empty() || !s.archives_to_write.empty() || !s.azblob_templates_to_put.empty())
     {
         providers.push_back(std::make_unique<ArchivesBinaryProvider>(std::move(s.archives_to_read),
@@ -2160,6 +2397,10 @@ void vcpkg::help_topic_binary_caching(const VcpkgPaths&)
                "**Experimental: will change or be removed without warning** Adds a Google Cloud Storage (GCS) source. "
                "Uses the gsutil CLI for uploads and downloads. Prefix should include the gs:// scheme and be suffixed "
                "with a `/`.");
+    tbl.format("x-aws,<prefix>[,<rw>]",
+               "**Experimental: will change or be removed without warning** Adds an AWS S3 source. "
+               "Uses the aws CLI for uploads and downloads. Prefix should include s3:// scheme and be suffixed "
+               "with a `/`.");
     tbl.format("interactive", "Enables interactive credential management for some source types");
     tbl.blank();
     tbl.text("The `<rw>` optional parameter for certain strings controls whether they will be consulted for "

src/vcpkg/tools.cpp

@@ -567,6 +567,35 @@ gsutil version: 4.58
         }
     };
 
+    struct AwsCliProvider : ToolProvider
+    {
+        std::string m_exe = "aws";
+
+        virtual const std::string& tool_data_name() const override { return m_exe; }
+        virtual const std::string& exe_stem() const override { return m_exe; }
+        virtual std::array<int, 3> default_min_version() const override { return {2, 4, 4}; }
+
+        virtual ExpectedS<std::string> get_version(const VcpkgPaths&, const Path& exe_path) const override
+        {
+            auto cmd = Command(exe_path).string_arg("--version");
+            auto rc = cmd_execute_and_capture_output(cmd);
+            if (rc.exit_code != 0)
+            {
+                return {Strings::concat(std::move(rc.output), "\n\nFailed to get version of ", exe_path, "\n"),
+                        expected_right_tag};
+            }
+
+            /* Sample output:
+aws-cli/2.4.4 Python/3.8.8 Windows/10 exe/AMD64 prompt/off
+                */
+
+            const auto idx = rc.output.find("aws-cli/");
+            Checks::check_exit(
+                VCPKG_LINE_INFO, idx != std::string::npos, "Unexpected format of awscli version string: %s", rc.output);
+            return {rc.output.substr(idx), expected_left_tag};
+        }
+    };
+
     struct IfwInstallerBaseProvider : ToolProvider
     {
         std::string m_exe;
@@ -745,6 +774,14 @@ gsutil version: 4.58
                     }
                     return get_path(paths, GsutilProvider());
                 }
+                if (tool == Tools::AWSCLI)
+                {
+                    if (get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value())
+                    {
+                        return {"aws", "0"};
+                    }
+                    return get_path(paths, AwsCliProvider());
+                }
                 if (tool == Tools::TAR)
                 {
                     auto tars = paths.get_filesystem().find_from_PATH("tar");