Skip to content

Commit

Permalink
Update fetch_cvd to support CAS downloading
Browse files Browse the repository at this point in the history
  • Loading branch information
frankfeng579 authored and Databean committed Dec 19, 2024
1 parent e5c0fa2 commit 2b4df0e
Show file tree
Hide file tree
Showing 6 changed files with 320 additions and 4 deletions.
3 changes: 3 additions & 0 deletions base/cvd/cuttlefish/host/commands/cvd/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,8 @@ cc_binary(
cc_test(
name = "cvd_test",
srcs = [
"unittests/fetch/fetch_cvd_parser_test.cpp",
"unittests/fetch/fetch_cvd_test.cpp",
"unittests/parser/configs_inheritance_test.cc",
"unittests/parser/flags_parser_test.cc",
"unittests/parser/instance/boot_configs_test.cc",
Expand Down Expand Up @@ -339,6 +341,7 @@ cc_test(
"//cuttlefish/common/libs/utils:result_matchers",
"//cuttlefish/host/libs/config",
"//libbase",
"@fmt",
"@googletest//:gtest",
"@googletest//:gtest_main",
"@jsoncpp",
Expand Down
56 changes: 52 additions & 4 deletions base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd.cc
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>

#include <android-base/logging.h>
Expand All @@ -44,6 +45,7 @@
#include "host/libs/web/android_build_api.h"
#include "host/libs/web/android_build_string.h"
#include "host/libs/web/caching_build_api.h"
#include "host/libs/web/cas/cas_downloader.h"
#include "host/libs/web/chrome_os_build_string.h"
#include "host/libs/web/credential_source.h"
#include "host/libs/web/http_client/curl_global_init.h"
Expand Down Expand Up @@ -338,6 +340,9 @@ Result<std::unique_ptr<CredentialSource>> GetCredentialSourceFromFlags(
flags.credential_flags.credential_filepath,
flags.credential_flags.service_account_filepath));
}

} // namespace

Result<std::unique_ptr<BuildApi>> GetBuildApi(const BuildApiFlags& flags) {
auto resolver =
flags.external_dns_resolver ? GetEntDnsResolve : NameResolver();
Expand Down Expand Up @@ -365,13 +370,23 @@ Result<std::unique_ptr<BuildApi>> GetBuildApi(const BuildApiFlags& flags) {
oauth_filepath));

const auto cache_base_path = PerUserDir() + "/cache";

std::unique_ptr<CasDownloader> cas_downloader = nullptr;
Result<std::unique_ptr<CasDownloader>> result =
CasDownloader::Create(flags.cas_downloader_flags,
flags.credential_flags.service_account_filepath);
if (result.ok()) {
cas_downloader = std::move(result.value());
}
return CreateBuildApi(std::move(retrying_http_client), std::move(curl),
std::move(credential_source), std::move(flags.api_key),
flags.wait_retry_period, std::move(flags.api_base_url),
std::move(flags.project_id), flags.enable_caching,
std::move(cache_base_path));
std::move(cache_base_path), std::move(cas_downloader));
}

namespace {

Result<LuciBuildApi> GetLuciBuildApi(const BuildApiFlags& flags) {
auto resolver =
flags.external_dns_resolver ? GetEntDnsResolve : NameResolver();
Expand Down Expand Up @@ -475,6 +490,39 @@ Result<void> SaveConfig(FetcherConfig& config,
return {};
}

Result<std::vector<std::string>> ExtractImageContents(
const std::string& image_filepath, const std::string& target_dir,
const bool keep_archive) {
if (!IsDirectory(image_filepath)) {
return CF_EXPECT(
ExtractArchiveContents(image_filepath, target_dir, keep_archive));
}

// The image is already uncompressed. Link or move its contents.
std::vector<std::string> files;
const std::function<bool(const std::string&)> file_collector =
[&files, &image_filepath,
&target_dir](const std::string& filepath) mutable {
std::string target_filepath =
target_dir + "/" + filepath.substr(image_filepath.size() + 1);
if (!IsDirectory(filepath)) {
files.push_back(target_filepath);
}
return true;
};
CF_EXPECT(WalkDirectory(image_filepath, file_collector));

if (keep_archive) {
// Must use hard linking due to the way fetch_cvd uses the cache.
CF_EXPECT(HardLinkDirecoryContentsRecursively(image_filepath, target_dir));
} else {
CF_EXPECT(MoveDirectoryContents(image_filepath, target_dir));
// Ignore even if removing directory fails - harmless.
RecursivelyRemoveDirectory(image_filepath);
}
return files;
}

Result<void> FetchDefaultTarget(BuildApi& build_api, const Builds& builds,
const TargetDirectories& target_directories,
const DownloadFlags& flags,
Expand Down Expand Up @@ -503,9 +551,9 @@ Result<void> FetchDefaultTarget(BuildApi& build_api, const Builds& builds,
*builds.default_build, target_directories.root, img_zip_name));
trace.CompletePhase("Download image zip",
FileSize(default_img_zip_filepath));
std::vector<std::string> image_files = CF_EXPECT(ExtractArchiveContents(
default_img_zip_filepath, target_directories.root,
keep_downloaded_archives));
std::vector<std::string> image_files = CF_EXPECT(
ExtractImageContents(default_img_zip_filepath, target_directories.root,
keep_downloaded_archives));
trace.CompletePhase("Extract image zip contents");
LOG(DEBUG) << "Adding img-zip files for default build";
for (auto& file : image_files) {
Expand Down
85 changes: 85 additions & 0 deletions base/cvd/cuttlefish/host/commands/cvd/fetch/fetch_cvd_parser.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "host/commands/cvd/fetch/fetch_cvd_parser.h"

#include <chrono>
#include <cstdint>
#include <iostream>
#include <optional>
#include <sstream>
Expand All @@ -30,6 +31,7 @@
#include "common/libs/utils/flag_parser.h"
#include "common/libs/utils/result.h"
#include "host/libs/web/android_build_string.h"
#include "host/libs/web/cas/cas_downloader.h"

namespace cuttlefish {
namespace {
Expand Down Expand Up @@ -58,6 +60,19 @@ Flag GflagsCompatFlagSeconds(const std::string& name,
});
}

Flag GflagsCompatFlagInt64(const std::string& name, int64_t& value) {
return GflagsCompatFlag(name)
.Getter([&value]() { return std::to_string(value); })
.Setter([&value](const FlagMatch& match) -> Result<void> {
int64_t parsed_int;
CF_EXPECTF(android::base::ParseInt(match.value, &parsed_int),
"Failed to parse \"{}\" as an integer (int64_t)",
match.value);
value = parsed_int;
return {};
});
}

std::vector<Flag> GetFlagsVector(FetchFlags& fetch_flags,
std::string& directory) {
std::vector<Flag> flags;
Expand Down Expand Up @@ -118,6 +133,76 @@ std::vector<Flag> GetFlagsVector(FetchFlags& fetch_flags,
.Help("Enforce reading service account credentials "
"from the given filepath."));

// Most of these flags are passed to casdownloader (a go binary). A prefix
// "cas-" is added to the flag name if it doesn't already have one to minimize
// chances of flag name conflicts and "-" is replaced with "_" for consistency
// with cvd flags. E.g. the casdownloader flag "cache-dir" becomes cvd flag
// "cas_cache_dir".
CasDownloaderFlags& cas_downloader_flags =
fetch_flags.build_api_flags.cas_downloader_flags;
flags.emplace_back(GflagsCompatFlag("cas_config_filepath",
cas_downloader_flags.cas_config_filepath)
.Help("Path to the CAS downloader config file. Other "
"CAS flags will be ignored if this is set."));
flags.emplace_back(
GflagsCompatFlag("cas_downloader_path",
cas_downloader_flags.downloader_path)
.Help("Path to the CAS downloader binary. Enables CAS downloading if "
"specified."));
flags.emplace_back(
GflagsCompatFlag("cas_prefer_uncompressed",
cas_downloader_flags.prefer_uncompressed)
.Help("Download uncompressed artifacts if available."));
flags.emplace_back(
GflagsCompatFlag("cas_cache_dir", cas_downloader_flags.cache_dir)
.Help("Cache directory to store downloaded files (casdownloader "
"flag: cache-dir)."));
flags.emplace_back(
GflagsCompatFlagInt64("cas_cache_max_size",
cas_downloader_flags.cache_max_size)
.Help("Cache is trimmed if the cache gets larger than "
"this value in bytes (casdownloader flag: cache-max-size)."));
flags.emplace_back(
GflagsCompatFlag("cas_cache_lock", cas_downloader_flags.use_hardlink)
.Help("Enable cache lock (casdownloader flag: cache-lock)."));
flags.emplace_back(
GflagsCompatFlag("cas_use_hardlink", cas_downloader_flags.use_hardlink)
.Help("By default local cache will use hardlink when push and pull "
"files (casdownloader flag: use-hardlink)."));
flags.emplace_back(
GflagsCompatFlag("cas_concurrency", cas_downloader_flags.cas_concurrency)
.Help("the maximum number of concurrent download operations "
"(casdownloader flag: cas-concurrency)."));
flags.emplace_back(
GflagsCompatFlag("cas_memory_limit", cas_downloader_flags.memory_limit)
.Help("Memory limit in MiB (casdownloader flag: memory-limit)."));
flags.emplace_back(
GflagsCompatFlag("cas_rpc_timeout", cas_downloader_flags.rpc_timeout)
.Help("Default RPC timeout in seconds (casdownloader flag: "
"rpc-timeout)."));
flags.emplace_back(
GflagsCompatFlag("cas_get_capabilities_timeout",
cas_downloader_flags.get_capabilities_timeout)
.Help("RPC timeout for GetCapabilities in seconds (casdownloader "
"flag: get-capabilities-timeout)."));
flags.emplace_back(GflagsCompatFlag("cas_get_tree_timeout",
cas_downloader_flags.get_tree_timeout)
.Help("RPC timeout for GetTree in seconds "
"(casdownloader flag: get-tree-timeout)."));
flags.emplace_back(
GflagsCompatFlag("cas_batch_read_blobs_timeout",
cas_downloader_flags.batch_read_blobs_timeout)
.Help("RPC timeout for BatchReadBlobs in seconds (casdownloader "
"flag: batch-read-blobs-timeout)."));
flags.emplace_back(
GflagsCompatFlag("cas_batch_update_blobs_timeout",
cas_downloader_flags.batch_update_blobs_timeout)
.Help("RPC timeout for BatchUpdateBlobs in seconds (casdownloader "
"flag: batch-update-blobs-timeout)."));
flags.emplace_back(GflagsCompatFlag("version", cas_downloader_flags.version)
.Help("Print CAS downloader version information "
"(casdownloader flag: version)."));

VectorFlags& vector_flags = fetch_flags.vector_flags;
flags.emplace_back(
GflagsCompatFlag("default_build", vector_flags.default_build)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include "common/libs/utils/result.h"
#include "host/libs/web/android_build_api.h"
#include "host/libs/web/android_build_string.h"
#include "host/libs/web/cas/cas_downloader.h"
#include "host/libs/web/chrome_os_build_string.h"

namespace cuttlefish {
Expand Down Expand Up @@ -68,6 +69,7 @@ struct BuildApiFlags {
bool external_dns_resolver = kDefaultExternalDnsResolver;
std::string api_base_url = kAndroidBuildServiceUrl;
bool enable_caching = kDefaultEnableCaching;
CasDownloaderFlags cas_downloader_flags;
};

struct VectorFlags {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
//
// Copyright (C) 2024 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "host/commands/cvd/fetch/fetch_cvd_parser.h"

#include <string>

#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "common/libs/utils/result.h"
#include "common/libs/utils/result_matchers.h"

namespace cuttlefish {

using testing::Eq;

inline constexpr char kFetchCvd[] = "fetch_cvd";
inline constexpr char kTargetDirectory[] = "--target_directory=/tmp/fetch_test";
inline constexpr char kDefaultBuild[] =
"--default_build=aosp-main/aosp_cf_x86_64_phone-trunk_staging-userdebug";
inline constexpr char kCasDownloaderPath[] =
"--cas_downloader_path=/tmp/casdownloader";
inline constexpr char kCasCacheDir[] = "--cas_cache_dir=/tmp/cas_cache";
inline constexpr char kCasCacheMaxSize[] = "--cas_cache_max_size=10000000000";

TEST(FetchCvdParserTests, CreatesCasDownloaderFlags) {
std::string fetch_cvd = std::string(kFetchCvd);
std::string target_directory = std::string(kTargetDirectory);
std::string default_build = std::string(kDefaultBuild);
std::string cas_downloader_path = std::string(kCasDownloaderPath);
std::string cas_cache_dir = std::string(kCasCacheDir);
std::string cas_cache_max_size = std::string(kCasCacheMaxSize);
char* argv[] = {fetch_cvd.data(),
target_directory.data(),
default_build.data(),
cas_downloader_path.data(),
cas_cache_dir.data(),
cas_cache_max_size.data(),
nullptr};
int argc = sizeof(argv) / sizeof(char*) - 1;

Result<FetchFlags> flagsRes = GetFlagValues(argc, argv);

EXPECT_THAT(flagsRes, IsOk());
FetchFlags flags = flagsRes.value();
EXPECT_THAT(flags.build_api_flags.cas_downloader_flags.downloader_path,
Eq("/tmp/casdownloader"));
EXPECT_THAT(flags.build_api_flags.cas_downloader_flags.cache_dir,
Eq("/tmp/cas_cache"));
EXPECT_THAT(flags.build_api_flags.cas_downloader_flags.cache_max_size,
Eq(10000000000));
}

} // namespace cuttlefish
Loading

0 comments on commit 2b4df0e

Please sign in to comment.