Skip to content

Commit

Permalink
Move (un)zip functions to archive.h and checks if (de)compression fai…
Browse files Browse the repository at this point in the history 
…ls (#338)

* Move (un)zip functions to archive.h and checks if (de)compression fails

* Don't use * to match files
  • Loading branch information
@autoantwort
autoantwort authored Feb 11, 2022
1 parent dac008b commit d13ece3
Show file tree
Hide file tree
Showing 5 changed files with 103 additions and 82 deletions.
10 changes: 10 additions & 0 deletions include/vcpkg/archives.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
#pragma once

#include <vcpkg/base/fwd/span.h>
#include <vcpkg/base/fwd/system.process.h>

#include <vcpkg/fwd/vcpkgpaths.h>

#include <vcpkg/base/files.h>
Expand All @@ -10,4 +13,11 @@ namespace vcpkg
void extract_tar(const Path& tar_tool, const Path& archive, const Path& to_path);
// Extract `archive` to `to_path`, deleting `to_path` first.
void extract_archive(const VcpkgPaths& paths, const Path& archive, const Path& to_path);

// Compress the source directory into the destination file.
int compress_directory_to_zip(const VcpkgPaths& paths, const Path& source, const Path& destination);

Command decompress_zip_archive_cmd(const VcpkgPaths& paths, const Path& dst, const Path& archive_path);

std::vector<ExitCodeAndOutput> decompress_in_parallel(View<Command> jobs);
}
1 change: 1 addition & 0 deletions include/vcpkg/base/fwd/system.process.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@ namespace vcpkg
struct CMakeVariable;
struct Command;
struct CommandLess;
struct ExitCodeAndOutput;
}
61 changes: 61 additions & 0 deletions src/vcpkg/archives.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#include <vcpkg/base/system.debug.h>
#include <vcpkg/base/system.print.h>
#include <vcpkg/base/system.process.h>

Expand Down Expand Up @@ -181,4 +182,64 @@ namespace vcpkg
Path to_path_partial = extract_archive_to_temp_subdirectory(paths, archive, to_path);
fs.rename_with_retry(to_path_partial, to_path, VCPKG_LINE_INFO);
}

int compress_directory_to_zip(const VcpkgPaths& paths, const Path& source, const Path& destination)
{
auto& fs = paths.get_filesystem();
fs.remove(destination, VCPKG_LINE_INFO);
#if defined(_WIN32)
auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

return cmd_execute_and_capture_output(
Command{seven_zip_exe}.string_arg("a").path_arg(destination).path_arg(source / "*"),
get_clean_environment())
.exit_code;

#else
return cmd_execute_clean(Command{"zip"}
.string_arg("--quiet")
.string_arg("-y")
.string_arg("-r")
.path_arg(destination)
.string_arg("*")
.string_arg("--exclude")
.string_arg(".DS_Store"),
InWorkingDirectory{source});
#endif
}

Command decompress_zip_archive_cmd(const VcpkgPaths& paths, const Path& dst, const Path& archive_path)
{
Command cmd;
#if defined(_WIN32)
auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
cmd.path_arg(seven_zip_exe)
.string_arg("x")
.path_arg(archive_path)
.string_arg("-o" + dst.native())
.string_arg("-y");
#else
(void)paths;
cmd.string_arg("unzip").string_arg("-qq").path_arg(archive_path).string_arg("-d" + dst.native());
#endif
return cmd;
}

std::vector<ExitCodeAndOutput> decompress_in_parallel(View<Command> jobs)
{
auto results = cmd_execute_and_capture_output_parallel(jobs, get_clean_environment());
#ifdef __APPLE__
int i = 0;
for (auto& result : results)
{
if (result.exit_code == 127 && result.output.empty())
{
Debug::print(jobs[i].command_line(), ": pclose returned 127, try again \n");
result = cmd_execute_and_capture_output(jobs[i], get_clean_environment());
}
++i;
}
#endif
return results;
}
}
92 changes: 27 additions & 65 deletions src/vcpkg/binarycaching.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <vcpkg/base/system.process.h>
#include <vcpkg/base/xmlserializer.h>

#include <vcpkg/archives.h>
#include <vcpkg/binarycaching.h>
#include <vcpkg/binarycaching.private.h>
#include <vcpkg/build.h>
Expand Down Expand Up @@ -183,63 +184,6 @@ namespace
Checks::check_exit(VCPKG_LINE_INFO, created_last, "unable to clear path: %s", dir);
}

static Command decompress_archive_cmd(const VcpkgPaths& paths, const Path& dst, const Path& archive_path)
{
Command cmd;
#if defined(_WIN32)
auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);
cmd.path_arg(seven_zip_exe)
.string_arg("x")
.path_arg(archive_path)
.string_arg("-o" + dst.native())
.string_arg("-y");
#else
(void)paths;
cmd.string_arg("unzip").string_arg("-qq").path_arg(archive_path).string_arg("-d" + dst.native());
#endif
return cmd;
}

static std::vector<ExitCodeAndOutput> decompress_in_parallel(View<Command> jobs)
{
auto results = cmd_execute_and_capture_output_parallel(jobs, get_clean_environment());
#ifdef __APPLE__
int i = 0;
for (auto& result : results)
{
if (result.exit_code == 127 && result.output.empty())
{
Debug::print(jobs[i].command_line(), ": pclose returned 127, try again \n");
result = cmd_execute_and_capture_output(jobs[i], get_clean_environment());
}
++i;
}
#endif
return results;
}

// Compress the source directory into the destination file.
static void compress_directory(const VcpkgPaths& paths, const Path& source, const Path& destination)
{
auto& fs = paths.get_filesystem();
fs.remove(destination, VCPKG_LINE_INFO);
#if defined(_WIN32)
auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

cmd_execute_and_capture_output(
Command{seven_zip_exe}.string_arg("a").path_arg(destination).path_arg(source / "*"),
get_clean_environment());
#else
cmd_execute_clean(Command{"zip"}
.string_arg("--quiet")
.string_arg("-y")
.string_arg("-r")
.path_arg(destination)
.string_arg("*"),
InWorkingDirectory{source});
#endif
}

static Path make_temp_archive_path(const Path& buildtrees, const PackageSpec& spec)
{
return buildtrees / spec.name() / (spec.triplet().to_string() + ".zip");
Expand Down Expand Up @@ -322,7 +266,7 @@ namespace
{
auto pkg_path = paths.package_dir(spec);
clean_prepare_dir(fs, pkg_path);
jobs.push_back(decompress_archive_cmd(paths, pkg_path, archive_path));
jobs.push_back(decompress_zip_archive_cmd(paths, pkg_path, archive_path));
action_idxs.push_back(i);
archive_paths.push_back(std::move(archive_path));
}
Expand Down Expand Up @@ -384,7 +328,14 @@ namespace
auto& fs = paths.get_filesystem();
const auto archive_subpath = make_archive_subpath(abi_tag);
const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec);
compress_directory(paths, paths.package_dir(spec), tmp_archive_path);
int code = compress_directory_to_zip(paths, paths.package_dir(spec), tmp_archive_path);
if (code != 0)
{
vcpkg::print2(
Color::warning, "Failed to compress folder '", paths.package_dir(spec), "', exit code: ", code);
return;
}

size_t http_remotes_pushed = 0;
for (auto&& put_url_template : m_put_url_templates)
{
Expand Down Expand Up @@ -529,7 +480,7 @@ namespace
if (codes[i] == 200)
{
action_idxs.push_back(i);
jobs.push_back(decompress_archive_cmd(
jobs.push_back(decompress_zip_archive_cmd(
paths, paths.package_dir(actions[url_indices[i]].spec), url_paths[i].second));
}
}
Expand Down Expand Up @@ -1066,7 +1017,7 @@ namespace
auto&& action = actions[url_indices[idx]];
auto&& url_path = url_paths[idx];
if (!gsutil_download_file(url_path.first, url_path.second)) continue;
jobs.push_back(decompress_archive_cmd(paths, paths.package_dir(action.spec), url_path.second));
jobs.push_back(decompress_zip_archive_cmd(paths, paths.package_dir(action.spec), url_path.second));
idxs.push_back(idx);
}

Expand Down Expand Up @@ -1106,8 +1057,13 @@ namespace
const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO);
auto& spec = action.spec;
const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec);
compress_directory(paths, paths.package_dir(spec), tmp_archive_path);

int code = compress_directory_to_zip(paths, paths.package_dir(spec), tmp_archive_path);
if (code != 0)
{
vcpkg::print2(
Color::warning, "Failed to compress folder '", paths.package_dir(spec), "', exit code: ", code);
return;
}
size_t upload_count = 0;
for (const auto& prefix : m_write_prefixes)
{
Expand Down Expand Up @@ -1249,7 +1205,7 @@ namespace
auto&& action = actions[url_indices[idx]];
auto&& url_path = url_paths[idx];
if (!awscli_download_file(paths, url_path.first, url_path.second)) continue;
jobs.push_back(decompress_archive_cmd(paths, paths.package_dir(action.spec), url_path.second));
jobs.push_back(decompress_zip_archive_cmd(paths, paths.package_dir(action.spec), url_path.second));
idxs.push_back(idx);
}

Expand Down Expand Up @@ -1287,7 +1243,13 @@ namespace
const auto& abi = action.package_abi().value_or_exit(VCPKG_LINE_INFO);
auto& spec = action.spec;
const auto tmp_archive_path = make_temp_archive_path(paths.buildtrees(), spec);
compress_directory(paths, paths.package_dir(spec), tmp_archive_path);
int code = compress_directory_to_zip(paths, paths.package_dir(spec), tmp_archive_path);
if (code != 0)
{
vcpkg::print2(
Color::warning, "Failed to compress folder '", paths.package_dir(spec), "', exit code: ", code);
return;
}

size_t upload_count = 0;
for (const auto& prefix : m_write_prefixes)
Expand Down
21 changes: 4 additions & 17 deletions src/vcpkg/export.prefab.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <vcpkg/base/system.print.h>
#include <vcpkg/base/system.process.h>

#include <vcpkg/archives.h>
#include <vcpkg/build.h>
#include <vcpkg/cmakevars.h>
#include <vcpkg/commands.h>
Expand Down Expand Up @@ -194,22 +195,6 @@ namespace vcpkg::Export::Prefab
return {};
}

static void compress_directory(const VcpkgPaths& paths, const Path& source, const Path& destination)
{
auto& fs = paths.get_filesystem();
fs.remove(destination, VCPKG_LINE_INFO);
#if defined(_WIN32)
auto&& seven_zip_exe = paths.get_tool_exe(Tools::SEVEN_ZIP);

cmd_execute_and_capture_output(
Command(seven_zip_exe).string_arg("a").path_arg(destination).path_arg(source / "*"),
get_clean_environment());
#else
cmd_execute_clean(Command{"zip"}.string_arg("--quiet").string_arg("-r").path_arg(destination).string_arg("*"),
InWorkingDirectory{source});
#endif
}

static void maven_install(const Path& aar, const Path& pom, const Options& prefab_options)
{
if (prefab_options.enable_debug)
Expand Down Expand Up @@ -633,7 +618,9 @@ namespace vcpkg::Export::Prefab
"[DEBUG] Exporting AAR And POM\n\tAAR Path %s\n\tPOM Path %s\n", exported_archive_path, pom_path));
}

compress_directory(paths, package_directory, exported_archive_path);
Checks::check_exit(VCPKG_LINE_INFO,
compress_directory_to_zip(paths, package_directory, exported_archive_path) != 0,
Strings::concat("Failed to compress folder ", package_directory));

std::string POM = R"(<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
Expand Down

0 comments on commit d13ece3

Please sign in to comment.