From 39ba81a4eb7404f9a12cf0768da51c0e0e26b588 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:52:57 -0400 Subject: [PATCH 1/4] Improve internal API docs for two file hashing functions Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/hash.hh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4e3..c3aa5cd8184 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -145,13 +145,17 @@ std::string printHash16or32(const Hash & hash); Hash hashString(HashType ht, std::string_view s); /** - * Compute the hash of the given file. + * Compute the hash of the given file, hashing its contents directly. + * + * (Metadata, such as the executable permission bit, is ignored.) */ Hash hashFile(HashType ht, const Path & path); /** - * Compute the hash of the given path. The hash is defined as - * (essentially) hashString(ht, dumpPath(path)). + * Compute the hash of the given path, serializing as a Nix Archive and + * then hashing that. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, From 9d6114313b1b06a020bed230cfe7f05e6075b875 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:31:19 -0400 Subject: [PATCH 2/4] Move `ParseSink` to its own header We will soon add a new implemenation so the one for NARs in `archive.cc` isn't the only one. Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/archive.hh | 17 +---------------- src/libutil/fs-sink.hh | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 src/libutil/fs-sink.hh diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 2cf164a417d..3530783c1d2 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "fs-sink.hh" namespace nix { @@ -72,22 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * \todo Fix this API, it sucks. - */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; -}; - /** * If the NAR archive contains a single file at top-level, then save * the contents of the file to `s`. Otherwise barf. diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 00000000000..ed6484e6111 --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,25 @@ +#pragma once +///@file + +#include "types.hh" +#include "serialise.hh" + +namespace nix { + +/** + * \todo Fix this API, it sucks. + */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void closeRegularFile() { }; + virtual void isExecutable() { }; + virtual void preallocateContents(uint64_t size) { }; + virtual void receiveContents(std::string_view data) { }; + + virtual void createSymlink(const Path & path, const std::string & target) { }; +}; + +} From 8a416e819c9727c2e9efd70f1d2a4b0bfda11663 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:49:01 -0400 Subject: [PATCH 3/4] Move `RestoreSink` to `fs-sink.cc` Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/archive.cc | 67 ---------------------------------- src/libutil/fs-sink.cc | 82 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 67 deletions(-) create mode 100644 src/libutil/fs-sink.cc diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 268a798d900..0f0cae7c02a 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -27,8 +27,6 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, false, "preallocate-contents", - "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; @@ -302,71 +300,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - void restorePath(const Path & path, Source & source) { RestoreSink sink; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc new file mode 100644 index 00000000000..fcffd32165e --- /dev/null +++ b/src/libutil/fs-sink.cc @@ -0,0 +1,82 @@ +#include + +#include "config.hh" +#include "fs-sink.hh" + +namespace nix { + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{this, false, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; +}; + +static RestoreSinkSettings restoreSinkSettings; + +static GlobalConfig::Register r1(&restoreSinkSettings); + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) override + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); + }; + + void createRegularFile(const Path & path) override + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); + } + + void closeRegularFile() override + { + /* Call close explicitly to make sure the error is checked */ + fd.close(); + } + + void isExecutable() override + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(uint64_t len) override + { + if (!archiveSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif + } + + void receiveContents(std::string_view data) override + { + writeFull(fd.get(), data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + +} From f2e201fbdb93925b9588f78a03d67fcd91c74b65 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:49:01 -0400 Subject: [PATCH 4/4] Expose `RestoreSink` in header (`fs-sink.hh`) Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/fs-sink.cc | 101 ++++++++++++++++++++--------------------- src/libutil/fs-sink.hh | 17 +++++++ 2 files changed, 65 insertions(+), 53 deletions(-) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index fcffd32165e..a08a723a4fc 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -15,68 +15,63 @@ static RestoreSinkSettings restoreSinkSettings; static GlobalConfig::Register r1(&restoreSinkSettings); -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; +void RestoreSink::createDirectory(const Path & path) +{ + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); +}; - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } +void RestoreSink::createRegularFile(const Path & path) +{ + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); +} - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } +void RestoreSink::closeRegularFile() +{ + /* Call close explicitly to make sure the error is checked */ + fd.close(); +} - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } +void RestoreSink::isExecutable() +{ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); +} - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; +void RestoreSink::preallocateContents(uint64_t len) +{ + if (!restoreSinkSettings.preallocateContents) + return; #if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); } +#endif +} - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } +void RestoreSink::receiveContents(std::string_view data) +{ + writeFull(fd.get(), data); +} - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; +void RestoreSink::createSymlink(const Path & path, const std::string & target) +{ + Path p = dstPath + path; + nix::createSymlink(target, p); +} } diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index ed6484e6111..6837e2fc4b2 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -22,4 +22,21 @@ struct ParseSink virtual void createSymlink(const Path & path, const std::string & target) { }; }; +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void closeRegularFile() override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; + void receiveContents(std::string_view data) override; + + void createSymlink(const Path & path, const std::string & target) override; +}; + }