From 901b34fb71ff224a3dde41d0cf24a428511f5246 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 13:22:29 -0400 Subject: [PATCH] WIP --- src/libstore/dummy-store.cc | 6 +- src/libstore/http-binary-cache-store.cc | 5 +- src/libstore/http-binary-cache-store.hh | 2 +- src/libstore/local-binary-cache-store.hh | 2 +- src/libstore/local-store.cc | 4 +- src/libstore/local-store.hh | 2 +- src/libstore/s3-binary-cache-store.cc | 216 ++++++++++-------- src/libstore/s3-binary-cache-store.hh | 2 + src/libstore/store-api.hh | 2 +- .../unix/build/local-derivation-goal.cc | 63 +++-- 10 files changed, 180 insertions(+), 124 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 968c2d3fbac..3d2931cd9ba 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -36,7 +36,7 @@ struct DummyStoreConfig : virtual StoreConfig { return {"dummy"}; } - std::shared_ptr openStore() const override; + ref openStore() const override; }; @@ -100,9 +100,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -std::shared_ptr DummyStore::Config::openStore() const +ref DummyStore::Config::openStore() const { - return std::make_shared(*this); + return make_ref(*this); } static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 0a663ec036d..accd84899f1 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -207,8 +207,9 @@ class HttpBinaryCacheStore : } }; -std::shared_ptr HttpBinaryCacheStore::Config::openStore() const { - return std::make_shared(*this); +ref HttpBinaryCacheStore::Config::openStore() const +{ + return make_ref(*this); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index d15c6e3d0db..397bd0592a6 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -25,7 +25,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; } diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 01f7969eb61..647cd83d099 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -22,7 +22,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d3677fdeb39..2a930051d4f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -113,9 +113,9 @@ std::string LocalStoreConfig::doc() ; } -std::shared_ptr LocalStore::Config::openStore() const +ref LocalStore::Config::openStore() const { - return std::make_shared(*this); + return make_ref(*this); } struct LocalStore::State::Stmts { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 45cb04b7159..3d5aa1d3dbd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -72,7 +72,7 @@ struct LocalStoreConfig : std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; class LocalStore : diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index abd2727194b..6b9a809af96 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,6 +7,8 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" #include #include @@ -185,93 +187,119 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } - const Setting profile{ - this, - "", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; - - const Setting region{ - this, - Aws::Region::US_EAST_1, - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - - -S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , BinaryCacheStore(params) -{ } +S3BinaryCacheStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , BinaryCacheStore::Config::Descriptions{BinaryCacheStore::Config::descriptions} + , S3BinaryCacheStoreConfigT{ + .profile{ + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )", + }, + .region{ + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + .scheme{ + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + .endpoint{ + .name = "endpoint", + .description = R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )", + }, + .narinfoCompression{ + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + .lsCompression{ + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + .logCompression{ + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + .multipartUpload{ + .name = "multipart-upload", + .description = "Whether to use multi-part uploads.", + }, + .bufferSize{ + .name = "buffer-size", + .description = "Size (in bytes) of each part in multi-part uploads.", + }, + } +{} + +const S3BinaryCacheStore::Config::Descriptions S3BinaryCacheStore::Config::descriptions{}; + +decltype(S3BinaryCacheStore::Config::defaults) S3BinaryCacheStore::Config::defaults = { + .profile{""}, + .region{Aws::Region::US_EAST_1}, + .scheme{""}, + .endpoint{""}, + .narinfoCompression{""}, + .lsCompression{""}, + .logCompression{""}, + .multipartUpload{false}, + .bufferSize{5 * 1024 * 1024}, +}; -S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , bucketName(bucketName) +S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : Store::Config(params) + , BinaryCacheStore::Config(params) + , S3BinaryCacheStoreConfigT{ + CONFIG_ROW(profile), + CONFIG_ROW(region), + CONFIG_ROW(scheme), + CONFIG_ROW(endpoint), + CONFIG_ROW(narinfoCompression), + CONFIG_ROW(lsCompression), + CONFIG_ROW(logCompression), + CONFIG_ROW(multipartUpload), + CONFIG_ROW(bufferSize), + } { if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); + throw UsageError("`%s` store requires a bucket name in its Store URI", scheme); } + +S3BinaryCacheStore::S3BinaryCacheStore(const Config & config) + : BinaryCacheStore(config) +{ } + std::string S3BinaryCacheStoreConfig::doc() { return @@ -282,22 +310,17 @@ std::string S3BinaryCacheStoreConfig::doc() struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; - S3BinaryCacheStoreImpl( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) - , Store(params) - , BinaryCacheStore(params) - , S3BinaryCacheStore(params) + S3BinaryCacheStoreImpl(const Config & config) + : Store::Config{config} + , BinaryCacheStore::Config{config} + , S3BinaryCacheStore::Config{config} + , Store{static_cast(*this)} + , BinaryCacheStore{static_cast(*this)} + , S3BinaryCacheStore{static_cast(*this)} , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); @@ -539,7 +562,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual } }; -static RegisterStoreImplementation regS3BinaryCacheStore; +ref S3BinaryCacheStoreImpl::Config::openStore() const +{ + return make_ref(*this); +} + +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index c28e9320378..bff26f6b3d0 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -53,6 +53,8 @@ struct S3BinaryCacheStoreConfig : } std::string doc() override; + + ref openStore() const override; }; struct S3BinaryCacheStore : virtual BinaryCacheStore diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6571fd37c10..56fef0f1a7b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -154,7 +154,7 @@ struct StoreConfig : * Open a store of the type corresponding to this configuration * type. */ - virtual std::shared_ptr openStore() const = 0; + virtual ref openStore() const = 0; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index f968bbc5b7f..681ac032150 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1254,29 +1254,46 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) return this->isAllowed(pathPartOfReq(req)); } - struct RestrictedStoreConfig : virtual LocalFSStoreConfig { - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; + const std::string name() override { return "Restricted Store"; } + + ref openStore() const override; -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore -{ ref next; LocalDerivationGoal & goal; - RestrictedStore(const Params & params, ref next, LocalDerivationGoal & goal) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RestrictedStoreConfig(params) - , Store(params) - , LocalFSStore(params) - , next(next), goal(goal) + RestrictedStoreConfig( + const StoreReference::Params & params, + ref next, + LocalDerivationGoal & goal) + : Store::Config{params} + , LocalFSStore::Config{params} + , next{next} + , goal{goal} + { + } +}; + +/** + * A wrapper around LocalStore that only allows building/querying of + * paths that are in the input closures of the build or were added via + * recursive Nix calls. + */ +struct RestrictedStore : + virtual RestrictedStoreConfig, + virtual IndirectRootStore, + virtual GcStore +{ + using Config = RestrictedStoreConfig; + + RestrictedStore(const RestrictedStoreConfig & config) + : Store::Config(config) + , LocalFSStore::Config(config) + , RestrictedStore::Config(config) + , Store(static_cast(*this)) + , LocalFSStore(static_cast(*this)) { } Path getRealStoreDir() override @@ -1480,20 +1497,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In }; +ref RestrictedStore::Config::openStore() const +{ + return make_ref(*this); +} + + void LocalDerivationGoal::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); - Store::Params params; + StoreReference::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; if (auto & optRoot = getLocalStore().rootDir.get()) params["root"] = *optRoot; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; - auto store = make_ref(params, + auto store = RestrictedStore::Config{ + params, ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); + *this, + }.openStore(); addedPaths.clear();