Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
Ericson2314 committed Jul 18, 2024
1 parent ca1e895 commit 901b34f
Show file tree
Hide file tree
Showing 10 changed files with 180 additions and 124 deletions.
6 changes: 3 additions & 3 deletions src/libstore/dummy-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ struct DummyStoreConfig : virtual StoreConfig {
return {"dummy"};
}

std::shared_ptr<Store> openStore() const override;
ref<Store> openStore() const override;
};


Expand Down Expand Up @@ -100,9 +100,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store
{ unsupported("getFSAccessor"); }
};

std::shared_ptr<Store> DummyStore::Config::openStore() const
ref<Store> DummyStore::Config::openStore() const
{
return std::make_shared<DummyStore>(*this);
return make_ref<DummyStore>(*this);
}

static RegisterStoreImplementation<DummyStore> regDummyStore;
Expand Down
5 changes: 3 additions & 2 deletions src/libstore/http-binary-cache-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -207,8 +207,9 @@ class HttpBinaryCacheStore :
}
};

std::shared_ptr<Store> HttpBinaryCacheStore::Config::openStore() const {
return std::make_shared<HttpBinaryCacheStore>(*this);
ref<Store> HttpBinaryCacheStore::Config::openStore() const
{
return make_ref<HttpBinaryCacheStore>(*this);
}

static RegisterStoreImplementation<HttpBinaryCacheStore> regHttpBinaryCacheStore;
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/http-binary-cache-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig

std::string doc() override;

std::shared_ptr<Store> openStore() const override;
ref<Store> openStore() const override;
};

}
2 changes: 1 addition & 1 deletion src/libstore/local-binary-cache-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig

std::string doc() override;

std::shared_ptr<Store> openStore() const override;
ref<Store> openStore() const override;
};

}
4 changes: 2 additions & 2 deletions src/libstore/local-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,9 @@ std::string LocalStoreConfig::doc()
;
}

std::shared_ptr<Store> LocalStore::Config::openStore() const
ref<Store> LocalStore::Config::openStore() const
{
return std::make_shared<LocalStore>(*this);
return make_ref<LocalStore>(*this);
}

struct LocalStore::State::Stmts {
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/local-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ struct LocalStoreConfig :

std::string doc() override;

std::shared_ptr<Store> openStore() const override;
ref<Store> openStore() const override;
};

class LocalStore :
Expand Down
216 changes: 122 additions & 94 deletions src/libstore/s3-binary-cache-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
#include "globals.hh"
#include "compression.hh"
#include "filetransfer.hh"
#include "config-parse-impl.hh"
#include "store-registration.hh"

#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
Expand Down Expand Up @@ -185,93 +187,119 @@ S3Helper::FileTransferResult S3Helper::getObject(
return res;
}

const Setting<std::string> profile{
this,
"",
"profile",
R"(
The name of the AWS configuration profile to use. By default
Nix will use the `default` profile.
)"};

const Setting<std::string> region{
this,
Aws::Region::US_EAST_1,
"region",
R"(
The region of the S3 bucket. If your bucket is not in
`us–east-1`, you should always explicitly specify the region
parameter.
)"};

const Setting<std::string> scheme{
this,
"",
"scheme",
R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)"};

const Setting<std::string> endpoint{
this,
"",
"endpoint",
R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)"};

const Setting<std::string> narinfoCompression{
this, "", "narinfo-compression", "Compression method for `.narinfo` files."};

const Setting<std::string> lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."};

const Setting<std::string> logCompression{
this,
"",
"log-compression",
R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)"};

const Setting<bool> multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."};

const Setting<uint64_t> bufferSize{
this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."};


S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
: BinaryCacheStoreConfig(params)
, BinaryCacheStore(params)
{ }

S3BinaryCacheStore::Config::Descriptions::Descriptions()
: Store::Config::Descriptions{Store::Config::descriptions}
, BinaryCacheStore::Config::Descriptions{BinaryCacheStore::Config::descriptions}
, S3BinaryCacheStoreConfigT<config::SettingInfo>{
.profile{
.name = "profile",
.description = R"(
The name of the AWS configuration profile to use. By default
Nix will use the `default` profile.
)",
},
.region{
.name = "region",
.description = R"(
The region of the S3 bucket. If your bucket is not in
`us–east-1`, you should always explicitly specify the region
parameter.
)",
},
.scheme{
.name = "scheme",
.description = R"(
The scheme used for S3 requests, `https` (default) or `http`. This
option allows you to disable HTTPS for binary caches which don't
support it.
> **Note**
>
> HTTPS should be used if the cache might contain sensitive
> information.
)",
},
.endpoint{
.name = "endpoint",
.description = R"(
The URL of the endpoint of an S3-compatible service such as MinIO.
Do not specify this setting if you're using Amazon S3.
> **Note**
>
> This endpoint must support HTTPS and will use path-based
> addressing instead of virtual host based addressing.
)",
},
.narinfoCompression{
.name = "narinfo-compression",
.description = "Compression method for `.narinfo` files.",
},
.lsCompression{
.name = "ls-compression",
.description = "Compression method for `.ls` files.",
},
.logCompression{
.name = "log-compression",
.description = R"(
Compression method for `log/*` files. It is recommended to
use a compression method supported by most web browsers
(e.g. `brotli`).
)",
},
.multipartUpload{
.name = "multipart-upload",
.description = "Whether to use multi-part uploads.",
},
.bufferSize{
.name = "buffer-size",
.description = "Size (in bytes) of each part in multi-part uploads.",
},
}
{}

const S3BinaryCacheStore::Config::Descriptions S3BinaryCacheStore::Config::descriptions{};

decltype(S3BinaryCacheStore::Config::defaults) S3BinaryCacheStore::Config::defaults = {
.profile{""},
.region{Aws::Region::US_EAST_1},
.scheme{""},
.endpoint{""},
.narinfoCompression{""},
.lsCompression{""},
.logCompression{""},
.multipartUpload{false},
.bufferSize{5 * 1024 * 1024},
};

S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
std::string_view uriScheme,
std::string_view bucketName,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, bucketName(bucketName)
S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig(
std::string_view scheme,
std::string_view authority,
const StoreReference::Params & params)
: Store::Config(params)
, BinaryCacheStore::Config(params)
, S3BinaryCacheStoreConfigT<config::JustValue>{
CONFIG_ROW(profile),
CONFIG_ROW(region),
CONFIG_ROW(scheme),
CONFIG_ROW(endpoint),
CONFIG_ROW(narinfoCompression),
CONFIG_ROW(lsCompression),
CONFIG_ROW(logCompression),
CONFIG_ROW(multipartUpload),
CONFIG_ROW(bufferSize),
}
{
if (bucketName.empty())
throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme);
throw UsageError("`%s` store requires a bucket name in its Store URI", scheme);
}


S3BinaryCacheStore::S3BinaryCacheStore(const Config & config)
: BinaryCacheStore(config)
{ }

std::string S3BinaryCacheStoreConfig::doc()
{
return
Expand All @@ -282,22 +310,17 @@ std::string S3BinaryCacheStoreConfig::doc()

struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore
{
std::string bucketName;

Stats stats;

S3Helper s3Helper;

S3BinaryCacheStoreImpl(
std::string_view uriScheme,
std::string_view bucketName,
const Params & params)
: StoreConfig(params)
, BinaryCacheStoreConfig(params)
, S3BinaryCacheStoreConfig(uriScheme, bucketName, params)
, Store(params)
, BinaryCacheStore(params)
, S3BinaryCacheStore(params)
S3BinaryCacheStoreImpl(const Config & config)
: Store::Config{config}
, BinaryCacheStore::Config{config}
, S3BinaryCacheStore::Config{config}
, Store{static_cast<const Store::Config &>(*this)}
, BinaryCacheStore{static_cast<const BinaryCacheStore::Config &>(*this)}
, S3BinaryCacheStore{static_cast<const S3BinaryCacheStore::Config &>(*this)}
, s3Helper(profile, region, scheme, endpoint)
{
diskCache = getNarInfoDiskCache();
Expand Down Expand Up @@ -539,7 +562,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
}
};

static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regS3BinaryCacheStore;
ref<Store> S3BinaryCacheStoreImpl::Config::openStore() const
{
return make_ref<S3BinaryCacheStoreImpl>(*this);
}

static RegisterStoreImplementation<S3BinaryCacheStoreImpl> regS3BinaryCacheStore;

}

Expand Down
2 changes: 2 additions & 0 deletions src/libstore/s3-binary-cache-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ struct S3BinaryCacheStoreConfig :
}

std::string doc() override;

ref<Store> openStore() const override;
};

struct S3BinaryCacheStore : virtual BinaryCacheStore
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/store-api.hh
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ struct StoreConfig :
* Open a store of the type corresponding to this configuration
* type.
*/
virtual std::shared_ptr<Store> openStore() const = 0;
virtual ref<Store> openStore() const = 0;
};

class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
Expand Down
Loading

0 comments on commit 901b34f

Please sign in to comment.