Skip to content

Commit

Permalink
Add support to specifying storage account in AzureConfig (#8090)
Browse files Browse the repository at this point in the history
We want to be able to specify the storage account via the toml
configuration, so that we can connect to multiple storage accounts in
the same process.

https://neondb.slack.com/archives/C06SJG60FRB/p1718702144270139
  • Loading branch information
arpad-m authored Jun 18, 2024
1 parent 4feb6ba commit 68a2298
Show file tree
Hide file tree
Showing 4 changed files with 22 additions and 4 deletions.
3 changes: 2 additions & 1 deletion docs/pageserver-services.md
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,12 @@ or
```toml
[remote_storage]
container_name = 'some-container-name'
storage_account = 'somestorageaccnt'
container_region = 'us-east'
prefix_in_container = '/test-prefix/'
```

`AZURE_STORAGE_ACCOUNT` and `AZURE_STORAGE_ACCESS_KEY` env variables can be used to specify the azure credentials if needed.
The `AZURE_STORAGE_ACCESS_KEY` env variable can be used to specify the azure credentials if needed.

## Repository background tasks

Expand Down
5 changes: 4 additions & 1 deletion libs/remote_storage/src/azure_blob.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,10 @@ impl AzureBlobStorage {
azure_config.container_name
);

let account = env::var("AZURE_STORAGE_ACCOUNT").expect("missing AZURE_STORAGE_ACCOUNT");
// Use the storage account from the config by default, fall back to env var if not present.
let account = azure_config.storage_account.clone().unwrap_or_else(|| {
env::var("AZURE_STORAGE_ACCOUNT").expect("missing AZURE_STORAGE_ACCOUNT")
});

// If the `AZURE_STORAGE_ACCESS_KEY` env var has an access key, use that,
// otherwise try the token based credentials.
Expand Down
17 changes: 15 additions & 2 deletions libs/remote_storage/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,11 @@ impl GenericRemoteStorage {
Self::AwsS3(Arc::new(S3Bucket::new(s3_config, timeout)?))
}
RemoteStorageKind::AzureContainer(azure_config) => {
info!("Using azure container '{}' in region '{}' as a remote storage, prefix in container: '{:?}'",
let storage_account = azure_config
.storage_account
.as_deref()
.unwrap_or("<AZURE_STORAGE_ACCOUNT>");
info!("Using azure container '{}' in account '{storage_account}' in region '{}' as a remote storage, prefix in container: '{:?}'",
azure_config.container_name, azure_config.container_region, azure_config.prefix_in_container);
Self::AzureBlob(Arc::new(AzureBlobStorage::new(azure_config, timeout)?))
}
Expand Down Expand Up @@ -589,6 +593,8 @@ impl Debug for S3Config {
pub struct AzureConfig {
/// Name of the container to connect to.
pub container_name: String,
/// Name of the storage account the container is inside of
pub storage_account: Option<String>,
/// The region where the bucket is located at.
pub container_region: String,
/// A "subfolder" in the container, to use the same container separately by multiple remote storage users at once.
Expand All @@ -603,8 +609,9 @@ impl Debug for AzureConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AzureConfig")
.field("bucket_name", &self.container_name)
.field("storage_account", &self.storage_account)
.field("bucket_region", &self.container_region)
.field("prefix_in_bucket", &self.prefix_in_container)
.field("prefix_in_container", &self.prefix_in_container)
.field("concurrency_limit", &self.concurrency_limit)
.field(
"max_keys_per_list_response",
Expand Down Expand Up @@ -718,6 +725,12 @@ impl RemoteStorageConfig {
(None, None, None, Some(container_name), Some(container_region)) => {
RemoteStorageKind::AzureContainer(AzureConfig {
container_name: parse_toml_string("container_name", container_name)?,
storage_account: toml
.get("storage_account")
.map(|storage_account| {
parse_toml_string("storage_account", storage_account)
})
.transpose()?,
container_region: parse_toml_string("container_region", container_region)?,
prefix_in_container: toml
.get("prefix_in_container")
Expand Down
1 change: 1 addition & 0 deletions libs/remote_storage/tests/test_real_azure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ fn create_azure_client(
let remote_storage_config = RemoteStorageConfig {
storage: RemoteStorageKind::AzureContainer(AzureConfig {
container_name: remote_storage_azure_container,
storage_account: None,
container_region: remote_storage_azure_region,
prefix_in_container: Some(format!("test_{millis}_{random:08x}/")),
concurrency_limit: NonZeroUsize::new(100).unwrap(),
Expand Down

1 comment on commit 68a2298

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

3310 tests run: 3183 passed, 1 failed, 126 skipped (full report)


Failures on Postgres 14

  • test_location_conf_churn[3]: debug
# Run all failed tests locally:
scripts/pytest -vv -n $(nproc) -k "test_location_conf_churn[debug-pg14-3]"
Flaky tests (1)

Postgres 14

  • test_pageserver_restarts_under_worload: release

Test coverage report is not available

The comment gets automatically updated with the latest test results
68a2298 at 2024-06-18T15:17:03.744Z :recycle:

Please sign in to comment.