diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index df43ae3bf76a..5ae9d2d9c83f 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -56,8 +56,6 @@ jobs: run: cargo clippy -- -D warnings - name: Run clippy with aws feature run: cargo clippy --features aws -- -D warnings - - name: Run clippy with aws_profile feature - run: cargo clippy --features aws_profile -- -D warnings - name: Run clippy with gcp feature run: cargo clippy --features gcp -- -D warnings - name: Run clippy with azure feature diff --git a/object_store/Cargo.toml b/object_store/Cargo.toml index c6b89fa23186..bd9c973e052a 100644 --- a/object_store/Cargo.toml +++ b/object_store/Cargo.toml @@ -53,11 +53,6 @@ reqwest = { version = "0.11", default-features = false, features = ["rustls-tls" ring = { version = "0.16", default-features = false, features = ["std"], optional = true } rustls-pemfile = { version = "1.0", default-features = false, optional = true } -# AWS Profile support -aws-types = { version = "0.55", optional = true } -aws-credential-types = { version = "0.55", optional = true } -aws-config = { version = "0.55", optional = true } - [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tokio = { version = "1.25.0", features = ["sync", "macros", "rt", "time", "io-util", "fs"] } @@ -74,9 +69,6 @@ gcp = ["cloud", "rustls-pemfile"] aws = ["cloud"] http = ["cloud"] -# Experimental support for AWS_PROFILE -aws_profile = ["aws", "aws-config", "aws-types", "aws-credential-types"] - [dev-dependencies] # In alphabetical order dotenv = "0.15.0" tempfile = "3.1.0" diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs index a10561ba613b..a7f43d1532ab 100644 --- a/object_store/src/aws/mod.rs +++ b/object_store/src/aws/mod.rs @@ -64,9 +64,6 @@ mod checksum; mod client; mod credential; -#[cfg(feature = "aws_profile")] -mod profile; - // http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html // // Do not URI-encode any of the unreserved characters that RFC 3986 defines: @@ -106,9 +103,6 @@ enum Error { #[snafu(display("Missing SecretAccessKey"))] MissingSecretAccessKey, - #[snafu(display("Profile support requires aws_profile feature"))] - MissingProfileFeature, - #[snafu(display("ETag Header missing from response"))] MissingEtag, @@ -427,8 +421,6 @@ pub struct AmazonS3Builder { checksum_algorithm: Option>, /// Metadata endpoint, see metadata_endpoint: Option, - /// Profile name, see - profile: Option, /// Client options client_options: ClientOptions, /// Credentials @@ -559,13 +551,6 @@ pub enum AmazonS3ConfigKey { /// - `metadata_endpoint` MetadataEndpoint, - /// AWS profile name - /// - /// Supported keys: - /// - `aws_profile` - /// - `profile` - Profile, - /// Client options Client(ClientConfigKey), } @@ -583,7 +568,6 @@ impl AsRef for AmazonS3ConfigKey { Self::VirtualHostedStyleRequest => "aws_virtual_hosted_style_request", Self::DefaultRegion => "aws_default_region", Self::MetadataEndpoint => "aws_metadata_endpoint", - Self::Profile => "aws_profile", Self::UnsignedPayload => "aws_unsigned_payload", Self::Checksum => "aws_checksum_algorithm", Self::Client(opt) => opt.as_ref(), @@ -612,7 +596,6 @@ impl FromStr for AmazonS3ConfigKey { "aws_virtual_hosted_style_request" | "virtual_hosted_style_request" => { Ok(Self::VirtualHostedStyleRequest) } - "aws_profile" | "profile" => Ok(Self::Profile), "aws_imdsv1_fallback" | "imdsv1_fallback" => Ok(Self::ImdsV1Fallback), "aws_metadata_endpoint" | "metadata_endpoint" => Ok(Self::MetadataEndpoint), "aws_unsigned_payload" | "unsigned_payload" => Ok(Self::UnsignedPayload), @@ -643,7 +626,6 @@ impl AmazonS3Builder { /// * `AWS_SESSION_TOKEN` -> token /// * `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` -> /// * `AWS_ALLOW_HTTP` -> set to "true" to permit HTTP connections without TLS - /// * `AWS_PROFILE` -> set profile name, requires `aws_profile` feature enabled /// # Example /// ``` /// use object_store::aws::AmazonS3Builder; @@ -727,7 +709,6 @@ impl AmazonS3Builder { AmazonS3ConfigKey::MetadataEndpoint => { self.metadata_endpoint = Some(value.into()) } - AmazonS3ConfigKey::Profile => self.profile = Some(value.into()), AmazonS3ConfigKey::UnsignedPayload => self.unsigned_payload.parse(value), AmazonS3ConfigKey::Checksum => { self.checksum_algorithm = Some(ConfigValue::Deferred(value.into())) @@ -794,7 +775,6 @@ impl AmazonS3Builder { Some(self.virtual_hosted_style_request.to_string()) } AmazonS3ConfigKey::MetadataEndpoint => self.metadata_endpoint.clone(), - AmazonS3ConfigKey::Profile => self.profile.clone(), AmazonS3ConfigKey::UnsignedPayload => Some(self.unsigned_payload.to_string()), AmazonS3ConfigKey::Checksum => { self.checksum_algorithm.as_ref().map(ToString::to_string) @@ -982,24 +962,6 @@ impl AmazonS3Builder { self } - /// Set the AWS profile name, see - /// - /// This makes use of [aws-config] to provide credentials and therefore requires - /// the `aws-profile` feature to be enabled - /// - /// It is strongly encouraged that users instead make use of a credential manager - /// such as [aws-vault] not only to avoid the significant additional dependencies, - /// but also to avoid storing credentials in [plain text on disk] - /// - /// [aws-config]: https://docs.rs/aws-config - /// [aws-vault]: https://github.com/99designs/aws-vault - /// [plain text on disk]: https://99designs.com.au/blog/engineering/aws-vault/ - #[cfg(feature = "aws_profile")] - pub fn with_profile(mut self, profile: impl Into) -> Self { - self.profile = Some(profile.into()); - self - } - /// Create a [`AmazonS3`] instance from the provided values, /// consuming `self`. pub fn build(mut self) -> Result { @@ -1007,14 +969,8 @@ impl AmazonS3Builder { self.parse_url(&url)?; } - let region = match (self.region, self.profile.clone()) { - (Some(region), _) => Some(region), - (None, Some(profile)) => profile_region(profile), - (None, None) => None, - }; - let bucket = self.bucket_name.context(MissingBucketNameSnafu)?; - let region = region.context(MissingRegionSnafu)?; + let region = self.region.context(MissingRegionSnafu)?; let checksum = self.checksum_algorithm.map(|x| x.get()).transpose()?; let credentials = if let Some(credentials) = self.credentials { @@ -1065,9 +1021,6 @@ impl AmazonS3Builder { client, self.retry_config.clone(), )) as _ - } else if let Some(profile) = self.profile { - info!("Using profile \"{}\" credential provider", profile); - profile_credentials(profile, region.clone())? } else { info!("Using Instance credential provider"); @@ -1123,37 +1076,6 @@ impl AmazonS3Builder { } } -#[cfg(feature = "aws_profile")] -fn profile_region(profile: String) -> Option { - use tokio::runtime::Handle; - - let handle = Handle::current(); - let provider = profile::ProfileProvider::new(profile, None); - - handle.block_on(provider.get_region()) -} - -#[cfg(feature = "aws_profile")] -fn profile_credentials(profile: String, region: String) -> Result { - Ok(Arc::new(profile::ProfileProvider::new( - profile, - Some(region), - ))) -} - -#[cfg(not(feature = "aws_profile"))] -fn profile_region(_profile: String) -> Option { - None -} - -#[cfg(not(feature = "aws_profile"))] -fn profile_credentials( - _profile: String, - _region: String, -) -> Result { - Err(Error::MissingProfileFeature.into()) -} - #[cfg(test)] mod tests { use super::*; @@ -1638,50 +1560,3 @@ mod s3_resolve_bucket_region_tests { assert!(result.is_err()); } } - -#[cfg(all(test, feature = "aws_profile"))] -mod profile_tests { - use super::*; - use std::env; - - use super::profile::{TEST_PROFILE_NAME, TEST_PROFILE_REGION}; - - #[tokio::test] - async fn s3_test_region_from_profile() { - let s3_url = "s3://bucket/prefix".to_owned(); - - let s3 = AmazonS3Builder::new() - .with_url(s3_url) - .with_profile(TEST_PROFILE_NAME) - .build() - .unwrap(); - - let region = &s3.client.config().region; - - assert_eq!(region, TEST_PROFILE_REGION); - } - - #[test] - fn s3_test_region_override() { - let s3_url = "s3://bucket/prefix".to_owned(); - - let aws_profile = - env::var("AWS_PROFILE").unwrap_or_else(|_| TEST_PROFILE_NAME.into()); - - let aws_region = - env::var("AWS_REGION").unwrap_or_else(|_| "object_store:fake_region".into()); - - env::set_var("AWS_PROFILE", aws_profile); - - let s3 = AmazonS3Builder::from_env() - .with_url(s3_url) - .with_region(aws_region.clone()) - .build() - .unwrap(); - - let actual = &s3.client.config().region; - let expected = &aws_region; - - assert_eq!(actual, expected); - } -} diff --git a/object_store/src/aws/profile.rs b/object_store/src/aws/profile.rs deleted file mode 100644 index 3fc08056444e..000000000000 --- a/object_store/src/aws/profile.rs +++ /dev/null @@ -1,133 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![cfg(feature = "aws_profile")] - -use async_trait::async_trait; -use aws_config::meta::region::ProvideRegion; -use aws_config::profile::profile_file::ProfileFiles; -use aws_config::profile::ProfileFileCredentialsProvider; -use aws_config::profile::ProfileFileRegionProvider; -use aws_config::provider_config::ProviderConfig; -use aws_credential_types::provider::ProvideCredentials; -use aws_types::region::Region; -use std::sync::Arc; -use std::time::Instant; -use std::time::SystemTime; - -use crate::aws::AwsCredential; -use crate::client::token::{TemporaryToken, TokenCache}; -use crate::client::CredentialProvider; -use crate::Result; - -#[cfg(test)] -pub static TEST_PROFILE_NAME: &str = "object_store:fake_profile"; - -#[cfg(test)] -pub static TEST_PROFILE_REGION: &str = "object_store:fake_region_from_profile"; - -#[derive(Debug)] -pub struct ProfileProvider { - name: String, - region: Option, - cache: TokenCache>, -} - -impl ProfileProvider { - pub fn new(name: String, region: Option) -> Self { - Self { - name, - region, - cache: Default::default(), - } - } - - #[cfg(test)] - fn profile_files(&self) -> ProfileFiles { - use aws_config::profile::profile_file::ProfileFileKind; - - let config = format!( - "[profile {}]\nregion = {}", - TEST_PROFILE_NAME, TEST_PROFILE_REGION - ); - - ProfileFiles::builder() - .with_contents(ProfileFileKind::Config, config) - .build() - } - - #[cfg(not(test))] - fn profile_files(&self) -> ProfileFiles { - ProfileFiles::default() - } - - pub async fn get_region(&self) -> Option { - if let Some(region) = self.region.clone() { - return Some(region); - } - - let provider = ProfileFileRegionProvider::builder() - .profile_files(self.profile_files()) - .profile_name(&self.name) - .build(); - - let region = provider.region().await; - - region.map(|r| r.as_ref().to_owned()) - } -} - -#[async_trait] -impl CredentialProvider for ProfileProvider { - type Credential = AwsCredential; - - async fn get_credential(&self) -> Result> { - self.cache - .get_or_insert_with(move || async move { - let region = self.region.clone().map(Region::new); - - let config = ProviderConfig::default().with_region(region); - - let credentials = ProfileFileCredentialsProvider::builder() - .configure(&config) - .profile_name(&self.name) - .build(); - - let c = credentials.provide_credentials().await.map_err(|source| { - crate::Error::Generic { - store: "S3", - source: Box::new(source), - } - })?; - let t_now = SystemTime::now(); - let expiry = c - .expiry() - .and_then(|e| e.duration_since(t_now).ok()) - .map(|ttl| Instant::now() + ttl); - - Ok(TemporaryToken { - token: Arc::new(AwsCredential { - key_id: c.access_key_id().to_string(), - secret_key: c.secret_access_key().to_string(), - token: c.session_token().map(ToString::to_string), - }), - expiry, - }) - }) - .await - } -}