Merge pull request #6623 from Xuanwo/allow-read-public

feat: Allow COPY and CREATE STAGE from public read buckets without credentials
databendlabs · Jul 14, 2022 · df68a1d · df68a1d · vercel · Jul 14, 2022
2 parents 95cf4a1 + 5132080
commit df68a1d
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 39 deletions.
diff --git a/.github/actions/check/action.yml b/.github/actions/check/action.yml
@@ -28,7 +28,7 @@ runs:
 
     - name: Clippy
       shell: bash
-      run: cargo -Z sparse-registry clippy --workspace --all-targets -- -D warnings
+      run: cargo -Z sparse-registry clippy --workspace --all-targets --all-features -- -D warnings
 
     - name: Audit dependencies
       shell: bash

diff --git a/common/storage/src/location.rs b/common/storage/src/location.rs
@@ -63,23 +63,9 @@ pub fn parse_uri_location(l: &UriLocation) -> Result<(StorageParams, String)> {
             account_name: l
                 .connection
                 .get("account_name")
-                .ok_or_else(|| {
-                    Error::new(
-                        ErrorKind::InvalidInput,
-                        anyhow!("account_name is required for storage azblob"),
-                    )
-                })?
-                .to_string(),
-            account_key: l
-                .connection
-                .get("account_key")
-                .ok_or_else(|| {
-                    Error::new(
-                        ErrorKind::InvalidInput,
-                        anyhow!("account_name is required for storage azblob"),
-                    )
-                })?
-                .to_string(),
+                .cloned()
+                .unwrap_or_default(),
+            account_key: l.connection.get("account_key").cloned().unwrap_or_default(),
             root: root.to_string(),
         }),
         #[cfg(feature = "storage-hdfs")]
@@ -108,24 +94,14 @@ pub fn parse_uri_location(l: &UriLocation) -> Result<(StorageParams, String)> {
                 .connection
                 .get("access_key_id")
                 .or_else(|| l.connection.get("aws_key_id"))
-                .ok_or_else(|| {
-                    Error::new(
-                        ErrorKind::InvalidInput,
-                        anyhow!("access_key_id is required for storage s3"),
-                    )
-                })?
-                .to_string(),
+                .cloned()
+                .unwrap_or_default(),
             secret_access_key: l
                 .connection
                 .get("secret_access_key")
                 .or_else(|| l.connection.get("aws_secret_key"))
-                .ok_or_else(|| {
-                    Error::new(
-                        ErrorKind::InvalidInput,
-                        anyhow!("secret_access_key is required for storage s3"),
-                    )
-                })?
-                .to_string(),
+                .cloned()
+                .unwrap_or_default(),
             master_key: l.connection.get("master_key").cloned().unwrap_or_default(),
             root: root.to_string(),
             disable_credential_loader: true,

diff --git a/query/src/catalogs/hive/hive_table.rs b/query/src/catalogs/hive/hive_table.rs
@@ -26,8 +26,6 @@ use common_planners::ReadDataSourcePlan;
 use common_planners::Statistics;
 use common_planners::TruncateTablePlan;
 use common_streams::SendableDataBlockStream;
-use common_tracing::tracing_futures::Instrument;
-use futures::StreamExt;
 use futures::TryStreamExt;
 use opendal::ObjectMode;
 
@@ -38,7 +36,6 @@ use crate::pipelines::processors::port::OutputPort;
 use crate::pipelines::processors::processor::ProcessorPtr;
 use crate::pipelines::processors::SyncSource;
 use crate::pipelines::processors::SyncSourcer;
-use crate::pipelines::Pipe;
 use crate::pipelines::Pipeline;
 use crate::pipelines::SourcePipeBuilder;
 use crate::sessions::QueryContext;
@@ -270,6 +267,7 @@ struct HiveSource {
 }
 
 impl HiveSource {
+    #[allow(dead_code)]
     pub fn create(
         ctx: Arc<QueryContext>,
         output: Arc<OutputPort>,

diff --git a/query/src/storages/hive/hive_parquet_block_reader.rs b/query/src/storages/hive/hive_parquet_block_reader.rs
@@ -35,12 +35,8 @@ use common_exception::ErrorCode;
 use common_exception::Result;
 use common_planners::PartInfoPtr;
 use common_tracing::tracing;
-use common_tracing::tracing::debug_span;
 use common_tracing::tracing::warn;
-use common_tracing::tracing::Instrument;
 use futures::AsyncReadExt;
-use futures::StreamExt;
-use futures::TryStreamExt;
 use opendal::Object;
 use opendal::Operator;