diff --git a/core/src/services/azblob/docs.md b/core/src/services/azblob/docs.md index 8ad55f19c663..e02eeabe7f7a 100644 --- a/core/src/services/azblob/docs.md +++ b/core/src/services/azblob/docs.md @@ -1,4 +1,4 @@ -# Capabilities +## Capabilities This service can be used to: @@ -15,7 +15,7 @@ This service can be used to: - [x] presign - [ ] blocking -# Configuration +## Configuration - `root`: Set the work dir for backend. - `container`: Set the container name for backend. @@ -25,18 +25,18 @@ This service can be used to: Refer to public API docs for more information. -# Examples +## Examples This example works on [Azurite](https://github.com/Azure/Azurite) for local developments. -## Start local blob service +### Start local blob service ```shell docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite az storage container create --name test --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" ``` -## Init OpenDAL Operator +### Init OpenDAL Operator ### Via Builder diff --git a/core/src/services/azdfs/docs.md b/core/src/services/azdfs/docs.md index 3dbf9f60d957..23b283d1c47c 100644 --- a/core/src/services/azdfs/docs.md +++ b/core/src/services/azdfs/docs.md @@ -2,7 +2,7 @@ As known as `abfs`, `azdfs` or `azdls`. This service will visist the [ABFS](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-abfs-driver) URI supported by [Azure Data Lake Storage Gen2](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction). -# Capabilities +## Capabilities This service can be used to: @@ -18,7 +18,7 @@ This service can be used to: - [ ] presign - [ ] blocking -# Configuration +## Configuration - `root`: Set the work dir for backend. - `filesystem`: Set the filesystem name for backend. @@ -28,9 +28,7 @@ This service can be used to: Refer to public API docs for more information. -# Example - -## Init OpenDAL Operator +## Example ### Via Builder @@ -69,4 +67,4 @@ async fn main() -> Result<()> { Ok(()) } -``` \ No newline at end of file +``` diff --git a/core/src/services/cos/docs.md b/core/src/services/cos/docs.md index f52e168a882f..6d5ee73b1539 100644 --- a/core/src/services/cos/docs.md +++ b/core/src/services/cos/docs.md @@ -1,4 +1,4 @@ -# Capabilities +## Capabilities This service can be used to: @@ -14,7 +14,7 @@ This service can be used to: - [ ] presign - [ ] blocking -# Configuration +## Configuration - `root`: Set the work directory for backend - `bucket`: Set the container name for backend @@ -24,9 +24,9 @@ This service can be used to: You can refer to [`CosBuilder`]'s docs for more information -# Example +## Example -## Via Builder +### Via Builder ```rust use anyhow::Result; @@ -54,4 +54,4 @@ async fn main() -> Result<()> { Ok(()) } -``` \ No newline at end of file +``` diff --git a/core/src/services/dashmap/docs.md b/core/src/services/dashmap/docs.md index 6cabad8173b6..89546b9c9f51 100644 --- a/core/src/services/dashmap/docs.md +++ b/core/src/services/dashmap/docs.md @@ -1,4 +1,4 @@ -# Capabilities +## Capabilities This service can be used to: @@ -12,4 +12,4 @@ This service can be used to: - [ ] list - [x] scan - [ ] presign -- [ ] blocking \ No newline at end of file +- [ ] blocking diff --git a/core/src/services/fs/docs.md b/core/src/services/fs/docs.md index f47d36be1fb8..0d5517c6315d 100644 --- a/core/src/services/fs/docs.md +++ b/core/src/services/fs/docs.md @@ -1,4 +1,4 @@ -# Capabilities +## Capabilities This service can be used to: @@ -15,13 +15,13 @@ This service can be used to: - [ ] ~~presign~~ - [x] blocking -# Configuration +## Configuration - `root`: Set the work dir for backend. Refer to public API docs for more information. -# Example +## Example ### Via Builder diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 6d18b8f65517..99ec8bfab904 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -55,232 +55,11 @@ static ENDPOINT_TEMPLATES: Lazy> = Lazy::new }); const DEFAULT_WRITE_MIN_SIZE: usize = 8 * 1024 * 1024; + /// Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. /// For more information about s3-compatible services, refer to [Compatible Services](#compatible-services). /// -/// # Capabilities -/// -/// This service can be used to: -/// -/// - [x] stat -/// - [x] read -/// - [x] write -/// - [x] create_dir -/// - [x] delete -/// - [x] copy -/// - [ ] rename -/// - [x] list -/// - [x] scan -/// - [x] presign -/// - [ ] blocking -/// -/// # Configuration -/// -/// - `root`: Set the work dir for backend. -/// - `bucket`: Set the container name for backend. -/// - `endpoint`: Set the endpoint for backend. -/// - `region`: Set the region for backend. -/// - `access_key_id`: Set the access_key_id for backend. -/// - `secret_access_key`: Set the secret_access_key for backend. -/// - `security_token`: Set the security_token for backend. -/// - `default_storage_class`: Set the default storage_class for backend. -/// - `server_side_encryption`: Set the server_side_encryption for backend. -/// - `server_side_encryption_aws_kms_key_id`: Set the server_side_encryption_aws_kms_key_id for backend. -/// - `server_side_encryption_customer_algorithm`: Set the server_side_encryption_customer_algorithm for backend. -/// - `server_side_encryption_customer_key`: Set the server_side_encryption_customer_key for backend. -/// - `server_side_encryption_customer_key_md5`: Set the server_side_encryption_customer_key_md5 for backend. -/// - `disable_config_load`: Disable aws config load from env -/// - `enable_virtual_host_style`: Enable virtual host style. -/// -/// Refer to [`S3Builder`]'s public API docs for more information. -/// -/// # Temporary security credentials -/// -/// OpenDAL now provides support for S3 temporary security credentials in IAM. -/// -/// The way to take advantage of this feature is to build your S3 backend with `Builder::security_token`. -/// -/// But OpenDAL will not refresh the temporary security credentials, please keep in mind to refresh those credentials in time. -/// -/// # Server Side Encryption -/// -/// OpenDAL provides full support of S3 Server Side Encryption(SSE) features. -/// -/// The easiest way to configure them is to use helper functions like -/// -/// - SSE-KMS: `server_side_encryption_with_aws_managed_kms_key` -/// - SSE-KMS: `server_side_encryption_with_customer_managed_kms_key` -/// - SSE-S3: `server_side_encryption_with_s3_key` -/// - SSE-C: `server_side_encryption_with_customer_key` -/// -/// If those functions don't fulfill need, low-level options are also provided: -/// -/// - Use service managed kms key -/// - `server_side_encryption="aws:kms"` -/// - Use customer provided kms key -/// - `server_side_encryption="aws:kms"` -/// - `server_side_encryption_aws_kms_key_id="your-kms-key"` -/// - Use S3 managed key -/// - `server_side_encryption="AES256"` -/// - Use customer key -/// - `server_side_encryption_customer_algorithm="AES256"` -/// - `server_side_encryption_customer_key="base64-of-your-aes256-key"` -/// - `server_side_encryption_customer_key_md5="base64-of-your-aes256-key-md5"` -/// -/// After SSE have been configured, all requests send by this backed will attach those headers. -/// -/// Reference: [Protecting data using server-side encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/serv-side-encryption.html) -/// -/// # Example -/// -/// ## Basic Setup -/// -/// ```no_run -/// use std::sync::Arc; -/// -/// use anyhow::Result; -/// use opendal::services::S3; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// // Create s3 backend builder. -/// let mut builder = S3::default(); -/// // Set the root for s3, all operations will happen under this root. -/// // -/// // NOTE: the root must be absolute path. -/// builder.root("/path/to/dir"); -/// // Set the bucket name, this is required. -/// builder.bucket("test"); -/// // Set the endpoint. -/// // -/// // For examples: -/// // - "https://s3.amazonaws.com" -/// // - "http://127.0.0.1:9000" -/// // - "https://oss-ap-northeast-1.aliyuncs.com" -/// // - "https://cos.ap-seoul.myqcloud.com" -/// // -/// // Default to "https://s3.amazonaws.com" -/// builder.endpoint("https://s3.amazonaws.com"); -/// // Set the access_key_id and secret_access_key. -/// // -/// // OpenDAL will try load credential from the env. -/// // If credential not set and no valid credential in env, OpenDAL will -/// // send request without signing like anonymous user. -/// builder.access_key_id("access_key_id"); -/// builder.secret_access_key("secret_access_key"); -/// -/// let op: Operator = Operator::new(builder)?.finish(); -/// -/// Ok(()) -/// } -/// ``` -/// -/// ## S3 with SSE-C -/// -/// ```no_run -/// use anyhow::Result; -/// use log::info; -/// use opendal::services::S3; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = S3::default(); -/// -/// // Setup builders -/// -/// // Enable SSE-C -/// builder.server_side_encryption_with_customer_key("AES256", "customer_key".as_bytes()); -/// -/// let op = Operator::new(builder)?.finish(); -/// info!("operator: {:?}", op); -/// -/// // Writing your testing code here. -/// -/// Ok(()) -/// } -/// ``` -/// -/// ## S3 with SSE-KMS and aws managed kms key -/// -/// ```no_run -/// use anyhow::Result; -/// use log::info; -/// use opendal::services::S3; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = S3::default(); -/// -/// // Setup builders -/// -/// // Enable SSE-KMS with aws managed kms key -/// builder.server_side_encryption_with_aws_managed_kms_key(); -/// -/// let op = Operator::new(builder)?.finish(); -/// info!("operator: {:?}", op); -/// -/// // Writing your testing code here. -/// -/// Ok(()) -/// } -/// ``` -/// -/// ## S3 with SSE-KMS and customer managed kms key -/// -/// ```no_run -/// use anyhow::Result; -/// use log::info; -/// use opendal::services::S3; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = S3::default(); -/// -/// // Setup builders -/// -/// // Enable SSE-KMS with customer managed kms key -/// builder.server_side_encryption_with_customer_managed_kms_key("aws_kms_key_id"); -/// -/// let op = Operator::new(builder)?.finish(); -/// info!("operator: {:?}", op); -/// -/// // Writing your testing code here. -/// -/// Ok(()) -/// } -/// ``` -/// -/// ## S3 with SSE-S3 -/// -/// ```no_run -/// use anyhow::Result; -/// use log::info; -/// use opendal::services::S3; -/// use opendal::Operator; -/// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = S3::default(); -/// -/// // Setup builders -/// -/// // Enable SSE-S3 -/// builder.server_side_encryption_with_s3_key(); -/// -/// let op = Operator::new(builder)?.finish(); -/// info!("operator: {:?}", op); -/// -/// // Writing your testing code here. -/// -/// Ok(()) -/// } -/// ``` -/// -/// # Compatible Services +#[doc = include_str!("docs.md")] #[doc = include_str!("compatible_services.md")] #[derive(Default)] pub struct S3Builder { diff --git a/core/src/services/s3/compatible_services.md b/core/src/services/s3/compatible_services.md index b2dd144fbb26..432ce7c892c0 100644 --- a/core/src/services/s3/compatible_services.md +++ b/core/src/services/s3/compatible_services.md @@ -1,4 +1,7 @@ -## AWS S3 + +## Compatible Services + +### AWS S3 [AWS S3](https://aws.amazon.com/s3/) is the default implementations of s3 services. Only `bucket` is required. @@ -6,7 +9,7 @@ builder.bucket(""); ``` -## Alibaba Object Storage Service (OSS) +### Alibaba Object Storage Service (OSS) [OSS](https://www.alibabacloud.com/product/object-storage-service) is a s3 compatible service provided by [Alibaba Cloud](https://www.alibabacloud.com). @@ -38,7 +41,7 @@ builder.bucket(""); builder.enable_virtual_host_style(); ``` -## Minio +### Minio [minio](https://min.io/) is an open-source s3 compatible services. @@ -54,7 +57,7 @@ builder.region(""); builder.bucket(""); ``` -## QingStor Object Storage +### QingStor Object Storage [QingStor Object Storage](https://www.qingcloud.com/products/qingstor) is a S3-compatible service provided by [QingCloud](https://www.qingcloud.com/). @@ -63,7 +66,7 @@ To connect to QingStor Object Storage, we need to set: - `endpoint`: The endpoint of QingStor s3 compatible endpoint, for example: `https://s3.pek3b.qingstor.com` - `bucket`: The bucket name. -## Scaleway Object Storage +### Scaleway Object Storage [Scaleway Object Storage](https://www.scaleway.com/en/object-storage/) is a S3-compatible and multi-AZ redundant object storage service. @@ -73,7 +76,7 @@ To connect to Scaleway Object Storage, we need to set: - `region`: The region of scaleway. - `bucket`: The bucket name of scaleway. -## Tencent Cloud Object Storage (COS) +### Tencent Cloud Object Storage (COS) [COS](https://intl.cloud.tencent.com/products/cos) is a s3 compatible service provided by [Tencent Cloud](https://intl.cloud.tencent.com/). @@ -82,7 +85,7 @@ To connect to COS, we need to set: - `endpoint`: The endpoint of cos, for example: `https://cos.ap-beijing.myqcloud.com` - `bucket`: The bucket name of cos. -## Wasabi Object Storage +### Wasabi Object Storage [Wasabi](https://wasabi.com/) is a s3 compatible service. diff --git a/core/src/services/s3/docs.md b/core/src/services/s3/docs.md new file mode 100644 index 000000000000..bbd8d8e7ccc9 --- /dev/null +++ b/core/src/services/s3/docs.md @@ -0,0 +1,251 @@ +## Capabilities + +This service can be used to: + +- [x] stat +- [x] read +- [x] write +- [x] create_dir +- [x] delete +- [x] copy +- [ ] rename +- [x] list +- [x] scan +- [x] presign +- [ ] blocking + +## Configuration + +- `root`: Set the work dir for backend. +- `bucket`: Set the container name for backend. +- `endpoint`: Set the endpoint for backend. +- `region`: Set the region for backend. +- `access_key_id`: Set the access_key_id for backend. +- `secret_access_key`: Set the secret_access_key for backend. +- `security_token`: Set the security_token for backend. +- `default_storage_class`: Set the default storage_class for backend. +- `server_side_encryption`: Set the server_side_encryption for backend. +- `server_side_encryption_aws_kms_key_id`: Set the server_side_encryption_aws_kms_key_id for backend. +- `server_side_encryption_customer_algorithm`: Set the server_side_encryption_customer_algorithm for backend. +- `server_side_encryption_customer_key`: Set the server_side_encryption_customer_key for backend. +- `server_side_encryption_customer_key_md5`: Set the server_side_encryption_customer_key_md5 for backend. +- `disable_config_load`: Disable aws config load from env +- `enable_virtual_host_style`: Enable virtual host style. + +Refer to [`S3Builder`]'s public API docs for more information. + +## Temporary security credentials + +OpenDAL now provides support for S3 temporary security credentials in IAM. + +The way to take advantage of this feature is to build your S3 backend with `Builder::security_token`. + +But OpenDAL will not refresh the temporary security credentials, please keep in mind to refresh those credentials in time. + +## Server Side Encryption + +OpenDAL provides full support of S3 Server Side Encryption(SSE) features. + +The easiest way to configure them is to use helper functions like + +- SSE-KMS: `server_side_encryption_with_aws_managed_kms_key` +- SSE-KMS: `server_side_encryption_with_customer_managed_kms_key` +- SSE-S3: `server_side_encryption_with_s3_key` +- SSE-C: `server_side_encryption_with_customer_key` + +If those functions don't fulfill need, low-level options are also provided: + +- Use service managed kms key + - `server_side_encryption="aws:kms"` +- Use customer provided kms key + - `server_side_encryption="aws:kms"` + - `server_side_encryption_aws_kms_key_id="your-kms-key"` +- Use S3 managed key + - `server_side_encryption="AES256"` +- Use customer key + - `server_side_encryption_customer_algorithm="AES256"` + - `server_side_encryption_customer_key="base64-of-your-aes256-key"` + - `server_side_encryption_customer_key_md5="base64-of-your-aes256-key-md5"` + +After SSE have been configured, all requests send by this backed will attach those headers. + +Reference: [Protecting data using server-side encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/serv-side-encryption.html) + +## Example + +## Via Builder + +### Basic Setup + +```rust +use std::sync::Arc; + +use anyhow::Result; +use opendal::services::S3; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + // Create s3 backend builder. + let mut builder = S3::default(); + // Set the root for s3, all operations will happen under this root. + // + // NOTE: the root must be absolute path. + builder.root("/path/to/dir"); + // Set the bucket name. This is required. + builder.bucket("test"); + // Set the region. This is only required for aws s3. + builder.region("us-east-1"); + // Set the endpoint. + // + // For examples: + // - "https://s3.amazonaws.com" + // - "http://127.0.0.1:9000" + // - "https://oss-ap-northeast-1.aliyuncs.com" + // - "https://cos.ap-seoul.myqcloud.com" + // + // Default to "https://s3.amazonaws.com" + builder.endpoint("https://s3.amazonaws.com"); + // Set the access_key_id and secret_access_key. + // + // OpenDAL will try load credential from the env. + // If credential not set and no valid credential in env, OpenDAL will + // send request without signing like anonymous user. + builder.access_key_id("access_key_id"); + builder.secret_access_key("secret_access_key"); + + let op: Operator = Operator::new(builder)?.finish(); + + Ok(()) +} + +``` + +### S3 with SSE-C + +```rust +use anyhow::Result; +use log::info; +use opendal::services::S3; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = S3::default(); + + // Setup builders + builder.root("/path/to/dir"); + builder.bucket("test"); + builder.region("us-east-1"); + builder.endpoint("https://s3.amazonaws.com"); + builder.access_key_id("access_key_id"); + builder.secret_access_key("secret_access_key"); + + // Enable SSE-C + builder.server_side_encryption_with_customer_key("AES256", "customer_key".as_bytes()); + + let op = Operator::new(builder)?.finish(); + info!("operator: {:?}", op); + + // Writing your testing code here. + + Ok(()) +} +``` + +### S3 with SSE-KMS and aws managed kms key + +```rust +use anyhow::Result; +use log::info; +use opendal::services::S3; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = S3::default(); + + // Setup builders + builder.root("/path/to/dir"); + builder.bucket("test"); + builder.region("us-east-1"); + builder.endpoint("https://s3.amazonaws.com"); + builder.access_key_id("access_key_id"); + builder.secret_access_key("secret_access_key"); + + // Enable SSE-KMS with aws managed kms key + builder.server_side_encryption_with_aws_managed_kms_key(); + + let op = Operator::new(builder)?.finish(); + info!("operator: {:?}", op); + + // Writing your testing code here. + + Ok(()) +} +``` + +### S3 with SSE-KMS and customer managed kms key + +```rust +use anyhow::Result; +use log::info; +use opendal::services::S3; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = S3::default(); + + // Setup builders + builder.root("/path/to/dir"); + builder.bucket("test"); + builder.region("us-east-1"); + builder.endpoint("https://s3.amazonaws.com"); + builder.access_key_id("access_key_id"); + builder.secret_access_key("secret_access_key"); + + // Enable SSE-KMS with customer managed kms key + builder.server_side_encryption_with_customer_managed_kms_key("aws_kms_key_id"); + + let op = Operator::new(builder)?.finish(); + info!("operator: {:?}", op); + + // Writing your testing code here. + + Ok(()) +} +``` + +### S3 with SSE-S3 + +```rust +use anyhow::Result; +use log::info; +use opendal::services::S3; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = S3::default(); + + // Setup builders + builder.root("/path/to/dir"); + builder.bucket("test"); + builder.region("us-east-1"); + builder.endpoint("https://s3.amazonaws.com"); + builder.access_key_id("access_key_id"); + builder.secret_access_key("secret_access_key"); + + // Enable SSE-S3 + builder.server_side_encryption_with_s3_key(); + + let op = Operator::new(builder)?.finish(); + info!("operator: {:?}", op); + + // Writing your testing code here. + + Ok(()) +} +``` + diff --git a/website/docs/services/s3.mdx b/website/docs/services/s3.mdx new file mode 100644 index 000000000000..36f06da89f2a --- /dev/null +++ b/website/docs/services/s3.mdx @@ -0,0 +1,85 @@ +--- +title: S3 +--- + +Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. + +For more information about s3-compatible services, refer to [Compatible Services](#compatible-services). + +import Docs from '../../../core/src/services/s3/docs.md' + + + +## Via Config + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + +```rust +use std::sync::Arc; + +use anyhow::Result; +use opendal::Scheme; +use opendal::Operator; + +#[tokio::main] +async fn main() -> Result<()> { + let mut map = HashMap::new(); + + map.insert("root".to_string(), "/path/to/dir".to_string()); + map.insert("bucket".to_string(), "test".to_string()); + map.insert("region".to_string(), "us-east-1".to_string()); + map.insert("endpoint".to_string(), "https://s3.amazonaws.com".to_string()); + map.insert("access_key_id".to_string(), "access_key_id".to_string()); + map.insert("secret_access_key".to_string(), "secret_access_key".to_string()); + + let op: Operator = Operator::via_map(Scheme::S3, map)?; + + Ok(()) +} +``` + + + + +```javascript +import { Operator } from "opendal"; + +async function main() { + const op = new Operator("azblob", { + root: "/path/to/dir", + bucket: "test", + region: "us-east-1", + endpoint: "https://s3.amazonaws.com", + access_key_id: "access_key_id", + secret_access_key: "secret_access_key", + }); +} +``` + + + + +```python +import opendal + +op = opendal.Operator("azblob", + root="/path/to/dir", + bucket="test", + region="us-east-1", + endpoint="https://s3.amazonaws.com", + access_key_id="access_key_id", + secret_access_key="secret_access_key", +) +``` + + + + + +import CompatibleServices from '../../../core/src/services/s3/compatible_services.md' + +