Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Co-authored-by: Pooya Parsa <pooya@pi0.io>
- Loading branch information
1 parent
745f58a
commit 90ab690
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| --- | ||
| icon: simple-icons:amazons3 | ||
| --- | ||
|
|
||
| # S3 | ||
|
|
||
| > Store data to storage to S3-compatible providers. | ||
| S3 driver allows storing KV data to [Amazon S3](https://aws.amazon.com/s3/) or any other S3-compatible provider. | ||
|
|
||
| Driver implementation is lightweight and based on [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) working with Node.js as well as edge workers. | ||
|
|
||
| ## Setup | ||
|
|
||
| Setup a "Bucket" in your S3-compatible provider. You need this info: | ||
|
|
||
| - Access Key ID | ||
| - Secret Access Key | ||
| - Bucket name | ||
| - Endpoint | ||
| - Region | ||
|
|
||
| Make sure to install required peer dependencies: | ||
|
|
||
| :pm-install{name="aws4fetch"} | ||
|
|
||
| Then please make sure to set all driver's options: | ||
|
|
||
| ```ts | ||
| import { createStorage } from "unstorage"; | ||
| import s3Driver from "unstorage/drivers/s3"; | ||
|
|
||
| const storage = createStorage({ | ||
| driver: s3Driver({ | ||
| accessKeyId: "", // Access Key ID | ||
| secretAccessKey: "", // Secret Access Key | ||
| endpoint: "", | ||
| bucket: "", | ||
| region: "", | ||
| }), | ||
| }); | ||
| ``` | ||
|
|
||
| **Options:** | ||
|
|
||
| - `bulkDelete`: Enabled by default to speedup `clear()` operation. Set to `false` if provider is not implementing [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html). | ||
|
|
||
| ## Tested providers | ||
|
|
||
| Any S3-compatible provider should work out of the box. | ||
| Pull-Requests are more than welcome to add info about other any other tested provider. | ||
|
|
||
| ### Amazon S3 | ||
|
|
||
| :read-more{to="https://aws.amazon.com/s3/" title="Amazon S3"} | ||
|
|
||
| Options: | ||
|
|
||
| - Set `endpoint` to `https://s3.[region].amazonaws.com/` | ||
|
|
||
| ### Cloudflare R2 | ||
|
|
||
| :read-more{to="https://www.cloudflare.com/developer-platform/products/r2/" title="Cloudflare R2"} | ||
|
|
||
| Options: | ||
|
|
||
| - Set `endpoint` to `https://[uid].r2.cloudflarestorage.com/` | ||
| - Set `region` to `auto` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,237 @@ | ||
| import { | ||
| defineDriver, | ||
| createRequiredError, | ||
| normalizeKey, | ||
| createError, | ||
| } from "./utils"; | ||
| import { AwsClient } from "aws4fetch"; | ||
|
|
||
| export interface S3DriverOptions { | ||
| /** | ||
| * Access Key ID | ||
| */ | ||
| accessKeyId: string; | ||
|
|
||
| /** | ||
| * Secret Access Key | ||
| */ | ||
| secretAccessKey: string; | ||
|
|
||
| /** | ||
| * The endpoint URL of the S3 service. | ||
| * | ||
| * - For AWS S3: "https://s3.[region].amazonaws.com/" | ||
| * - For cloudflare R2: "https://[uid].r2.cloudflarestorage.com/" | ||
| */ | ||
| endpoint: string; | ||
|
|
||
| /** | ||
| * The region of the S3 bucket. | ||
| * | ||
| * - For AWS S3, this is the region of the bucket. | ||
| * - For cloudflare, this is can be set to `auto`. | ||
| */ | ||
| region: string; | ||
|
|
||
| /** | ||
| * The name of the bucket. | ||
| */ | ||
| bucket: string; | ||
|
|
||
| /** | ||
| * Enabled by default to speedup `clear()` operation. Set to `false` if provider is not implementing [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html). | ||
| */ | ||
| bulkDelete?: boolean; | ||
| } | ||
|
|
||
| const DRIVER_NAME = "s3"; | ||
|
|
||
| export default defineDriver((options: S3DriverOptions) => { | ||
| let _awsClient: AwsClient; | ||
| const getAwsClient = () => { | ||
| if (!_awsClient) { | ||
| if (!options.accessKeyId) { | ||
| throw createRequiredError(DRIVER_NAME, "accessKeyId"); | ||
| } | ||
| if (!options.secretAccessKey) { | ||
| throw createRequiredError(DRIVER_NAME, "secretAccessKey"); | ||
| } | ||
| if (!options.endpoint) { | ||
| throw createRequiredError(DRIVER_NAME, "endpoint"); | ||
| } | ||
| if (!options.region) { | ||
| throw createRequiredError(DRIVER_NAME, "region"); | ||
| } | ||
| _awsClient = new AwsClient({ | ||
| service: "s3", | ||
| accessKeyId: options.accessKeyId, | ||
| secretAccessKey: options.secretAccessKey, | ||
| region: options.region, | ||
| }); | ||
| } | ||
| return _awsClient; | ||
| }; | ||
|
|
||
| const baseURL = `${options.endpoint.replace(/\/$/, "")}/${options.bucket || ""}`; | ||
|
|
||
| const url = (key: string = "") => `${baseURL}/${normalizeKey(key)}`; | ||
|
|
||
| const awsFetch = async (url: string, opts?: RequestInit) => { | ||
| const request = await getAwsClient().sign(url, opts); | ||
| const res = await fetch(request); | ||
| if (!res.ok) { | ||
| if (res.status === 404) { | ||
| return null; | ||
| } | ||
| throw createError( | ||
| DRIVER_NAME, | ||
| `[${request.method}] ${url}: ${res.status} ${res.statusText} ${await res.text()}` | ||
| ); | ||
| } | ||
| return res; | ||
| }; | ||
|
|
||
| // https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html | ||
| const headObject = async (key: string) => { | ||
| const res = await awsFetch(url(key), { method: "HEAD" }); | ||
| if (!res) { | ||
| return null; | ||
| } | ||
| const metaHeaders: HeadersInit = {}; | ||
| for (const [key, value] of res.headers.entries()) { | ||
| const match = /x-amz-meta-(.*)/.exec(key); | ||
| if (match?.[1]) { | ||
| metaHeaders[match[1]] = value; | ||
| } | ||
| } | ||
| return metaHeaders; | ||
| }; | ||
|
|
||
| // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html | ||
| const listObjects = async (prefix?: string) => { | ||
| const res = await awsFetch(baseURL).then((r) => r?.text()); | ||
| if (!res) { | ||
| console.log("no list", prefix ? `${baseURL}?prefix=${prefix}` : baseURL); | ||
| return null; | ||
| } | ||
| return parseList(res); | ||
| }; | ||
|
|
||
| // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html | ||
| const getObject = (key: string) => { | ||
| return awsFetch(url(key)); | ||
| }; | ||
|
|
||
| // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html | ||
| const putObject = async (key: string, value: string) => { | ||
| return awsFetch(url(key), { | ||
| method: "PUT", | ||
| body: value, | ||
| }); | ||
| }; | ||
|
|
||
| // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html | ||
| const deleteObject = async (key: string) => { | ||
| return awsFetch(url(key), { method: "DELETE" }).then((r) => { | ||
| if (r?.status !== 204) { | ||
| throw createError(DRIVER_NAME, `Failed to delete ${key}`); | ||
| } | ||
| }); | ||
| }; | ||
|
|
||
| // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html | ||
| const deleteObjects = async (base: string) => { | ||
| const keys = await listObjects(base); | ||
| if (!keys?.length) { | ||
| return null; | ||
| } | ||
| if (options.bulkDelete === false) { | ||
| await Promise.all(keys.map((key) => deleteObject(key))); | ||
| } else { | ||
| const body = deleteKeysReq(keys); | ||
| await awsFetch(`${baseURL}?delete`, { | ||
| method: "POST", | ||
| headers: { | ||
| "x-amz-checksum-sha256": await sha256Base64(body), | ||
| }, | ||
| body, | ||
| }); | ||
| } | ||
| }; | ||
|
|
||
| return { | ||
| name: DRIVER_NAME, | ||
| options, | ||
| getItem(key) { | ||
| return getObject(key).then((res) => (res ? res.text() : null)); | ||
| }, | ||
| getItemRaw(key) { | ||
| return getObject(key).then((res) => (res ? res.arrayBuffer() : null)); | ||
| }, | ||
| async setItem(key, value) { | ||
| await putObject(key, value); | ||
| }, | ||
| async setItemRaw(key, value) { | ||
| await putObject(key, value); | ||
| }, | ||
| getMeta(key) { | ||
| return headObject(key); | ||
| }, | ||
| hasItem(key) { | ||
| return headObject(key).then((meta) => !!meta); | ||
| }, | ||
| getKeys(base) { | ||
| return listObjects(base).then((keys) => keys || []); | ||
| }, | ||
| async removeItem(key) { | ||
| await deleteObject(key); | ||
| }, | ||
| async clear(base) { | ||
| await deleteObjects(base); | ||
| }, | ||
| }; | ||
| }); | ||
|
|
||
| // --- utils --- | ||
|
|
||
| function deleteKeysReq(keys: string[]) { | ||
| return `<Delete>${keys | ||
| .map((key) => { | ||
| // prettier-ignore | ||
| key = key.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">") | ||
| return /* xml */ `<Object><Key>${key}</Key></Object>`; | ||
| }) | ||
| .join("")}</Delete>`; | ||
| } | ||
|
|
||
| async function sha256Base64(str: string) { | ||
| const buffer = new TextEncoder().encode(str); | ||
| const hash = await crypto.subtle.digest("SHA-256", buffer); | ||
| const bytes = new Uint8Array(hash); | ||
| const binaryString = String.fromCharCode(...bytes); // eslint-disable-line unicorn/prefer-code-point | ||
| return btoa(binaryString); | ||
| } | ||
|
|
||
| function parseList(xml: string) { | ||
| if (!xml.startsWith("<?xml")) { | ||
| throw new Error("Invalid XML"); | ||
| } | ||
| const listBucketResult = xml.match( | ||
| /<ListBucketResult[^>]*>([\s\S]*)<\/ListBucketResult>/ | ||
| )?.[1]; | ||
| if (!listBucketResult) { | ||
| throw new Error("Missing <ListBucketResult>"); | ||
| } | ||
| const contents = listBucketResult.match( | ||
| /<Contents[^>]*>([\s\S]*?)<\/Contents>/g | ||
| ); | ||
| if (!contents?.length) { | ||
| return []; | ||
| } | ||
| return contents | ||
| .map((content) => { | ||
| const key = content.match(/<Key>([\s\S]+?)<\/Key>/)?.[1]; | ||
| return key; | ||
| }) | ||
| .filter(Boolean) as string[]; | ||
| } | ||
Oops, something went wrong.