diff --git a/docs/2.drivers/s3.md b/docs/2.drivers/s3.md new file mode 100644 index 00000000..e2947150 --- /dev/null +++ b/docs/2.drivers/s3.md @@ -0,0 +1,68 @@ +--- +icon: simple-icons:amazons3 +--- + +# S3 + +> Store data to storage to S3-compatible providers. + +S3 driver allows storing KV data to [Amazon S3](https://aws.amazon.com/s3/) or any other S3-compatible provider. + +Driver implementation is lightweight and based on [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) working with Node.js as well as edge workers. + +## Setup + +Setup a "Bucket" in your S3-compatible provider. You need this info: + +- Access Key ID +- Secret Access Key +- Bucket name +- Endpoint +- Region + +Make sure to install required peer dependencies: + +:pm-install{name="aws4fetch"} + +Then please make sure to set all driver's options: + +```ts +import { createStorage } from "unstorage"; +import s3Driver from "unstorage/drivers/s3"; + +const storage = createStorage({ + driver: s3Driver({ + accessKeyId: "", // Access Key ID + secretAccessKey: "", // Secret Access Key + endpoint: "", + bucket: "", + region: "", + }), +}); +``` + +**Options:** + +- `bulkDelete`: Enabled by default to speedup `clear()` operation. Set to `false` if provider is not implementing [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html). + +## Tested providers + +Any S3-compatible provider should work out of the box. +Pull-Requests are more than welcome to add info about other any other tested provider. + +### Amazon S3 + +:read-more{to="https://aws.amazon.com/s3/" title="Amazon S3"} + +Options: + +- Set `endpoint` to `https://s3.[region].amazonaws.com/` + +### Cloudflare R2 + +:read-more{to="https://www.cloudflare.com/developer-platform/products/r2/" title="Cloudflare R2"} + +Options: + +- Set `endpoint` to `https://[uid].r2.cloudflarestorage.com/` +- Set `region` to `auto` diff --git a/package.json b/package.json index bfb393b3..fa30bc07 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "@vercel/blob": "^0.27.0", "@vercel/kv": "^3.0.0", "@vitest/coverage-v8": "^2.1.8", + "aws4fetch": "^1.0.20", "azurite": "^3.33.0", "better-sqlite3": "^11.7.0", "changelogen": "^0.5.7", @@ -117,6 +118,7 @@ "@upstash/redis": "^1.34.3", "@vercel/blob": ">=0.27.0", "@vercel/kv": "^1.0.1", + "aws4fetch": "^1.0.20", "db0": ">=0.2.1", "idb-keyval": "^6.2.1", "ioredis": "^5.4.1" @@ -161,6 +163,9 @@ "@vercel/kv": { "optional": true }, + "aws4fetch": { + "optional": true + }, "db0": { "optional": true }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2b89b330..d6b75336 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -102,6 +102,9 @@ importers: '@vitest/coverage-v8': specifier: ^2.1.8 version: 2.1.8(vitest@2.1.8(@types/node@22.10.2)(jsdom@25.0.1)) + aws4fetch: + specifier: ^1.0.20 + version: 1.0.20 azurite: specifier: ^3.33.0 version: 3.33.0 @@ -1542,6 +1545,9 @@ packages: resolution: {integrity: sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==} engines: {node: '>= 6.0.0'} + aws4fetch@1.0.20: + resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} + axios@0.21.4: resolution: {integrity: sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==} @@ -5980,6 +5986,8 @@ snapshots: aws-ssl-profiles@1.1.2: {} + aws4fetch@1.0.20: {} + axios@0.21.4: dependencies: follow-redirects: 1.15.9(debug@4.4.0) diff --git a/src/_drivers.ts b/src/_drivers.ts index b6a05496..ada0fbb0 100644 --- a/src/_drivers.ts +++ b/src/_drivers.ts @@ -25,12 +25,13 @@ import type { ExtraOptions as NetlifyBlobsOptions } from "unstorage/drivers/netl import type { OverlayStorageOptions as OverlayOptions } from "unstorage/drivers/overlay"; import type { PlanetscaleDriverOptions as PlanetscaleOptions } from "unstorage/drivers/planetscale"; import type { RedisOptions as RedisOptions } from "unstorage/drivers/redis"; +import type { S3DriverOptions as S3Options } from "unstorage/drivers/s3"; import type { SessionStorageOptions as SessionStorageOptions } from "unstorage/drivers/session-storage"; import type { UpstashOptions as UpstashOptions } from "unstorage/drivers/upstash"; import type { VercelBlobOptions as VercelBlobOptions } from "unstorage/drivers/vercel-blob"; import type { VercelKVOptions as VercelKVOptions } from "unstorage/drivers/vercel-kv"; -export type BuiltinDriverName = "azure-app-configuration" | "azureAppConfiguration" | "azure-cosmos" | "azureCosmos" | "azure-key-vault" | "azureKeyVault" | "azure-storage-blob" | "azureStorageBlob" | "azure-storage-table" | "azureStorageTable" | "capacitor-preferences" | "capacitorPreferences" | "cloudflare-kv-binding" | "cloudflareKVBinding" | "cloudflare-kv-http" | "cloudflareKVHttp" | "cloudflare-r2-binding" | "cloudflareR2Binding" | "db0" | "deno-kv-node" | "denoKVNode" | "deno-kv" | "denoKV" | "fs-lite" | "fsLite" | "fs" | "github" | "http" | "indexedb" | "localstorage" | "lru-cache" | "lruCache" | "memory" | "mongodb" | "netlify-blobs" | "netlifyBlobs" | "null" | "overlay" | "planetscale" | "redis" | "session-storage" | "sessionStorage" | "upstash" | "vercel-blob" | "vercelBlob" | "vercel-kv" | "vercelKV"; +export type BuiltinDriverName = "azure-app-configuration" | "azureAppConfiguration" | "azure-cosmos" | "azureCosmos" | "azure-key-vault" | "azureKeyVault" | "azure-storage-blob" | "azureStorageBlob" | "azure-storage-table" | "azureStorageTable" | "capacitor-preferences" | "capacitorPreferences" | "cloudflare-kv-binding" | "cloudflareKVBinding" | "cloudflare-kv-http" | "cloudflareKVHttp" | "cloudflare-r2-binding" | "cloudflareR2Binding" | "db0" | "deno-kv-node" | "denoKVNode" | "deno-kv" | "denoKV" | "fs-lite" | "fsLite" | "fs" | "github" | "http" | "indexedb" | "localstorage" | "lru-cache" | "lruCache" | "memory" | "mongodb" | "netlify-blobs" | "netlifyBlobs" | "null" | "overlay" | "planetscale" | "redis" | "s3" | "session-storage" | "sessionStorage" | "upstash" | "vercel-blob" | "vercelBlob" | "vercel-kv" | "vercelKV"; export type BuiltinDriverOptions = { "azure-app-configuration": AzureAppConfigurationOptions; @@ -71,6 +72,7 @@ export type BuiltinDriverOptions = { "overlay": OverlayOptions; "planetscale": PlanetscaleOptions; "redis": RedisOptions; + "s3": S3Options; "session-storage": SessionStorageOptions; "sessionStorage": SessionStorageOptions; "upstash": UpstashOptions; @@ -121,6 +123,7 @@ export const builtinDrivers = { "overlay": "unstorage/drivers/overlay", "planetscale": "unstorage/drivers/planetscale", "redis": "unstorage/drivers/redis", + "s3": "unstorage/drivers/s3", "session-storage": "unstorage/drivers/session-storage", "sessionStorage": "unstorage/drivers/session-storage", "upstash": "unstorage/drivers/upstash", diff --git a/src/drivers/s3.ts b/src/drivers/s3.ts new file mode 100644 index 00000000..30798b11 --- /dev/null +++ b/src/drivers/s3.ts @@ -0,0 +1,237 @@ +import { + defineDriver, + createRequiredError, + normalizeKey, + createError, +} from "./utils"; +import { AwsClient } from "aws4fetch"; + +export interface S3DriverOptions { + /** + * Access Key ID + */ + accessKeyId: string; + + /** + * Secret Access Key + */ + secretAccessKey: string; + + /** + * The endpoint URL of the S3 service. + * + * - For AWS S3: "https://s3.[region].amazonaws.com/" + * - For cloudflare R2: "https://[uid].r2.cloudflarestorage.com/" + */ + endpoint: string; + + /** + * The region of the S3 bucket. + * + * - For AWS S3, this is the region of the bucket. + * - For cloudflare, this is can be set to `auto`. + */ + region: string; + + /** + * The name of the bucket. + */ + bucket: string; + + /** + * Enabled by default to speedup `clear()` operation. Set to `false` if provider is not implementing [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html). + */ + bulkDelete?: boolean; +} + +const DRIVER_NAME = "s3"; + +export default defineDriver((options: S3DriverOptions) => { + let _awsClient: AwsClient; + const getAwsClient = () => { + if (!_awsClient) { + if (!options.accessKeyId) { + throw createRequiredError(DRIVER_NAME, "accessKeyId"); + } + if (!options.secretAccessKey) { + throw createRequiredError(DRIVER_NAME, "secretAccessKey"); + } + if (!options.endpoint) { + throw createRequiredError(DRIVER_NAME, "endpoint"); + } + if (!options.region) { + throw createRequiredError(DRIVER_NAME, "region"); + } + _awsClient = new AwsClient({ + service: "s3", + accessKeyId: options.accessKeyId, + secretAccessKey: options.secretAccessKey, + region: options.region, + }); + } + return _awsClient; + }; + + const baseURL = `${options.endpoint.replace(/\/$/, "")}/${options.bucket || ""}`; + + const url = (key: string = "") => `${baseURL}/${normalizeKey(key)}`; + + const awsFetch = async (url: string, opts?: RequestInit) => { + const request = await getAwsClient().sign(url, opts); + const res = await fetch(request); + if (!res.ok) { + if (res.status === 404) { + return null; + } + throw createError( + DRIVER_NAME, + `[${request.method}] ${url}: ${res.status} ${res.statusText} ${await res.text()}` + ); + } + return res; + }; + + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html + const headObject = async (key: string) => { + const res = await awsFetch(url(key), { method: "HEAD" }); + if (!res) { + return null; + } + const metaHeaders: HeadersInit = {}; + for (const [key, value] of res.headers.entries()) { + const match = /x-amz-meta-(.*)/.exec(key); + if (match?.[1]) { + metaHeaders[match[1]] = value; + } + } + return metaHeaders; + }; + + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html + const listObjects = async (prefix?: string) => { + const res = await awsFetch(baseURL).then((r) => r?.text()); + if (!res) { + console.log("no list", prefix ? `${baseURL}?prefix=${prefix}` : baseURL); + return null; + } + return parseList(res); + }; + + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html + const getObject = (key: string) => { + return awsFetch(url(key)); + }; + + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html + const putObject = async (key: string, value: string) => { + return awsFetch(url(key), { + method: "PUT", + body: value, + }); + }; + + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html + const deleteObject = async (key: string) => { + return awsFetch(url(key), { method: "DELETE" }).then((r) => { + if (r?.status !== 204) { + throw createError(DRIVER_NAME, `Failed to delete ${key}`); + } + }); + }; + + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html + const deleteObjects = async (base: string) => { + const keys = await listObjects(base); + if (!keys?.length) { + return null; + } + if (options.bulkDelete === false) { + await Promise.all(keys.map((key) => deleteObject(key))); + } else { + const body = deleteKeysReq(keys); + await awsFetch(`${baseURL}?delete`, { + method: "POST", + headers: { + "x-amz-checksum-sha256": await sha256Base64(body), + }, + body, + }); + } + }; + + return { + name: DRIVER_NAME, + options, + getItem(key) { + return getObject(key).then((res) => (res ? res.text() : null)); + }, + getItemRaw(key) { + return getObject(key).then((res) => (res ? res.arrayBuffer() : null)); + }, + async setItem(key, value) { + await putObject(key, value); + }, + async setItemRaw(key, value) { + await putObject(key, value); + }, + getMeta(key) { + return headObject(key); + }, + hasItem(key) { + return headObject(key).then((meta) => !!meta); + }, + getKeys(base) { + return listObjects(base).then((keys) => keys || []); + }, + async removeItem(key) { + await deleteObject(key); + }, + async clear(base) { + await deleteObjects(base); + }, + }; +}); + +// --- utils --- + +function deleteKeysReq(keys: string[]) { + return `${keys + .map((key) => { + // prettier-ignore + key = key.replace(/&/g, "&").replace(//g, ">") + return /* xml */ `${key}`; + }) + .join("")}`; +} + +async function sha256Base64(str: string) { + const buffer = new TextEncoder().encode(str); + const hash = await crypto.subtle.digest("SHA-256", buffer); + const bytes = new Uint8Array(hash); + const binaryString = String.fromCharCode(...bytes); // eslint-disable-line unicorn/prefer-code-point + return btoa(binaryString); +} + +function parseList(xml: string) { + if (!xml.startsWith("]*>([\s\S]*)<\/ListBucketResult>/ + )?.[1]; + if (!listBucketResult) { + throw new Error("Missing "); + } + const contents = listBucketResult.match( + /]*>([\s\S]*?)<\/Contents>/g + ); + if (!contents?.length) { + return []; + } + return contents + .map((content) => { + const key = content.match(/([\s\S]+?)<\/Key>/)?.[1]; + return key; + }) + .filter(Boolean) as string[]; +} diff --git a/test/drivers/s3.test.ts b/test/drivers/s3.test.ts new file mode 100644 index 00000000..aa0ca4f9 --- /dev/null +++ b/test/drivers/s3.test.ts @@ -0,0 +1,15 @@ +import { describe } from "vitest"; +import s3Driver from "../../src/drivers/s3"; +import { testDriver } from "./utils"; + +describe("drivers: s3", () => { + testDriver({ + driver: s3Driver({ + accessKeyId: process.env.VITE_S3_ACCESS_KEY_ID!, + secretAccessKey: process.env.VITE_S3_SECRET_ACCESS_KEY!, + bucket: process.env.VITE_S3_BUCKET!, + endpoint: process.env.VITE_S3_ENDPOINT!, + region: process.env.VITE_S3_REGION!, + }), + }); +}); diff --git a/vite.config.ts b/vite.config.ts index 7fda8351..7752d104 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { testTimeout: 10_000, - retry: 3, + retry: process.env.CI ? 2 : undefined, typecheck: { enabled: true, },