feat: s3 driver

docs/2.drivers/s3.md

@@ -0,0 +1,68 @@
+---
+icon: simple-icons:amazons3
+---
+
+# S3
+
+> Store data to storage to S3-compatible providers.
+
+S3 driver allows storing KV data to [Amazon S3](https://aws.amazon.com/s3/) or any other S3-compatible provider.
+
+Driver implementation is lightweight and based on [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) working with Node.js as well as edge workers.
+
+## Setup
+
+Setup a "Bucket" in your S3-compatible provider. You need this info:
+
+- Access Key ID
+- Secret Access Key
+- Bucket name
+- Endpoint
+- Region
+
+Make sure to install required peer dependencies:
+
+:pm-install{name="aws4fetch"}
+
+Then please make sure to set all driver's options:
+
+```ts
+import { createStorage } from "unstorage";
+import s3Driver from "unstorage/drivers/s3";
+
+const storage = createStorage({
+  driver: s3Driver({
+    accessKeyId: "", // Access Key ID
+    secretAccessKey: "", // Secret Access Key
+    endpoint: "",
+    bucket: "",
+    region: "",
+  }),
+});
+```
+
+**Options:**
+
+- `bulkDelete`: Enabled by default to speedup `clear()` operation. Set to `false` if provider is not implementing [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html).
+
+## Tested providers
+
+Any S3-compatible provider should work out of the box.
+Pull-Requests are more than welcome to add info about other any other tested provider.
+
+### Amazon S3
+
+:read-more{to="https://aws.amazon.com/s3/" title="Amazon S3"}
+
+Options:
+
+- Set `endpoint` to `https://s3.[region].amazonaws.com/`
+
+### Cloudflare R2
+
+:read-more{to="https://www.cloudflare.com/developer-platform/products/r2/" title="Cloudflare R2"}
+
+Options:
+
+- Set `endpoint` to `https://[uid].r2.cloudflarestorage.com/`
+- Set `region` to `auto`

package.json

@@ -77,6 +77,7 @@
     "@vercel/blob": "^0.27.0",
     "@vercel/kv": "^3.0.0",
     "@vitest/coverage-v8": "^2.1.8",
+    "aws4fetch": "^1.0.20",
     "azurite": "^3.33.0",
     "better-sqlite3": "^11.7.0",
     "changelogen": "^0.5.7",
@@ -117,6 +118,7 @@
     "@upstash/redis": "^1.34.3",
     "@vercel/blob": ">=0.27.0",
     "@vercel/kv": "^1.0.1",
+    "aws4fetch": "^1.0.20",
     "db0": ">=0.2.1",
     "idb-keyval": "^6.2.1",
     "ioredis": "^5.4.1"
@@ -161,6 +163,9 @@
     "@vercel/kv": {
       "optional": true
     },
+    "aws4fetch": {
+      "optional": true
+    },
     "db0": {
       "optional": true
     },

src/_drivers.ts

@@ -25,12 +25,13 @@ import type { ExtraOptions as NetlifyBlobsOptions } from "unstorage/drivers/netl
 import type { OverlayStorageOptions as OverlayOptions } from "unstorage/drivers/overlay";
 import type { PlanetscaleDriverOptions as PlanetscaleOptions } from "unstorage/drivers/planetscale";
 import type { RedisOptions as RedisOptions } from "unstorage/drivers/redis";
+import type { S3DriverOptions as S3Options } from "unstorage/drivers/s3";
 import type { SessionStorageOptions as SessionStorageOptions } from "unstorage/drivers/session-storage";
 import type { UpstashOptions as UpstashOptions } from "unstorage/drivers/upstash";
 import type { VercelBlobOptions as VercelBlobOptions } from "unstorage/drivers/vercel-blob";
 import type { VercelKVOptions as VercelKVOptions } from "unstorage/drivers/vercel-kv";
 
-export type BuiltinDriverName = "azure-app-configuration" | "azureAppConfiguration" | "azure-cosmos" | "azureCosmos" | "azure-key-vault" | "azureKeyVault" | "azure-storage-blob" | "azureStorageBlob" | "azure-storage-table" | "azureStorageTable" | "capacitor-preferences" | "capacitorPreferences" | "cloudflare-kv-binding" | "cloudflareKVBinding" | "cloudflare-kv-http" | "cloudflareKVHttp" | "cloudflare-r2-binding" | "cloudflareR2Binding" | "db0" | "deno-kv-node" | "denoKVNode" | "deno-kv" | "denoKV" | "fs-lite" | "fsLite" | "fs" | "github" | "http" | "indexedb" | "localstorage" | "lru-cache" | "lruCache" | "memory" | "mongodb" | "netlify-blobs" | "netlifyBlobs" | "null" | "overlay" | "planetscale" | "redis" | "session-storage" | "sessionStorage" | "upstash" | "vercel-blob" | "vercelBlob" | "vercel-kv" | "vercelKV";
+export type BuiltinDriverName = "azure-app-configuration" | "azureAppConfiguration" | "azure-cosmos" | "azureCosmos" | "azure-key-vault" | "azureKeyVault" | "azure-storage-blob" | "azureStorageBlob" | "azure-storage-table" | "azureStorageTable" | "capacitor-preferences" | "capacitorPreferences" | "cloudflare-kv-binding" | "cloudflareKVBinding" | "cloudflare-kv-http" | "cloudflareKVHttp" | "cloudflare-r2-binding" | "cloudflareR2Binding" | "db0" | "deno-kv-node" | "denoKVNode" | "deno-kv" | "denoKV" | "fs-lite" | "fsLite" | "fs" | "github" | "http" | "indexedb" | "localstorage" | "lru-cache" | "lruCache" | "memory" | "mongodb" | "netlify-blobs" | "netlifyBlobs" | "null" | "overlay" | "planetscale" | "redis" | "s3" | "session-storage" | "sessionStorage" | "upstash" | "vercel-blob" | "vercelBlob" | "vercel-kv" | "vercelKV";
 
 export type BuiltinDriverOptions = {
   "azure-app-configuration": AzureAppConfigurationOptions;
@@ -71,6 +72,7 @@ export type BuiltinDriverOptions = {
   "overlay": OverlayOptions;
   "planetscale": PlanetscaleOptions;
   "redis": RedisOptions;
+  "s3": S3Options;
   "session-storage": SessionStorageOptions;
   "sessionStorage": SessionStorageOptions;
   "upstash": UpstashOptions;
@@ -121,6 +123,7 @@ export const builtinDrivers = {
   "overlay": "unstorage/drivers/overlay",
   "planetscale": "unstorage/drivers/planetscale",
   "redis": "unstorage/drivers/redis",
+  "s3": "unstorage/drivers/s3",
   "session-storage": "unstorage/drivers/session-storage",
   "sessionStorage": "unstorage/drivers/session-storage",
   "upstash": "unstorage/drivers/upstash",

src/drivers/s3.ts

@@ -0,0 +1,237 @@
+import {
+  defineDriver,
+  createRequiredError,
+  normalizeKey,
+  createError,
+} from "./utils";
+import { AwsClient } from "aws4fetch";
+
+export interface S3DriverOptions {
+  /**
+   * Access Key ID
+   */
+  accessKeyId: string;
+
+  /**
+   * Secret Access Key
+   */
+  secretAccessKey: string;
+
+  /**
+   * The endpoint URL of the S3 service.
+   *
+   * - For AWS S3: "https://s3.[region].amazonaws.com/"
+   * - For cloudflare R2: "https://[uid].r2.cloudflarestorage.com/"
+   */
+  endpoint: string;
+
+  /**
+   * The region of the S3 bucket.
+   *
+   * - For AWS S3, this is the region of the bucket.
+   * - For cloudflare, this is can be set to `auto`.
+   */
+  region: string;
+
+  /**
+   * The name of the bucket.
+   */
+  bucket: string;
+
+  /**
+   * Enabled by default to speedup `clear()` operation. Set to `false` if provider is not implementing [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html).
+   */
+  bulkDelete?: boolean;
+}
+
+const DRIVER_NAME = "s3";
+
+export default defineDriver((options: S3DriverOptions) => {
+  let _awsClient: AwsClient;
+  const getAwsClient = () => {
+    if (!_awsClient) {
+      if (!options.accessKeyId) {
+        throw createRequiredError(DRIVER_NAME, "accessKeyId");
+      }
+      if (!options.secretAccessKey) {
+        throw createRequiredError(DRIVER_NAME, "secretAccessKey");
+      }
+      if (!options.endpoint) {
+        throw createRequiredError(DRIVER_NAME, "endpoint");
+      }
+      if (!options.region) {
+        throw createRequiredError(DRIVER_NAME, "region");
+      }
+      _awsClient = new AwsClient({
+        service: "s3",
+        accessKeyId: options.accessKeyId,
+        secretAccessKey: options.secretAccessKey,
+        region: options.region,
+      });
+    }
+    return _awsClient;
+  };
+
+  const baseURL = `${options.endpoint.replace(/\/$/, "")}/${options.bucket || ""}`;
+
+  const url = (key: string = "") => `${baseURL}/${normalizeKey(key)}`;
+
+  const awsFetch = async (url: string, opts?: RequestInit) => {
+    const request = await getAwsClient().sign(url, opts);
+    const res = await fetch(request);
+    if (!res.ok) {
+      if (res.status === 404) {
+        return null;
+      }
+      throw createError(
+        DRIVER_NAME,
+        `[${request.method}] ${url}: ${res.status} ${res.statusText} ${await res.text()}`
+      );
+    }
+    return res;
+  };
+
+  // https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html
+  const headObject = async (key: string) => {
+    const res = await awsFetch(url(key), { method: "HEAD" });
+    if (!res) {
+      return null;
+    }
+    const metaHeaders: HeadersInit = {};
+    for (const [key, value] of res.headers.entries()) {
+      const match = /x-amz-meta-(.*)/.exec(key);
+      if (match?.[1]) {
+        metaHeaders[match[1]] = value;
+      }
+    }
+    return metaHeaders;
+  };
+
+  // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
+  const listObjects = async (prefix?: string) => {
+    const res = await awsFetch(baseURL).then((r) => r?.text());
+    if (!res) {
+      console.log("no list", prefix ? `${baseURL}?prefix=${prefix}` : baseURL);
+      return null;
+    }
+    return parseList(res);
+  };
+
+  // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
+  const getObject = (key: string) => {
+    return awsFetch(url(key));
+  };
+
+  // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
+  const putObject = async (key: string, value: string) => {
+    return awsFetch(url(key), {
+      method: "PUT",
+      body: value,
+    });
+  };
+
+  // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html
+  const deleteObject = async (key: string) => {
+    return awsFetch(url(key), { method: "DELETE" }).then((r) => {
+      if (r?.status !== 204) {
+        throw createError(DRIVER_NAME, `Failed to delete ${key}`);
+      }
+    });
+  };
+
+  // https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
+  const deleteObjects = async (base: string) => {
+    const keys = await listObjects(base);
+    if (!keys?.length) {
+      return null;
+    }
+    if (options.bulkDelete === false) {
+      await Promise.all(keys.map((key) => deleteObject(key)));
+    } else {
+      const body = deleteKeysReq(keys);
+      await awsFetch(`${baseURL}?delete`, {
+        method: "POST",
+        headers: {
+          "x-amz-checksum-sha256": await sha256Base64(body),
+        },
+        body,
+      });
+    }
+  };
+
+  return {
+    name: DRIVER_NAME,
+    options,
+    getItem(key) {
+      return getObject(key).then((res) => (res ? res.text() : null));
+    },
+    getItemRaw(key) {
+      return getObject(key).then((res) => (res ? res.arrayBuffer() : null));
+    },
+    async setItem(key, value) {
+      await putObject(key, value);
+    },
+    async setItemRaw(key, value) {
+      await putObject(key, value);
+    },
+    getMeta(key) {
+      return headObject(key);
+    },
+    hasItem(key) {
+      return headObject(key).then((meta) => !!meta);
+    },
+    getKeys(base) {
+      return listObjects(base).then((keys) => keys || []);
+    },
+    async removeItem(key) {
+      await deleteObject(key);
+    },
+    async clear(base) {
+      await deleteObjects(base);
+    },
+  };
+});
+
+// --- utils ---
+
+function deleteKeysReq(keys: string[]) {
+  return `<Delete>${keys
+    .map((key) => {
+      // prettier-ignore
+      key = key.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
+      return /* xml */ `<Object><Key>${key}</Key></Object>`;
+    })
+    .join("")}</Delete>`;
+}
+
+async function sha256Base64(str: string) {
+  const buffer = new TextEncoder().encode(str);
+  const hash = await crypto.subtle.digest("SHA-256", buffer);
+  const bytes = new Uint8Array(hash);
+  const binaryString = String.fromCharCode(...bytes); // eslint-disable-line unicorn/prefer-code-point
+  return btoa(binaryString);
+}
+
+function parseList(xml: string) {
+  if (!xml.startsWith("<?xml")) {
+    throw new Error("Invalid XML");
+  }
+  const listBucketResult = xml.match(
+    /<ListBucketResult[^>]*>([\s\S]*)<\/ListBucketResult>/
+  )?.[1];
+  if (!listBucketResult) {
+    throw new Error("Missing <ListBucketResult>");
+  }
+  const contents = listBucketResult.match(
+    /<Contents[^>]*>([\s\S]*?)<\/Contents>/g
+  );
+  if (!contents?.length) {
+    return [];
+  }
+  return contents
+    .map((content) => {
+      const key = content.match(/<Key>([\s\S]+?)<\/Key>/)?.[1];
+      return key;
+    })
+    .filter(Boolean) as string[];
+}