From 0f58c3fab2774eee0063af9e2f3fc4605bdaf99a Mon Sep 17 00:00:00 2001 From: MrBBot Date: Wed, 16 Nov 2022 09:39:03 +0000 Subject: [PATCH] Add API documentation and allow `string[]` for KV namespaces/R2 buckets (#432) * Allow `string[]` for `kvNamespaces`/`r2Buckets` like Miniflare 2 In this case, the binding name is assumed to be the same as the namespace ID/bucket name. * Add API documentation --- packages/miniflare/README.md | 435 +++++++++++++++++- packages/miniflare/src/index.ts | 2 +- packages/miniflare/src/plugins/kv/index.ts | 44 +- packages/miniflare/src/plugins/r2/index.ts | 41 +- .../miniflare/src/plugins/shared/index.ts | 12 + 5 files changed, 484 insertions(+), 50 deletions(-) diff --git a/packages/miniflare/README.md b/packages/miniflare/README.md index f1dcb7b4346ed..34369535b5381 100644 --- a/packages/miniflare/README.md +++ b/packages/miniflare/README.md @@ -1,4 +1,433 @@ -# `@miniflare/tre` +# 🔥 Miniflare -Experimental version of Miniflare powered by the soon-to-be open-sourced Workers -runtime... 👀 +**Miniflare 3** is a simulator for developing and testing +[**Cloudflare Workers**](https://workers.cloudflare.com/), powered by +[`workerd`](https://github.com/cloudflare/workerd). + +> :warning: Miniflare 3 is API-only, and does not expose a CLI. Use Wrangler +> with `wrangler dev --experimental-local` to develop your Workers locally with +> Miniflare 3. + +## Quick Start + +```shell +$ npm install @miniflare/tre --save-dev +``` + +```js +import { Miniflare } from "@miniflare/tre"; + +// Create a new Miniflare instance, starting a workerd server +const mf = new Miniflare({ + script: `addEventListener("fetch", (event) => { + event.respondWith(new Response("Hello Miniflare!")); + })`, +}); + +// Send a request to the workerd server, the host is ignored +const response = await mf.dispatchFetch("http://localhost:8787/"); +console.log(await response.text()); // Hello Miniflare! + +// Cleanup Miniflare, shutting down the workerd server +await mf.dispose(); +``` + +## API + +> :warning: Features marked **(Experimental)** may change at any point and are +> not subject to semantic versioning guarantees. + +### `type Awaitable` + +`T | Promise` + +Represents a value that can be `await`ed. Used in callback types to allow +`Promise`s to be returned if necessary. + +### `type Json` + +`string | number | boolean | null | Record | Json[]` + +Represents a JSON-serialisable value. + +### `type ModuleRuleType` + +`"ESModule" | "CommonJS" | "Text" | "Data" | "CompiledWasm"` + +Represents how a module's contents should be interpreted. + +- `"ESModule"`: interpret as + [ECMAScript module](https://tc39.es/ecma262/#sec-modules) +- `"CommonJS"`: interpret as + [CommonJS module](https://nodejs.org/api/modules.html#modules-commonjs-modules) +- `"Text"`: interpret as UTF8-encoded data, expose in runtime with + `string`-typed `default` export +- `"Data"`: interpret as arbitrary binary data, expose in runtime with + `ArrayBuffer`-typed `default` export +- `"CompiledWasm"`: interpret as binary WebAssembly module data, expose in + runtime with `WebAssembly.Module`-typed `default` export + +### `interface ModuleDefinition` + +Represents a manually defined module. + +- `type: ModuleRuleType` + + How this module's contents should be interpreted. + +- `path: string` + + Path of this module. The module's "name" will be obtained by converting this + to a relative path. The original path will be used to read `contents` if it's + omitted. + +- `contents?: string | Uint8Array` + + Contents override for this module. Binary data should be passed as + `Uint8Array`s. If omitted, will be read from `path`. + +### `interface ModuleRule` + +Represents a rule for identifying the `ModuleRuleType` of automatically located +modules. + +- `type: ModuleRuleType` + + How to interpret modules that match the `include` patterns. + +- `include: string[]` + + Glob patterns to match located module paths against (e.g. `["**/*.txt"]`). + +- `fallthrough?: boolean` + + If `true`, ignore any further rules of this `type`. This is useful for + disabling the built-in `ESModule` and `CommonJS` rules that match `*.mjs` and + `*.js`/`*.cjs` files respectively. + +### `type Persistence` + +`boolean | string | undefined` + +Represents where data should be persisted, if anywhere. + +- If this is `undefined` or `false`, data will be stored in-memory and only + persist between `Miniflare#setOptions()` calls, not restarts nor + `new Miniflare` instances. +- If this is `true`, data will be stored on the file-system, in the `$PWD/.mf` + directory. +- If this looks like a URL, then: + - If the protocol is `memory:`, data will be stored in-memory as above. + - If the protocol is `file:`, data will be stored on the file-system, in the + specified directory (e.g. `file:///path/to/directory`). If the `unsanitise` + search parameter is `true`, path sanitisation will be disabled. + - If the protocol is `sqlite:`, data will be stored in an SQLite database, at + the specified path (e.g. `sqlite:///path/to/db.sqlite`). + - **(Experimental)** If the protocol is `remote:`, data will be read/written + from/to real data stores on the Cloudflare network. By default, this will + cache data in-memory, but the `cache` search parameter can be set to a + URL-encoded persistence string to customise this. Note, this feature is only + supported for KV namespaces at the moment, and requires the + `cloudflareFetch` option to be set. +- Otherwise, if this is just a regular `string`, data will be stored on the + file-system, using the value as the directory path. + +### `interface WorkerOptions` + +Options for an individual Worker/"nanoservice". All bindings are accessible on +the global scope in service-worker format Workers, or via the 2nd `env` +parameter in module format Workers. + +#### Core + +- `name?: string` + + Unique name for this worker. Only required if multiple `workers` are + specified. + +- `script?: string` + + JavaScript code for this worker. If this is a service worker format Worker, it + must not have any imports. If this is a modules format Worker, it must not + have any _npm_ imports, and `modules: true` must be set. If it does have + imports, `scriptPath` must also be set so Miniflare knows where to resolve + them relative to. + +- `scriptPath?: string` + + Path of JavaScript entrypoint. If this is a service worker format Worker, it + must not have any imports. If this is a modules format Worker, it must not + have any _npm_ imports, and `modules: true` must be set. + +- `modules?: boolean | ModuleDefinition[]` + + - If `true`, Miniflare will treat `script`/`scriptPath` as an ES Module and + automatically locate transitive module dependencies according to + `modulesRules`. Note that automatic location is not perfect: if the + specifier to a dynamic `import()` or `require()` is not a string literal, an + exception will be thrown. + + - If set to an array, modules can be defined manually. Transitive dependencies + must also be defined. Note the first module must be the entrypoint and have + type `"ESModule"`. + + + + +- `modulesRules?: ModuleRule[]` + + Rules for identifying the `ModuleRuleType` of automatically located modules + when `modules: true` is set. Note the following default rules are always + included at the end: + + ```js + [ + { type: "ESModule", include: ["**/*.mjs"] }, + { type: "CommonJS", include: ["**/*.js", "**/*.cjs"] }, + ] + ``` + + > If `script` and `scriptPath` are set, and `modules` is set to an array, + > `modules` takes priority for a Worker's code, followed by `script`, then + > `scriptPath`. + + + +- `compatibilityDate?: string` + + [Compatibility date](https://developers.cloudflare.com/workers/platform/compatibility-dates/) + to use for this Worker. Defaults to a date far in the past. + +- `compatibilityFlags?: string[]` + + [Compatibility flags](https://developers.cloudflare.com/workers/platform/compatibility-dates/) + to use for this Worker. + +- `bindings?: Record` + + Record mapping binding name to arbitrary JSON-serialisable values to inject as + bindings into this Worker. + +- `wasmBindings?: Record` + + Record mapping binding name to paths containing binary WebAssembly module data + to inject as `WebAssembly.Module` bindings into this Worker. + +- `textBlobBindings?: Record` + + Record mapping binding name to paths containing UTF8-encoded data to inject as + `string` bindings into this Worker. + +- `dataBlobBindings?: Record` + + Record mapping binding name to paths containing arbitrary binary data to + inject as `ArrayBuffer` bindings into this Worker. + +- `serviceBindings?: Record Awaitable>` + + Record mapping binding name to service designators to inject as + `{ fetch: typeof fetch }` + [service bindings](https://developers.cloudflare.com/workers/platform/bindings/about-service-bindings/) + into this Worker. + + - If the designator is a `string`, requests will be dispatched to the Worker + with that `name`. + - If the designator is a function, requests will be dispatched to your custom + handler. This allows you to access data and functions defined in Node.js + from your Worker. + + +#### Cache + + + +- `cache?: boolean` + + _Not yet supported_, the Cache API is always enabled. + + + +- `cacheWarnUsage?: boolean` + + _Not yet supported_ + + + +#### Durable Objects + +- `durableObjects?: Record` + + Record mapping binding name to Durable Object class designators to inject as + `DurableObjectNamespace` bindings into this Worker. + + - If the designator is a `string`, it should be the name of a `class` exported + by this Worker. + - If the designator is an object, and `scriptName` is `undefined`, `className` + should be the name of a `class` exported by this Worker. + - Otherwise, `className` should be the name of a `class` exported by the + Worker with a `name` of `scriptName`. + +#### KV + +- `kvNamespaces?: Record | string[]` + + Record mapping binding name to KV namespace IDs to inject as `KVNamespace` + bindings into this Worker. Different Workers may bind to the same namespace ID + with different binding names. If a `string[]` of binding names is specified, + the binding name and KV namespace ID are assumed to be the same. + +- `sitePath?: string` + + Path to serve Workers Sites files from. If set, `__STATIC_CONTENT` and + `__STATIC_CONTENT_MANIFEST` bindings will be injected into this Worker. In + modules mode, `__STATIC_CONTENT_MANIFEST` will also be exposed as a module + with a `string`-typed `default` export, containing the JSON-stringified + manifest. Note Workers Sites files are never cached in Miniflare. + +- `siteInclude?: string[]` + + If set, only files with paths matching these glob patterns will be served. + +- `siteExclude?: string[]` + + If set, only files with paths _not_ matching these glob patterns will be + served. + +#### R2 + +- `r2Buckets?: Record | string[]` + + Record mapping binding name to R2 bucket names to inject as `R2Bucket` + bindings into this Worker. Different Workers may bind to the same bucket name + with different binding names. If a `string[]` of binding names is specified, + the binding name and bucket name are assumed to be the same. + +#### D1, Analytics Engine and Queues + +_Not yet supported_ + +### `interface SharedOptions` + +Options shared between all Workers/"nanoservices". + +#### Core + +- `host?: string` + + Hostname that the `workerd` server should listen on. Defaults to `127.0.0.1`. + +- `port?: number` + + Port that the `workerd` server should listen on. Tries to default to `8787`, + but falls back to a random free port if this is in use. Note if a manually + specified port is in use, Miniflare throws an error, rather than attempting to + find a free port. + +- `inspectorPort?: number` + + Port that `workerd` should start a DevTools inspector server on. Visit + `chrome://inspect` in a Chromium-based browser to connect to this. This can be + used to see detailed `console.log`s, profile CPU usage, and will eventually + allow step-through debugging. + +- `verbose?: boolean` + + Enable `workerd`'s `--verbose` flag for verbose logging. This can be used to + see simplified `console.log`s. + +- `cf?: boolean | string | Record` + + Controls the object returned from incoming `Request`'s `cf` property. + + - If set to a falsy value, an object with default placeholder values will be + used + - If set to an object, that object will be used + - If set to `true`, a real `cf` object will be fetched from a trusted + Cloudflare endpoint and cached in `node_modules/.mf` for 30 days + - If set to a `string`, a real `cf` object will be fetched and cached at the + provided path for 30 days + +- `liveReload?: boolean` + + If `true`, Miniflare will inject a script into HTML responses that + automatically reloads the page in-browser whenever the Miniflare instance's + options are updated. + +- **(Experimental)** + `cloudflareFetch?: (resource: string, searchParams?: URLSearchParams, init?: RequestInit) => Awaitable` + + Authenticated `fetch` used by `remote:` storage to communicate with the + Cloudflare API. `https://api.cloudflare.com/client/v4/accounts//` + should be prepended to `resource` to form the request URL. Appropriate + authorization headers should also be added. + + + +#### Cache, Durable Objects, KV and R2 + +- `cachePersist?: Persistence` + + Where to persist data cached in default or named caches. See docs for + `Persistence`. + +- `durableObjectsPersist?: Persistence` + + _Not yet supported_, Miniflare will throw if this is truthy and Durable Object + bindings are specified. + +- `kvPersist?: Persistence` + + Where to persist data stored in KV namespaces. See docs for `Persistence`. + +- `r2Persist?: Persistence` + + Where to persist data stored in R2 buckets. See docs for `Persistence`. + +#### D1, Analytics Engine and Queues + +_Not yet supported_ + +### `type MiniflareOptions` + +`SharedOptions & (WorkerOptions | workers: WorkerOptions[]))` + +Miniflare accepts either a single Worker configuration or multiple Worker +configurations in the `workers` array. When specifying an array of Workers, the +first Worker is designated the entrypoint and will receive all incoming HTTP +requests. Some options are shared between all workers and should always be +defined at the top-level. + +### `class Miniflare` + +- `constructor(opts: MiniflareOptions)` + + Creates a Miniflare instance and starts a new `workerd` server. Note unlike + Miniflare 2, Miniflare 3 _always_ starts a HTTP server listening on the + configured `host` and `port`: there are no `createServer`/`startServer` + functions. + +- `setOptions(opts: MiniflareOptions)` + + Updates the configuration for this Miniflare instance and restarts the + `workerd` server. Note unlike Miniflare 2, this does _not_ merge the new + configuration with the old configuration. + +- `ready: Promise` + + Returns a `Promise` that resolves with a `http` `URL` to the `workerd` server + once it has started and is able to accept requests. + +- `dispatchFetch(input: RequestInfo, init?: RequestInit): Promise` + + Sends a HTTP request to the `workerd` server, dispatching a `fetch` event in + the entrypoint Worker. Returns a `Promise` that resolves with the response. + Note that this implicitly waits for the `ready` `Promise` to resolve, there's + no need to do that yourself first. Additionally, the host of the request's URL + is always ignored and replaced with the `workerd` server's. + +- `dispose(): Promise` + + Cleans up the Miniflare instance, and shuts down the `workerd` server. Note + that after this is called, `setOptions` and `dispatchFetch` cannot be called. diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index 50de11238345d..af274f8aa9fb2 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -585,7 +585,7 @@ export class Miniflare { return fetch(url, forward as RequestInit); } - async dispose() { + async dispose(): Promise { this.#disposeController.abort(); try { await this.#initPromise; diff --git a/packages/miniflare/src/plugins/kv/index.ts b/packages/miniflare/src/plugins/kv/index.ts index 9485982eee7cd..fb466b50ad0f8 100644 --- a/packages/miniflare/src/plugins/kv/index.ts +++ b/packages/miniflare/src/plugins/kv/index.ts @@ -9,6 +9,7 @@ import { Plugin, SCRIPT_PLUGIN_NAMESPACE_PERSIST, encodePersist, + namespaceEntries, } from "../shared"; import { KV_PLUGIN_NAME } from "./constants"; import { KVGateway } from "./gateway"; @@ -17,8 +18,7 @@ import { KVRouter } from "./router"; import { SitesOptions, getSitesBindings, getSitesService } from "./sites"; export const KVOptionsSchema = z.object({ - // TODO: also allow array like Miniflare 2 - kvNamespaces: z.record(z.string()).optional(), + kvNamespaces: z.union([z.record(z.string()), z.string().array()]).optional(), // Workers Sites sitePath: z.string().optional(), @@ -48,9 +48,8 @@ export const KV_PLUGIN: Plugin< options: KVOptionsSchema, sharedOptions: KVSharedOptionsSchema, async getBindings(options) { - const bindings = Object.entries( - options.kvNamespaces ?? {} - ).map(([name, id]) => ({ + const namespaces = namespaceEntries(options.kvNamespaces); + const bindings = namespaces.map(([name, id]) => ({ name, kvNamespace: { name: `${SERVICE_NAMESPACE_PREFIX}:${id}` }, })); @@ -63,24 +62,23 @@ export const KV_PLUGIN: Plugin< }, getServices({ options, sharedOptions }) { const persistBinding = encodePersist(sharedOptions.kvPersist); - const services = Object.entries(options.kvNamespaces ?? []).map( - ([_, id]) => ({ - name: `${SERVICE_NAMESPACE_PREFIX}:${id}`, - worker: { - serviceWorkerScript: SCRIPT_PLUGIN_NAMESPACE_PERSIST, - compatibilityDate: "2022-09-01", - bindings: [ - ...persistBinding, - { name: BINDING_TEXT_PLUGIN, text: KV_PLUGIN_NAME }, - { name: BINDING_TEXT_NAMESPACE, text: id }, - { - name: BINDING_SERVICE_LOOPBACK, - service: { name: SERVICE_LOOPBACK }, - }, - ], - }, - }) - ); + const namespaces = namespaceEntries(options.kvNamespaces); + const services = namespaces.map(([_, id]) => ({ + name: `${SERVICE_NAMESPACE_PREFIX}:${id}`, + worker: { + serviceWorkerScript: SCRIPT_PLUGIN_NAMESPACE_PERSIST, + compatibilityDate: "2022-09-01", + bindings: [ + ...persistBinding, + { name: BINDING_TEXT_PLUGIN, text: KV_PLUGIN_NAME }, + { name: BINDING_TEXT_NAMESPACE, text: id }, + { + name: BINDING_SERVICE_LOOPBACK, + service: { name: SERVICE_LOOPBACK }, + }, + ], + }, + })); if (isWorkersSitesEnabled(options)) { services.push(getSitesService(options)); diff --git a/packages/miniflare/src/plugins/r2/index.ts b/packages/miniflare/src/plugins/r2/index.ts index 4340342f0a873..8ade7837cdbc1 100644 --- a/packages/miniflare/src/plugins/r2/index.ts +++ b/packages/miniflare/src/plugins/r2/index.ts @@ -9,12 +9,13 @@ import { Plugin, SCRIPT_PLUGIN_NAMESPACE_PERSIST, encodePersist, + namespaceEntries, } from "../shared"; import { R2Gateway } from "./gateway"; import { R2Router } from "./router"; export const R2OptionsSchema = z.object({ - r2Buckets: z.record(z.string()).optional(), + r2Buckets: z.union([z.record(z.string()), z.string().array()]).optional(), }); export const R2SharedOptionsSchema = z.object({ r2Persist: PersistenceSchema, @@ -31,14 +32,11 @@ export const R2_PLUGIN: Plugin< options: R2OptionsSchema, sharedOptions: R2SharedOptionsSchema, getBindings(options) { - const bindings = Object.entries( - options.r2Buckets ?? [] - ).map(([name, id]) => ({ + const buckets = namespaceEntries(options.r2Buckets); + return buckets.map(([name, id]) => ({ name, r2Bucket: { name: `${R2_PLUGIN_NAME}:${id}` }, })); - - return bindings; }, getServices({ options, sharedOptions }) { const persistBinding = encodePersist(sharedOptions.r2Persist); @@ -46,23 +44,20 @@ export const R2_PLUGIN: Plugin< name: BINDING_SERVICE_LOOPBACK, service: { name: SERVICE_LOOPBACK }, }; - const services = Object.entries(options.r2Buckets ?? []).map( - ([_, id]) => ({ - name: `${R2_PLUGIN_NAME}:${id}`, - worker: { - serviceWorkerScript: SCRIPT_PLUGIN_NAMESPACE_PERSIST, - bindings: [ - ...persistBinding, - { name: BINDING_TEXT_PLUGIN, text: R2_PLUGIN_NAME }, - { name: BINDING_TEXT_NAMESPACE, text: id }, - loopbackBinding, - ], - compatibilityDate: "2022-09-01", - }, - }) - ); - - return services; + const buckets = namespaceEntries(options.r2Buckets); + return buckets.map(([_, id]) => ({ + name: `${R2_PLUGIN_NAME}:${id}`, + worker: { + serviceWorkerScript: SCRIPT_PLUGIN_NAMESPACE_PERSIST, + bindings: [ + ...persistBinding, + { name: BINDING_TEXT_PLUGIN, text: R2_PLUGIN_NAME }, + { name: BINDING_TEXT_NAMESPACE, text: id }, + loopbackBinding, + ], + compatibilityDate: "2022-09-01", + }, + })); }, }; diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index 956887b18620f..2f7e364ba1ac3 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -47,6 +47,18 @@ export type Plugin< remoteStorage?: RemoteStorageConstructor; }); +export function namespaceEntries( + namespaces?: Record | string[] +): [bindingName: string, id: string][] { + if (Array.isArray(namespaces)) { + return namespaces.map(([bindingName]) => [bindingName, bindingName]); + } else if (namespaces !== undefined) { + return Object.entries(namespaces); + } else { + return []; + } +} + export * from "./constants"; export * from "./gateway"; export * from "./router";