diff --git a/apps/tests/aws-runtime/test/test-service.ts b/apps/tests/aws-runtime/test/test-service.ts index 9736267a8..637c7d9f2 100644 --- a/apps/tests/aws-runtime/test/test-service.ts +++ b/apps/tests/aws-runtime/test/test-service.ts @@ -547,7 +547,25 @@ export const createAndDestroyWorkflow = workflow( } ); -export const counter = entity<{ n: number }>("counter2", z.any()); +export const counter = entity("counter4", { + attributes: { + n: z.number(), + namespace: z.union([z.literal("different"), z.literal("default")]), + id: z.string(), + }, + partition: ["namespace", "id"], +}); + +export const counterCollection = entity("counter-collection", { + attributes: { + id: z.string(), + counterNumber: z.number(), + n: z.number(), + }, + partition: ["id"], + sort: ["counterNumber"], +}); + const entityEvent = event<{ id: string }>("entityEvent"); const entitySignal = signal("entitySignal"); const entitySignal2 = signal<{ n: number }>("entitySignal2"); @@ -560,19 +578,25 @@ export const counterWatcher = counter.stream( // TODO: compute the possible operations union from the operations array if (item.operation === "remove") { const { n } = item.oldValue!; - await entitySignal2.sendSignal(item.key, { n: n + 1 }); + await entitySignal2.sendSignal(item.key.id, { n: n + 1 }); } } ); export const counterNamespaceWatcher = counter.stream( "counterNamespaceWatch", - { namespacePrefixes: ["different"] }, + { queryKeys: [{ namespace: "different" }] }, async (item) => { + console.log(item); if (item.operation === "insert") { const value = await counter.get(item.key); - await counter.set(item.key, { n: (value?.n ?? 0) + 1 }); - await entitySignal.sendSignal(item.key); + await counter.set({ + namespace: "default", + id: value!.id, + n: (value?.n ?? 0) + 1, + }); + console.log("send signal to", value!.id); + await entitySignal.sendSignal(value!.id); } } ); @@ -581,8 +605,8 @@ export const onEntityEvent = subscription( "onEntityEvent", { events: [entityEvent] }, async ({ id }) => { - const value = await counter.get(id); - await counter.set(id, { n: (value?.n ?? 0) + 1 }); + const value = await counter.get({ namespace: "default", id }); + await counter.set({ namespace: "default", id, n: (value?.n ?? 0) + 1 }); await entitySignal.sendSignal(id); } ); @@ -590,51 +614,82 @@ export const onEntityEvent = subscription( export const entityTask = task( "entityAct", async (_, { execution: { id } }) => { - const value = await counter.get(id); - await counter.set(id, { n: (value?.n ?? 0) + 1 }); + const value = await counter.get(["default", id]); + await counter.set({ namespace: "default", id, n: (value?.n ?? 0) + 1 }); } ); export const entityWorkflow = workflow( "entityWorkflow", async (_, { execution: { id } }) => { - await counter.set(id, { n: 1 }); - counter.set({ key: id, namespace: "different!" }, { n: 0 }); + await counter.set({ namespace: "default", id, n: 1 }); + await counter.set({ namespace: "different", id, n: 1 }); await entitySignal.expectSignal(); await entityTask(); await Promise.all([entityEvent.emit({ id }), entitySignal.expectSignal()]); try { // will fail - await counter.set(id, { n: 0 }, { expectedVersion: 1 }); + await counter.set( + { namespace: "default", id, n: 0 }, + { expectedVersion: 1 } + ); } catch (err) { console.error("expected the entity set to fail", err); } - const { entity, version } = (await counter.getWithMetadata(id)) ?? {}; - await counter.set(id, { n: entity!.n + 1 }, { expectedVersion: version }); - const value = await counter.get(id); + const { value: entityValue, version } = + (await counter.getWithMetadata(["default", id])) ?? {}; + await counter.set( + { namespace: "default", id, n: entityValue!.n + 1 }, + { expectedVersion: version } + ); + const value = await counter.get(["default", id]); await Entity.transactWrite([ { entity: counter, - operation: { - operation: "set", - key: id, - value: { n: (value?.n ?? 0) + 1 }, - }, + operation: "set", + value: { namespace: "default", id, n: (value?.n ?? 0) + 1 }, }, ]); // send deletion, to be picked up by the stream - counter.delete(id); - await counter.list({}); + await counter.delete(["default", id]); + await counter.query(["default", id]); // this signal will contain the final value after deletion - return await entitySignal2.expectSignal(); + const result1 = await entitySignal2.expectSignal(); + + /** + * Testing sort keys and query + */ + + await Promise.all([ + counterCollection.set({ id, counterNumber: 1, n: 1 }), + counterCollection.set({ id, counterNumber: 2, n: 1 }), + counterCollection.set({ id, counterNumber: 3, n: 1 }), + ]); + + const counter1 = await counterCollection.get({ id, counterNumber: 1 }); + await counterCollection.set({ + id, + counterNumber: 2, + n: (counter1?.n ?? 0) + 1, + }); + + const counters = await counterCollection.query({ id }); + + return [ + result1, + counters.entries?.map((c) => [c.value.counterNumber, c.value.n]), + ]; } ); -export const check = entity<{ n: number }>("check"); +export const check = entity("check4", { + attributes: { n: z.number(), id: z.string() }, + partition: ["id"], +}); const gitErDone = transaction("gitErDone", async ({ id }: { id: string }) => { - const val = await check.get(id); - await check.set(id, { n: val?.n ?? 0 + 1 }); + const val = await check.get([id]); + await check.set({ id, n: val?.n ?? 0 + 1 }); return val?.n ?? 0 + 1; }); @@ -645,17 +700,25 @@ const noise = task( let transact: Promise | undefined; while (n-- > 0) { try { - await check.set(id, { n }); + await check.set({ id, n }); } catch (err) { + console.error(err); if (!(err instanceof TransactionConflictException)) { throw err; } } + console.log(n); if (n === x) { transact = gitErDone({ id }); } } - return await transact; + console.log("waiting..."); + try { + return await transact; + } catch (err) { + console.error("Transaction Errored", err); + throw err; + } } ); @@ -665,11 +728,11 @@ export const transactionWorkflow = workflow( const one = await noise({ x: 40 }); const two = await noise({ x: 60 }); const [, three] = await Promise.allSettled([ - check.set(id, { n: two ?? 0 + 1 }), + check.set({ id, n: two ?? 0 + 1 }), gitErDone({ id }), - check.set(id, { n: two ?? 0 + 1 }), + check.set({ id, n: two ?? 0 + 1 }), ]); - await check.delete(id); + await check.delete([id]); return [one, two, three.status === "fulfilled" ? three.value : "AHHH"]; } ); diff --git a/apps/tests/aws-runtime/test/tester.test.ts b/apps/tests/aws-runtime/test/tester.test.ts index f94fa3027..b7258514e 100644 --- a/apps/tests/aws-runtime/test/tester.test.ts +++ b/apps/tests/aws-runtime/test/tester.test.ts @@ -119,7 +119,14 @@ eventualRuntimeTestHarness( testCompletion("awsSdkCalls", createAndDestroyWorkflow, "done"); - testCompletion("ent", entityWorkflow, { n: 7 }); + testCompletion("ent", entityWorkflow, [ + { n: 7 }, + [ + [1, 1], + [2, 2], + [3, 1], + ], + ]); testCompletion("transaction", transactionWorkflow, ([one, two, three]) => { expect(one).not.toBeUndefined(); diff --git a/packages/@eventual/aws-cdk/src/entity-service.ts b/packages/@eventual/aws-cdk/src/entity-service.ts index 20c2bfe5c..e12b5db09 100644 --- a/packages/@eventual/aws-cdk/src/entity-service.ts +++ b/packages/@eventual/aws-cdk/src/entity-service.ts @@ -1,13 +1,21 @@ import { - EntityEntityRecord, entityServiceTableName, entityServiceTableSuffix, ENV_NAMES, } from "@eventual/aws-runtime"; -import { EntityRuntime, EntityStreamFunction } from "@eventual/core-runtime"; -import { TransactionSpec } from "@eventual/core/internal"; +import { + EntityRuntime, + EntityStreamFunction, + normalizeCompositeKey, +} from "@eventual/core-runtime"; +import { + assertNever, + KeyDefinitionPart, + TransactionSpec, +} from "@eventual/core/internal"; import { Duration, RemovalPolicy, Stack } from "aws-cdk-lib"; import { + Attribute, AttributeType, BillingMode, ITable, @@ -200,29 +208,28 @@ interface EntityStreamProps { table: ITable; serviceProps: EntityServiceProps; entityService: EntityService; + entity: EntityRuntime; stream: EntityStreamFunction; } -export class Entity extends Construct { +class Entity extends Construct { public table: ITable; public streams: Record; constructor(scope: Construct, props: EntityProps) { super(scope, props.entity.name); + const keyDefinition = props.entity.key; + this.table = new Table(this, "Table", { tableName: entityServiceTableName( props.serviceProps.serviceName, props.entity.name ), - partitionKey: { - name: "pk", - type: AttributeType.STRING, - }, - sortKey: { - name: "sk", - type: AttributeType.STRING, - }, + partitionKey: entityKeyDefinitionToAttribute(keyDefinition.partition), + sortKey: keyDefinition.sort + ? entityKeyDefinitionToAttribute(keyDefinition.sort) + : undefined, billingMode: BillingMode.PAY_PER_REQUEST, removalPolicy: RemovalPolicy.DESTROY, // only include the stream if there are listeners @@ -240,6 +247,7 @@ export class Entity extends Construct { props.entity.streams.map((s) => [ s.spec.name, new EntityStream(entityStreamScope, s.spec.name, { + entity: props.entity, entityService: props.entityService, serviceProps: props.serviceProps, stream: s, @@ -256,48 +264,60 @@ export class EntityStream extends Construct implements EventualResource { constructor(scope: Construct, id: string, props: EntityStreamProps) { super(scope, id); - const namespaces = props.stream.spec.options?.namespaces; - const namespacePrefixes = props.stream.spec.options?.namespacePrefixes; const streamName = props.stream.spec.name; const entityName = props.stream.spec.entityName; - const filters = { - ...(props.stream.spec.options?.operations - ? { - eventName: FilterRule.or( - ...(props.stream.spec.options?.operations?.map((op) => - op.toUpperCase() - ) ?? []) - ), - } - : undefined), - ...((namespaces && namespaces.length > 0) || - (namespacePrefixes && namespacePrefixes.length > 0) - ? { - dynamodb: { - Keys: { - pk: { - S: FilterRule.or( - // for each namespace given, match the complete name. - ...(namespaces - ? namespaces.map((n) => EntityEntityRecord.key(n)) - : []), - // for each namespace prefix given, build a prefix statement for each one. - ...(namespacePrefixes - ? namespacePrefixes.flatMap( - (n) => - FilterRule.beginsWith( - EntityEntityRecord.key(n) - ) as unknown as string[] - ) - : []) - ), - }, + const keyDefinition = props.entity.key; + + const normalizedQueryKeys = + props.stream.spec.options?.queryKeys?.map((q) => + normalizeCompositeKey(keyDefinition, q) + ) ?? []; + + const queryPatterns = normalizedQueryKeys.map((k) => { + return { + // if no part of the partition key is provided, do not include it + partition: + k.partition.keyValue !== undefined + ? k.partition.partialValue + ? FilterRule.beginsWith(k.partition.keyValue.toString()) + : k.partition.keyValue + : undefined, + sort: + k.sort && k.sort.keyValue !== undefined + ? k.sort?.partialValue + ? FilterRule.beginsWith(k.sort.keyValue.toString()) + : k.sort.keyValue + : undefined, + }; + }); + + const eventNameFilter = props.stream.spec.options?.operations + ? { + eventName: FilterRule.or( + ...(props.stream.spec.options?.operations?.map((op) => + op.toUpperCase() + ) ?? []) + ), + } + : undefined; + + // create a filter expression for each combination of key filter when present + // Would prefer to use $or within a single expression, but it seems it doesn't work with event source maps (yet?) + // TODO: can reduce the number of unique expressions by merging single field key queries togethers (all partition or all sort) + const filters = + !eventNameFilter && queryPatterns.length === 0 + ? [] + : eventNameFilter && queryPatterns.length === 0 + ? [FilterCriteria.filter(eventNameFilter)] + : queryPatterns.map((q) => + FilterCriteria.filter({ + ...eventNameFilter, + dynamodb: { + Keys: keyMatcher(q), }, - }, - } - : undefined), - }; + }) + ); this.handler = new ServiceFunction(this, "Handler", { build: props.serviceProps.build, @@ -315,9 +335,7 @@ export class EntityStream extends Construct implements EventualResource { new DynamoEventSource(props.table, { startingPosition: StartingPosition.TRIM_HORIZON, maxBatchingWindow: Duration.seconds(0), - ...(Object.keys(filters).length > 0 - ? { filters: [FilterCriteria.filter(filters)] } - : {}), + ...(filters.length > 0 ? { filters } : {}), }), ], }, @@ -334,5 +352,50 @@ export class EntityStream extends Construct implements EventualResource { props.entityService.configureReadWriteEntityTable(this.handler); this.grantPrincipal = this.handler.grantPrincipal; + + function keyMatcher(item: (typeof queryPatterns)[number]) { + return { + ...(item.partition + ? { + [keyDefinition.partition.keyAttribute]: { + [keyTypeToAttributeType(keyDefinition.partition)]: [ + item.partition, + ].flat(), + }, + } + : {}), + ...(keyDefinition.sort && item.sort + ? { + [keyDefinition.sort.keyAttribute]: { + [keyTypeToAttributeType(keyDefinition.sort)]: [ + item.sort, + ].flat(), + }, + } + : {}), + }; + + function keyTypeToAttributeType(keyDef: KeyDefinitionPart) { + return keyDef.type === "number" + ? "N" + : keyDef.type === "string" + ? "S" + : assertNever(keyDef.type); + } + } } } + +export function entityKeyDefinitionToAttribute( + part: KeyDefinitionPart +): Attribute { + return { + name: part.keyAttribute, + type: + part.type === "string" + ? AttributeType.STRING + : part.type === "number" + ? AttributeType.NUMBER + : assertNever(part.type), + }; +} diff --git a/packages/@eventual/aws-runtime/package.json b/packages/@eventual/aws-runtime/package.json index 79b7276a8..b68004f83 100644 --- a/packages/@eventual/aws-runtime/package.json +++ b/packages/@eventual/aws-runtime/package.json @@ -21,6 +21,7 @@ "@aws-sdk/client-scheduler": "^3.254.0", "@aws-sdk/client-sqs": "^3.254.0", "@aws-sdk/s3-request-presigner": "^3.254.0", + "@aws-sdk/util-dynamodb": "^3.254.0", "@eventual/aws-client": "workspace:^", "@eventual/core": "workspace:^", "@eventual/core-runtime": "workspace:^", diff --git a/packages/@eventual/aws-runtime/src/clients/transaction-client.ts b/packages/@eventual/aws-runtime/src/clients/transaction-client.ts index 1aeaa3368..fad8d55ed 100644 --- a/packages/@eventual/aws-runtime/src/clients/transaction-client.ts +++ b/packages/@eventual/aws-runtime/src/clients/transaction-client.ts @@ -16,6 +16,7 @@ export class AWSTransactionClient implements TransactionClient { public async executeTransaction( request: ExecuteTransactionRequest ): Promise { + console.debug("Invoking Transaction: ", request.transaction); const response = await this.props.lambda.send( new InvokeCommand({ FunctionName: getLazy(this.props.transactionWorkerFunctionArn), @@ -27,9 +28,16 @@ export class AWSTransactionClient implements TransactionClient { ); if (!response.Payload) { + console.error( + "Transaction Returned Invalid Response: ", + request.transaction, + response.FunctionError + ); throw new Error("Invalid response from the transaction worker"); } + console.debug("Transaction Complete: ", request.transaction); + return JSON.parse( Buffer.from(response.Payload).toString("utf-8") ) as ExecuteTransactionResponse; diff --git a/packages/@eventual/aws-runtime/src/create.ts b/packages/@eventual/aws-runtime/src/create.ts index 0b75e9c81..6a43b2f46 100644 --- a/packages/@eventual/aws-runtime/src/create.ts +++ b/packages/@eventual/aws-runtime/src/create.ts @@ -9,9 +9,9 @@ import { Client, Pluggable } from "@aws-sdk/types"; import { AWSHttpEventualClient } from "@eventual/aws-client"; import { LogLevel } from "@eventual/core"; import { - EntityClient, ExecutionQueueClient, ExecutionStore, + GlobalEntityProvider, GlobalTaskProvider, GlobalWorkflowProvider, LogAgent, @@ -217,15 +217,12 @@ export const createExecutionHistoryStateStore = /* @__PURE__ */ memoize( }) ); -export const createEntityClient = memoize( - () => new EntityClient(createEntityStore()) -); - export const createEntityStore = memoize( () => new AWSEntityStore({ dynamo: dynamo(), serviceName: env.serviceName, + entityProvider: new GlobalEntityProvider(), }) ); diff --git a/packages/@eventual/aws-runtime/src/handlers/apig-command-adapter.ts b/packages/@eventual/aws-runtime/src/handlers/apig-command-adapter.ts index 26ed713be..563109fd2 100644 --- a/packages/@eventual/aws-runtime/src/handlers/apig-command-adapter.ts +++ b/packages/@eventual/aws-runtime/src/handlers/apig-command-adapter.ts @@ -42,7 +42,7 @@ export function createApiGCommandAdaptor({ : undefined; registerWorkerIntrinsics({ bucketStore: undefined, - entityClient: undefined, + entityStore: undefined, serviceClient, serviceSpec, serviceUrl, diff --git a/packages/@eventual/aws-runtime/src/handlers/bucket-handler-worker.ts b/packages/@eventual/aws-runtime/src/handlers/bucket-handler-worker.ts index 16e1f9a74..f512de1d7 100644 --- a/packages/@eventual/aws-runtime/src/handlers/bucket-handler-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/bucket-handler-worker.ts @@ -10,7 +10,7 @@ import { import { S3Handler } from "aws-lambda"; import { createBucketStore, - createEntityClient, + createEntityStore, createServiceClient, } from "../create.js"; import { @@ -22,7 +22,7 @@ import { const worker = createBucketNotificationHandlerWorker({ bucketStore: createBucketStore(), - entityClient: createEntityClient(), + entityStore: createEntityStore(), serviceClient: createServiceClient({}), serviceName, serviceSpec, diff --git a/packages/@eventual/aws-runtime/src/handlers/command-worker.ts b/packages/@eventual/aws-runtime/src/handlers/command-worker.ts index 9c4787fdb..f3b866a37 100644 --- a/packages/@eventual/aws-runtime/src/handlers/command-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/command-worker.ts @@ -4,7 +4,7 @@ import serviceSpec from "@eventual/injected/spec"; import { createCommandWorker } from "@eventual/core-runtime"; import { createBucketStore, - createEntityClient, + createEntityStore, createEventClient, createServiceClient, createTransactionClient, @@ -21,7 +21,7 @@ import { serviceName } from "../env.js"; export default createApiGCommandAdaptor({ commandWorker: createCommandWorker({ bucketStore: createBucketStore(), - entityClient: createEntityClient(), + entityStore: createEntityStore(), // the service client, spec, and service url will be created at runtime, using a computed uri from the apigateway request serviceClient: undefined, serviceSpec: undefined, diff --git a/packages/@eventual/aws-runtime/src/handlers/entity-stream-worker.ts b/packages/@eventual/aws-runtime/src/handlers/entity-stream-worker.ts index f1c99d2d8..7462a326c 100644 --- a/packages/@eventual/aws-runtime/src/handlers/entity-stream-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/entity-stream-worker.ts @@ -2,17 +2,22 @@ import serviceSpec from "@eventual/injected/spec"; // the user's entry point will register streams as a side effect. import "@eventual/injected/entry"; -import { EntityStreamItem } from "@eventual/core"; +import type { AttributeValue } from "@aws-sdk/client-dynamodb"; +import { unmarshall } from "@aws-sdk/util-dynamodb"; +import type { EntityStreamItem } from "@eventual/core"; import { + GlobalEntityProvider, + convertNormalizedEntityKeyToMap, createEntityStreamWorker, getLazy, + normalizeCompositeKey, promiseAllSettledPartitioned, } from "@eventual/core-runtime"; -import { EntityStreamOperation } from "@eventual/core/internal"; -import { DynamoDBStreamHandler } from "aws-lambda"; +import type { EntityStreamOperation } from "@eventual/core/internal"; +import type { DynamoDBStreamHandler } from "aws-lambda"; import { createBucketStore, - createEntityClient, + createEntityStore, createServiceClient, } from "../create.js"; import { @@ -23,9 +28,11 @@ import { } from "../env.js"; import { EntityEntityRecord } from "../stores/entity-store.js"; +const entityProvider = new GlobalEntityProvider(); + const worker = createEntityStreamWorker({ bucketStore: createBucketStore(), - entityClient: createEntityClient(), + entityStore: createEntityStore(), serviceClient: createServiceClient({}), serviceSpec, serviceName, @@ -40,9 +47,6 @@ export default (async (event) => { records, async (record) => { try { - const keys = record.dynamodb?.Keys as Partial; - const pk = keys?.pk?.S; - const sk = keys?.sk?.S; const operation = record.eventName?.toLowerCase() as | EntityStreamOperation | undefined; @@ -53,29 +57,58 @@ export default (async (event) => { | Partial | undefined; - if (pk && sk && operation) { - const namespace = - EntityEntityRecord.parseNamespaceFromPartitionKey(pk); - const key = EntityEntityRecord.parseKeyFromSortKey(sk); + const _entityName = getLazy(entityName); + const entity = entityProvider.getEntity(_entityName); + + if (!entity) { + throw new Error(`Entity ${_entityName} was not found`); + } + + const newValue = newItem + ? unmarshall(newItem as Record) + : undefined; + const newVersion = newValue?.__version; + + const oldValue = oldItem + ? unmarshall(oldItem as Record) + : undefined; + const oldVersion = oldValue?.__version; + + const bestValue = newValue ?? oldValue; + if (!bestValue) { + throw new Error( + "Expected at least one of old value or new value in the stream event." + ); + } + + const normalizedKey = normalizeCompositeKey(entity, bestValue); + const keyMap = convertNormalizedEntityKeyToMap(normalizedKey); + + if (newValue) { + delete newValue[EntityEntityRecord.VERSION_FIELD]; + delete newValue[normalizedKey.partition.keyAttribute]; + if (normalizedKey.sort) { + delete newValue[normalizedKey.sort.keyAttribute]; + } + } + if (oldValue) { + delete oldValue[EntityEntityRecord.VERSION_FIELD]; + delete oldValue[normalizedKey.partition.keyAttribute]; + if (normalizedKey.sort) { + delete oldValue[normalizedKey.sort.keyAttribute]; + } + } - const item: EntityStreamItem = { + if (operation) { + const item: EntityStreamItem = { entityName: getLazy(entityName), streamName: getLazy(entityStreamName), - namespace, - key, - newValue: newItem?.value?.S - ? JSON.parse(newItem?.value?.S) - : undefined, - newVersion: newItem?.version?.N - ? Number(newItem?.version?.N) - : (undefined as any), + key: keyMap, + newValue: newValue as any, + newVersion, operation, - oldValue: oldItem?.value?.S - ? JSON.parse(oldItem?.value?.S) - : undefined, - oldVersion: oldItem?.version?.N - ? Number(oldItem?.version?.N) - : undefined, + oldValue, + oldVersion, }; return worker(item); diff --git a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts index 15856429b..3a58ad1f9 100644 --- a/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts +++ b/packages/@eventual/aws-runtime/src/handlers/orchestrator.ts @@ -1,16 +1,16 @@ import "@eventual/injected/entry"; import { + createOrchestrator, ExecutionQueueEventEnvelope, RemoteExecutorProvider, WorkflowCallExecutor, - createOrchestrator, } from "@eventual/core-runtime"; import type { SQSEvent, SQSRecord } from "aws-lambda"; import { AWSMetricsClient } from "../clients/metrics-client.js"; import { createBucketStore, - createEntityClient, + createEntityStore, createEventClient, createExecutionHistoryStateStore, createExecutionHistoryStore, @@ -37,7 +37,7 @@ const orchestrate = createOrchestrator({ metricsClient: AWSMetricsClient, callExecutor: new WorkflowCallExecutor({ bucketStore: createBucketStore(), - entityClient: createEntityClient(), + entityStore: createEntityStore(), eventClient: createEventClient(), executionQueueClient: createExecutionQueueClient(), taskClient: createTaskClient(), diff --git a/packages/@eventual/aws-runtime/src/handlers/subscription-worker.ts b/packages/@eventual/aws-runtime/src/handlers/subscription-worker.ts index f1d6ac482..65ec04a1e 100644 --- a/packages/@eventual/aws-runtime/src/handlers/subscription-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/subscription-worker.ts @@ -8,7 +8,7 @@ import { import type { EventBridgeEvent } from "aws-lambda"; import { createBucketStore, - createEntityClient, + createEntityStore, createEventClient, createServiceClient, createTransactionClient, @@ -17,7 +17,7 @@ import { serviceName, serviceUrl } from "../env.js"; export const processEvent = createSubscriptionWorker({ bucketStore: createBucketStore(), - entityClient: createEntityClient(), + entityStore: createEntityStore(), // partially uses the runtime clients and partially uses the http client serviceClient: createServiceClient({ eventClient: createEventClient(), diff --git a/packages/@eventual/aws-runtime/src/handlers/system-command-handler.ts b/packages/@eventual/aws-runtime/src/handlers/system-command-handler.ts index 464222c43..4f24c1f8e 100644 --- a/packages/@eventual/aws-runtime/src/handlers/system-command-handler.ts +++ b/packages/@eventual/aws-runtime/src/handlers/system-command-handler.ts @@ -36,7 +36,7 @@ function systemCommandWorker( return createApiGCommandAdaptor({ commandWorker: createCommandWorker({ bucketStore: createBucketStore(), - entityClient: undefined, + entityStore: undefined, serviceClient: undefined, serviceSpec: undefined, }), diff --git a/packages/@eventual/aws-runtime/src/handlers/task-worker.ts b/packages/@eventual/aws-runtime/src/handlers/task-worker.ts index 047eb421f..048f6e0f5 100644 --- a/packages/@eventual/aws-runtime/src/handlers/task-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/task-worker.ts @@ -11,7 +11,7 @@ import { import { AWSMetricsClient } from "../clients/metrics-client.js"; import { createBucketStore, - createEntityClient, + createEntityStore, createEventClient, createExecutionQueueClient, createExecutionStore, @@ -26,7 +26,7 @@ import { serviceName, serviceUrl } from "../env.js"; const worker = createTaskWorker({ bucketStore: createBucketStore(), - entityClient: createEntityClient(), + entityStore: createEntityStore(), eventClient: createEventClient(), executionQueueClient: createExecutionQueueClient(), logAgent: createLogAgent(), diff --git a/packages/@eventual/aws-runtime/src/handlers/transaction-worker.ts b/packages/@eventual/aws-runtime/src/handlers/transaction-worker.ts index f96d1a92a..0507ed0da 100644 --- a/packages/@eventual/aws-runtime/src/handlers/transaction-worker.ts +++ b/packages/@eventual/aws-runtime/src/handlers/transaction-worker.ts @@ -1,7 +1,10 @@ // the user's entry point will register transactions as a side effect. import "@eventual/injected/entry"; -import { createTransactionWorker } from "@eventual/core-runtime"; +import { + createTransactionWorker, + GlobalEntityProvider, +} from "@eventual/core-runtime"; import { createEntityStore, createEventClient, @@ -11,6 +14,7 @@ import { serviceName } from "../env.js"; export default createTransactionWorker({ entityStore: createEntityStore(), + entityProvider: new GlobalEntityProvider(), eventClient: createEventClient(), executionQueueClient: createExecutionQueueClient(), serviceName, diff --git a/packages/@eventual/aws-runtime/src/stores/entity-store.ts b/packages/@eventual/aws-runtime/src/stores/entity-store.ts index 822a4df7e..8f35d8358 100644 --- a/packages/@eventual/aws-runtime/src/stores/entity-store.ts +++ b/packages/@eventual/aws-runtime/src/stores/entity-store.ts @@ -13,24 +13,28 @@ import { Update, UpdateItemCommand, } from "@aws-sdk/client-dynamodb"; +import { marshall, unmarshall } from "@aws-sdk/util-dynamodb"; import { - CompositeKey, + Entity, + Attributes, EntityConsistencyOptions, - EntityListKeysResult, - EntityListRequest, - EntityListResult, + EntityQueryOptions, + EntityQueryResult, EntitySetOptions, - EntityTransactItem, + EntityWithMetadata, + TransactionCancelled, + TransactionConflict, + UnexpectedVersion, } from "@eventual/core"; import { + EntityProvider, EntityStore, - EntityWithMetadata, getLazy, LazyValue, - normalizeCompositeKey, - TransactionCancelledResult, - TransactionConflictResult, - UnexpectedVersionResult, + NormalizedEntityCompositeKey, + NormalizedEntityCompositeKeyComplete, + NormalizedEntityKeyCompletePart, + NormalizedEntityTransactItem, } from "@eventual/core-runtime"; import { assertNever } from "@eventual/core/internal"; import { entityServiceTableName, queryPageWithToken } from "../utils.js"; @@ -38,24 +42,41 @@ import { entityServiceTableName, queryPageWithToken } from "../utils.js"; export interface AWSEntityStoreProps { dynamo: DynamoDBClient; serviceName: LazyValue; + entityProvider: EntityProvider; } -export class AWSEntityStore implements EntityStore { - constructor(private props: AWSEntityStoreProps) {} +export type EntityAttributesFromEntity = E extends Entity< + infer Attributes, + any, + any +> + ? Attributes + : never; + +export type EntityAttributesWithVersion = + EntityAttributesFromEntity & { + __version: number; + }; + +export type MarshalledEntityAttributesWithVersion = { + [k in keyof EntityAttributesFromEntity]: AttributeValue; +} & { + __version: AttributeValue.NMember; +}; + +export class AWSEntityStore extends EntityStore { + constructor(private props: AWSEntityStoreProps) { + super(props.entityProvider); + } - public async getEntityValue( - name: string, - _key: string | CompositeKey - ): Promise | undefined> { + protected override async _getWithMetadata( + entity: Entity, + key: NormalizedEntityCompositeKeyComplete + ): Promise { const item = await this.props.dynamo.send( new GetItemCommand({ - Key: this.entityKey(_key), - TableName: this.tableName(name), - ProjectionExpression: "#value,#version", - ExpressionAttributeNames: { - "#version": "version", - "#value": "value", - }, + Key: this.entityKey(key), + TableName: this.tableName(entity), ConsistentRead: true, }) ); @@ -64,277 +85,322 @@ export class AWSEntityStore implements EntityStore { return undefined; } - const record = item.Item as EntityEntityRecord; + const { __version, ...value } = unmarshall( + item.Item + ) as EntityAttributesWithVersion; + + // if the key attributes are computed, remove them from the return value. + if (!(key.partition.keyAttribute in entity.attributes.shape)) { + delete value[key.partition.keyAttribute]; + } + if (key.sort && !(key.sort.keyAttribute in entity.attributes.shape)) { + delete value[key.sort.keyAttribute]; + } return { - entity: JSON.parse(record.value.S), - version: Number(record.version.N), + value, + version: __version, }; } - public async setEntityValue( - name: string, - _key: string | CompositeKey, + public override async _set( entity: Entity, + value: Attributes, + key: NormalizedEntityCompositeKeyComplete, options?: EntitySetOptions - ): Promise<{ version: number } | UnexpectedVersionResult> { + ): Promise<{ version: number }> { try { const result = await this.props.dynamo.send( new UpdateItemCommand({ - ...this.createSetRequest(name, _key, entity, options), + ...this.createSetRequest(entity, value, key, options), ReturnValues: ReturnValue.ALL_NEW, }) ); const record = result.Attributes as EntityEntityRecord; - return { version: Number(record.version.N) }; + return { version: Number(record.__version.N) }; } catch (err) { if (err instanceof ConditionalCheckFailedException) { - return { unexpectedVersion: true }; + throw new UnexpectedVersion("Unexpected Version"); } throw err; } } - private createSetRequest( - name: string, - _key: string | CompositeKey, + protected override async _delete( entity: Entity, - options?: EntitySetOptions - ): Update { - const value = JSON.stringify(entity); - return { - Key: this.entityKey(_key), - UpdateExpression: - "SET #value=:value, #version=if_not_exists(#version, :startingVersion) + :versionIncrement", - ExpressionAttributeNames: { - "#value": "value", - "#version": "version", - }, - ExpressionAttributeValues: { - ...(options?.expectedVersion - ? { - ":expectedVersion": { N: options.expectedVersion.toString() }, - } - : undefined), - ":value": { S: value }, - ":startingVersion": { N: "0" }, - ":versionIncrement": { - N: options?.incrementVersion === false ? "0" : "1", - }, - }, - ConditionExpression: - options?.expectedVersion !== undefined - ? options?.expectedVersion === 0 - ? "attribute_not_exists(#version)" - : "#version=:expectedVersion" - : undefined, - TableName: this.tableName(name), - }; - } - - public async deleteEntityValue( - name: string, - _key: string | CompositeKey, - options?: EntityConsistencyOptions - ): Promise { - await this.props.dynamo.send( - new DeleteItemCommand(this.createDeleteRequest(name, _key, options)) - ); + key: NormalizedEntityCompositeKeyComplete, + options?: EntityConsistencyOptions | undefined + ): Promise { + try { + await this.props.dynamo.send( + new DeleteItemCommand(this.createDeleteRequest(entity, key, options)) + ); + } catch (err) { + if (err instanceof ConditionalCheckFailedException) { + throw new UnexpectedVersion("Unexpected Version"); + } + } } private createDeleteRequest( - name: string, - _key: string | CompositeKey, + entity: Entity, + key: NormalizedEntityCompositeKeyComplete, options?: EntityConsistencyOptions ): Delete { return { - Key: this.entityKey(_key), + Key: this.entityKey(key), ConditionExpression: options?.expectedVersion !== undefined - ? "#version=:expectedVersion" + ? "#__version=:expectedVersion" : undefined, ExpressionAttributeNames: options?.expectedVersion !== undefined ? { - "#version": "version", + "#__version": "__version", } : undefined, ExpressionAttributeValues: options?.expectedVersion !== undefined ? { ":expectedVersion": { N: options.expectedVersion.toString() } } : undefined, - TableName: this.tableName(name), + TableName: this.tableName(entity), }; } - private entityKey(_key: string | CompositeKey) { - const { key, namespace } = normalizeCompositeKey(_key); - return { - pk: { S: EntityEntityRecord.key(namespace) }, - sk: { S: EntityEntityRecord.sortKey(key) }, - } satisfies Partial; - } + protected override async _query( + entity: Entity, + queryKey: NormalizedEntityCompositeKey, + options?: EntityQueryOptions + ): Promise { + const allAttributes = new Set([ + queryKey.partition.keyAttribute, + ...(queryKey.sort && queryKey.sort.keyValue !== undefined + ? [queryKey.sort.keyAttribute] + : []), + ]); - public async listEntityEntries( - name: string, - request: EntityListRequest - ): Promise> { - const result = await this.list(name, request, ["value", "sk", "version"]); + const result = await queryPageWithToken< + MarshalledEntityAttributesWithVersion + >( + { + dynamoClient: this.props.dynamo, + pageSize: options?.limit ?? 1000, + keys: queryKey.sort + ? [queryKey.partition.keyAttribute, queryKey.sort.keyAttribute] + : [queryKey.partition.keyAttribute], + nextToken: options?.nextToken, + }, + { + TableName: this.tableName(entity), + KeyConditionExpression: + queryKey.sort && queryKey.sort.keyValue !== undefined + ? queryKey.sort.partialValue + ? `${formatAttributeNameMapKey( + queryKey.partition.keyAttribute + )}=:pk AND begins_with(${formatAttributeNameMapKey( + queryKey.sort.keyAttribute + )}, :sk)` + : `${formatAttributeNameMapKey( + queryKey.partition.keyAttribute + )}=:pk AND ${formatAttributeNameMapKey( + queryKey.sort.keyAttribute + )}=:sk` + : `${formatAttributeNameMapKey( + queryKey.partition.keyAttribute + )}=:pk`, + ExpressionAttributeValues: { + ":pk": + typeof queryKey.partition.keyValue === "number" + ? { N: queryKey.partition.keyValue.toString() } + : { S: queryKey.partition.keyValue }, + ...(queryKey.sort && queryKey.sort.keyValue !== undefined + ? { + ":sk": + typeof queryKey.sort.keyValue === "string" + ? { S: queryKey.sort.keyValue } + : { N: queryKey.sort.keyValue.toString() }, + } + : {}), + }, + ExpressionAttributeNames: Object.fromEntries( + [...allAttributes]?.map((f) => [formatAttributeNameMapKey(f), f]) + ), + } + ); return { nextToken: result.nextToken, - entries: result.records.map((r) => ({ - entity: JSON.parse(r.value.S), - version: Number(r.version.N), - key: EntityEntityRecord.parseKeyFromSortKey(r.sk.S), + entries: result.records.map(({ __version, ...r }) => ({ + value: unmarshall(r), + version: Number(__version.N), })), }; } - public async listEntityKeys( - name: string, - request: EntityListRequest - ): Promise { - const result = await this.list(name, request, ["sk"]); - - return { - nextToken: result.nextToken, - keys: result.records.map((r) => - EntityEntityRecord.parseKeyFromSortKey(r.sk.S) - ), - }; - } - - public async transactWrite( - items: EntityTransactItem[] - ): Promise { + protected override async _transactWrite( + items: NormalizedEntityTransactItem[] + ): Promise { try { await this.props.dynamo.send( new TransactWriteItemsCommand({ - TransactItems: items.map((i): TransactWriteItem => { - if (i.operation.operation === "set") { - return { - Update: this.createSetRequest( - i.entity, - i.operation.key, - i.operation.value, - i.operation.options - ), - }; - } else if (i.operation.operation === "delete") { - return { - Delete: this.createDeleteRequest( - i.entity, - i.operation.key, - i.operation.options - ), - }; - } else if (i.operation.operation === "condition") { - return { - ConditionCheck: { - ConditionExpression: - i.operation.version !== undefined - ? i.operation.version === 0 - ? "attribute_not_exists(#version)" - : "#version=:expectedVersion" - : undefined, - TableName: this.tableName(i.entity), - Key: this.entityKey(i.operation.key), - ExpressionAttributeNames: { - "#version": "version", + TransactItems: items.map((item): TransactWriteItem => { + return item.operation === "set" + ? { + Update: this.createSetRequest( + item.entity, + item.value, + item.key, + item.options + ), + } + : item.operation === "delete" + ? { + Delete: this.createDeleteRequest( + item.entity, + item.key, + item.options + ), + } + : item.operation === "condition" + ? { + ConditionCheck: { + ConditionExpression: + item.version !== undefined + ? item.version === 0 + ? "attribute_not_exists(#version)" + : "#version=:expectedVersion" + : undefined, + TableName: this.tableName(item.entity), + Key: this.entityKey(item.key), + ExpressionAttributeNames: { + "#version": EntityEntityRecord.VERSION_FIELD, + }, + ExpressionAttributeValues: + item.version !== undefined + ? { + ":expectedVersion": { + N: item.version.toString(), + }, + } + : undefined, }, - ExpressionAttributeValues: - i.operation.version !== undefined - ? { - ":expectedVersion": { - N: i.operation.version.toString(), - }, - } - : undefined, - }, - }; - } - - return assertNever(i.operation); + } + : assertNever(item); }), }) ); } catch (err) { if (err instanceof TransactionCanceledException) { - return { - reasons: - err.CancellationReasons?.map((c) => { - // TODO: handle other failure reasons - if (c.Code === "NONE") { - return undefined; - } - return { unexpectedVersion: true }; - }) ?? [], - }; + throw new TransactionCancelled( + err.CancellationReasons?.map((r) => + r.Code === "NONE" + ? undefined + : new UnexpectedVersion("Unexpected Version") + ) ?? [] + ); } else if (err instanceof TransactionConflictException) { - return { transactionConflict: true }; + throw new TransactionConflict(); } throw err; } } - private list(name: string, request: EntityListRequest, fields?: string[]) { - return queryPageWithToken( - { - dynamoClient: this.props.dynamo, - pageSize: request.limit ?? 1000, - keys: ["pk", "sk"], - nextToken: request.nextToken, + private createSetRequest( + entity: Entity, + value: Attr, + key: NormalizedEntityCompositeKey, + options?: EntitySetOptions + ): Update { + const valueRecord = marshall(value, { removeUndefinedValues: true }); + + // if the key attributes are not computed and are in the original value, remove them from the set expression + delete valueRecord[key.partition.keyAttribute]; + if (key.sort) { + delete valueRecord[key.sort.keyAttribute]; + } + + return { + Key: this.entityKey(key), + UpdateExpression: [ + "SET #__version=if_not_exists(#__version, :__startingVersion) + :__versionIncrement", + ...Object.keys(valueRecord).map( + (key) => + `${formatAttributeNameMapKey(key)}=${formatAttributeValueMapKey( + key + )}` + ), + ].join(","), + ExpressionAttributeNames: { + "#__version": "__version", + ...Object.fromEntries( + Object.keys(valueRecord).map((key) => [ + formatAttributeNameMapKey(key), + key, + ]) + ), }, - { - TableName: this.tableName(name), - KeyConditionExpression: "pk=:pk AND begins_with(sk, :sk)", - ExpressionAttributeValues: { - ":pk": { S: EntityEntityRecord.key(request.namespace) }, - ":sk": { S: EntityEntityRecord.sortKey(request.prefix ?? "") }, + ExpressionAttributeValues: { + ...(options?.expectedVersion + ? { + ":__expectedVersion": { N: options.expectedVersion.toString() }, + } + : undefined), + ":__startingVersion": { N: "0" }, + ":__versionIncrement": { + N: options?.incrementVersion === false ? "0" : "1", }, - ExpressionAttributeNames: fields - ? Object.fromEntries(fields?.map((f) => [`#${f}`, f])) + ...Object.fromEntries( + Object.entries(valueRecord).map(([key, value]) => [ + formatAttributeValueMapKey(key), + value, + ]) + ), + }, + ConditionExpression: + options?.expectedVersion !== undefined + ? options?.expectedVersion === 0 + ? "attribute_not_exists(#__version)" + : "#__version=:__expectedVersion" : undefined, - ProjectionExpression: fields?.map((f) => `#${f}`).join(","), - } + TableName: this.tableName(entity), + }; + } + + private entityKey(key: NormalizedEntityCompositeKey) { + const marshalledKey = marshall( + { + [key.partition.keyAttribute]: key.partition.keyValue, + ...(key.sort ? { [key.sort.keyAttribute]: key.sort.keyValue } : {}), + }, + { removeUndefinedValues: true } ); + return marshalledKey; } - private tableName(entityName: string) { - return entityServiceTableName(getLazy(this.props.serviceName), entityName); + private tableName(entity: Entity) { + return entityServiceTableName(getLazy(this.props.serviceName), entity.name); } } export interface EntityEntityRecord extends Record { - pk: { S: `${typeof EntityEntityRecord.PARTITION_KEY_PREFIX}${string}` }; - sk: { S: `${typeof EntityEntityRecord.SORT_KEY_PREFIX}${string}` }; - /** - * A stringified value. - * - * https://dynamodbplace.com/c/questions/map-or-json-dump-string-which-is-better-to-optimize-space - */ - value: AttributeValue.SMember; - version: AttributeValue.NMember; + __version: AttributeValue.NMember; } export const EntityEntityRecord = { - PARTITION_KEY_PREFIX: `EntityEntry$`, - key(namespace?: string) { - return `${this.PARTITION_KEY_PREFIX}${namespace ?? ""}`; - }, - SORT_KEY_PREFIX: `#`, - sortKey(key: string) { - return `${this.SORT_KEY_PREFIX}${key}`; - }, - parseKeyFromSortKey(sortKey: string) { - return sortKey.slice(1); - }, - parseNamespaceFromPartitionKey(sortKey: string): string | undefined { - const namespace = sortKey.slice(this.PARTITION_KEY_PREFIX.length); - return namespace || undefined; - }, + VERSION_FIELD: "__version", }; + +function formatAttributeNameMapKey(key: string) { + return formatAttributeMapKey(key, "#"); +} + +function formatAttributeValueMapKey(key: string) { + return formatAttributeMapKey(key, ":"); +} + +function formatAttributeMapKey(key: string, prefix: string) { + return `${prefix}${key.replaceAll(/[|.\- ]/g, "_")}`; +} diff --git a/packages/@eventual/cli/src/display/event.ts b/packages/@eventual/cli/src/display/event.ts index 385d67b8c..1332a9d7d 100644 --- a/packages/@eventual/cli/src/display/event.ts +++ b/packages/@eventual/cli/src/display/event.ts @@ -1,10 +1,10 @@ -import type { EntityConditionalOperation } from "@eventual/core"; -import { normalizeCompositeKey } from "@eventual/core-runtime"; +import type { EntityTransactItem } from "@eventual/core"; import { BucketRequest, EntityOperation, isBucketRequest, isChildWorkflowScheduled, + isEntityOperationOfType, isEntityRequest, isSignalReceived, isSignalSent, @@ -47,48 +47,74 @@ export function displayEvent(event: WorkflowEvent) { return lines.join("\n"); } -function displayEntityCommand( - operation: EntityOperation | EntityConditionalOperation -) { +function displayEntityCommand(operation: EntityOperation) { const output: string[] = [`Operation: ${operation.operation}`]; if (operation.operation === "transact") { output.push(`Transaction Items:`); output.push( ...operation.items.flatMap((item, i) => [ `${i}:`, - ...displayEntityCommand({ - ...item.operation, - name: - typeof item.entity === "string" ? item.entity : item.entity.name, - }).map((v) => `\t${v}`), + ...displayEntityTransactItem(item).map((v) => `\t${v}`), ]) ); } else { - output.push(`Ent: ${operation.name}`); - if ("key" in operation) { - const { key, namespace } = normalizeCompositeKey(operation.key); - if (namespace) { - output.push(`Namespace: ${namespace}`); - } - output.push(`Key: ${key}`); - if (operation.operation === "set") { - output.push(`Entity: ${JSON.stringify(operation.value)}`); - if (operation.options?.expectedVersion) { - output.push(`Expected Version: ${operation.options.expectedVersion}`); - } - } - } else { - if (operation.request.namespace) { - output.push(`Namespace: ${operation.request.prefix}`); - } - if (operation.request.prefix) { - output.push(`Prefix: ${operation.request.prefix}`); + output.push(`Ent: ${operation.entityName}`); + if ( + isEntityOperationOfType("delete", operation) || + isEntityOperationOfType("get", operation) || + isEntityOperationOfType("getWithMetadata", operation) + ) { + const [key] = operation.params; + output.push(`Key: ${JSON.stringify(key)}`); + } + if (isEntityOperationOfType("set", operation)) { + const [value] = operation.params; + output.push(`Entity: ${JSON.stringify(value)}`); + } + if ( + isEntityOperationOfType("set", operation) || + isEntityOperationOfType("delete", operation) + ) { + const [, options] = operation.params; + if (options?.expectedVersion) { + output.push(`Expected Version: ${options.expectedVersion}`); } } + if (isEntityOperationOfType("query", operation)) { + const [key] = operation.params; + output.push(`Key: ${JSON.stringify(key)}`); + } } return output; } +function displayEntityTransactItem(item: EntityTransactItem): string[] { + const entityName = + typeof item.entity === "string" ? item.entity : item.entity.name; + if (item.operation === "set") { + return displayEntityCommand({ + operation: "set", + entityName, + params: [item.value, item.options], + }); + } else if (item.operation === "delete") { + return displayEntityCommand({ + operation: "delete", + entityName, + params: [item.key, item.options], + }); + } else { + const output = [ + `Operation: ${item.operation}`, + `Key: ${JSON.stringify(item.key)}`, + ]; + if (item.version !== undefined) { + output.push(`Version: ${item.version}`); + } + return output; + } +} + function displayBucketRequest(request: BucketRequest) { const output: string[] = [`Operation: ${request.operation.operation}`]; output.push(`Bucket: ${request.operation.bucketName}`); diff --git a/packages/@eventual/compiler/src/eventual-infer.ts b/packages/@eventual/compiler/src/eventual-infer.ts index 3375a1b4e..9e17e61f5 100644 --- a/packages/@eventual/compiler/src/eventual-infer.ts +++ b/packages/@eventual/compiler/src/eventual-infer.ts @@ -131,14 +131,8 @@ export function inferFromMemory(openApi: ServiceSpec["openApi"]): ServiceSpec { }, entities: { entities: [...entities().values()].map((d) => ({ - name: d.name, - schema: d.schema ? generateSchema(d.schema) : undefined, - streams: d.streams.map((s) => ({ - name: s.name, - entityName: s.entityName, - options: s.options, - sourceLocation: s.sourceLocation, - })), + ...d, + attributes: generateSchema(d.attributes), })), }, transactions: [...transactions().values()].map((t) => ({ diff --git a/packages/@eventual/core-runtime/src/clients/entity-client.ts b/packages/@eventual/core-runtime/src/clients/entity-client.ts deleted file mode 100644 index 0baf31b97..000000000 --- a/packages/@eventual/core-runtime/src/clients/entity-client.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { - EntityTransactItem, - TransactionCancelled, - UnexpectedVersion, -} from "@eventual/core"; -import { EntityHook, EntityMethods } from "@eventual/core/internal"; -import { - EntityStore, - isTransactionCancelledResult, - isTransactionConflictResult, - isUnexpectedVersionResult, -} from "../stores/entity-store.js"; - -export class EntityClient implements EntityHook { - constructor(private entityStore: EntityStore) {} - public async getEntity( - name: string - ): Promise | undefined> { - return { - get: async (key) => { - const entry = await this.entityStore.getEntityValue(name, key); - return entry?.entity; - }, - getWithMetadata: (key) => - this.entityStore.getEntityValue(name, key), - set: async (key, entity, options) => { - const result = await this.entityStore.setEntityValue( - name, - key, - entity, - options - ); - if (isUnexpectedVersionResult(result)) { - throw new UnexpectedVersion("Unexpected Version"); - } - return result; - }, - delete: async (key, options) => { - const result = await this.entityStore.deleteEntityValue( - name, - key, - options - ); - if (isUnexpectedVersionResult(result)) { - throw new UnexpectedVersion("Unexpected Version"); - } - return result; - }, - list: (request) => this.entityStore.listEntityEntries(name, request), - listKeys: (request) => this.entityStore.listEntityKeys(name, request), - }; - } - - public async transactWrite(items: EntityTransactItem[]): Promise { - const normalizedItems: EntityTransactItem[] = items.map( - (i) => ({ - ...i, - entity: typeof i.entity === "string" ? i.entity : i.entity.name, - }) - ); - const result = await this.entityStore.transactWrite(normalizedItems); - if (isTransactionCancelledResult(result)) { - throw new TransactionCancelled( - result.reasons.map((r) => - isUnexpectedVersionResult(r) - ? new UnexpectedVersion("Unexpected Version") - : undefined - ) - ); - } else if (isTransactionConflictResult(result)) { - throw new TransactionCancelled([]); - } - return result; - } -} diff --git a/packages/@eventual/core-runtime/src/clients/index.ts b/packages/@eventual/core-runtime/src/clients/index.ts index e2748218e..8070d6068 100644 --- a/packages/@eventual/core-runtime/src/clients/index.ts +++ b/packages/@eventual/core-runtime/src/clients/index.ts @@ -1,4 +1,3 @@ -export * from "./entity-client.js"; export * from "./event-client.js"; export * from "./execution-queue-client.js"; export * from "./logs-client.js"; diff --git a/packages/@eventual/core-runtime/src/handlers/transaction-worker.ts b/packages/@eventual/core-runtime/src/handlers/transaction-worker.ts index 1b7b13c2d..4698a9f3a 100644 --- a/packages/@eventual/core-runtime/src/handlers/transaction-worker.ts +++ b/packages/@eventual/core-runtime/src/handlers/transaction-worker.ts @@ -5,13 +5,15 @@ import type { import { transactions } from "@eventual/core/internal"; import type { EventClient } from "../clients/event-client.js"; import type { ExecutionQueueClient } from "../clients/execution-queue-client.js"; -import { isResolved } from "../result.js"; +import type { EntityProvider } from "../providers/entity-provider.js"; +import { isResolved, normalizeFailedResult } from "../result.js"; import type { EntityStore } from "../stores/entity-store.js"; import { createTransactionExecutor } from "../transaction-executor.js"; import { getLazy, LazyValue } from "../utils.js"; export interface TransactionWorkerProps { entityStore: EntityStore; + entityProvider: EntityProvider; executionQueueClient: ExecutionQueueClient; eventClient: EventClient; serviceName: LazyValue; @@ -28,6 +30,7 @@ export function createTransactionWorker( ): TransactionWorker { const transactionExecutor = createTransactionExecutor( props.entityStore, + props.entityProvider, props.executionQueueClient, props.eventClient ); @@ -60,7 +63,7 @@ export function createTransactionWorker( return { output: output.result.value, succeeded: true }; } else { // todo: add reasons - return { succeeded: false }; + return { succeeded: false, ...normalizeFailedResult(output.result) }; } }; } diff --git a/packages/@eventual/core-runtime/src/handlers/utils.ts b/packages/@eventual/core-runtime/src/handlers/utils.ts index 8e77b869b..4664bbd0f 100644 --- a/packages/@eventual/core-runtime/src/handlers/utils.ts +++ b/packages/@eventual/core-runtime/src/handlers/utils.ts @@ -6,13 +6,13 @@ import { registerServiceClient, ServiceSpec, } from "@eventual/core/internal"; -import { EntityClient } from "../clients/entity-client.js"; -import { BucketStore } from "../index.js"; +import type { BucketStore } from "../stores/bucket-store.js"; +import type { EntityStore } from "../stores/entity-store.js"; import { getLazy, LazyValue } from "../utils.js"; export interface WorkerIntrinsicDeps { bucketStore: BucketStore | undefined; - entityClient: EntityClient | undefined; + entityStore: EntityStore | undefined; serviceName: string | LazyValue; serviceClient: EventualServiceClient | undefined; serviceUrls?: (string | LazyValue)[]; @@ -21,8 +21,8 @@ export interface WorkerIntrinsicDeps { } export function registerWorkerIntrinsics(deps: WorkerIntrinsicDeps) { - if (deps.entityClient) { - registerEntityHook(deps.entityClient); + if (deps.entityStore) { + registerEntityHook(deps.entityStore); } if (deps.bucketStore) { registerBucketHook(deps.bucketStore); diff --git a/packages/@eventual/core-runtime/src/local/local-container.ts b/packages/@eventual/core-runtime/src/local/local-container.ts index ac6c06aa7..40fa6915f 100644 --- a/packages/@eventual/core-runtime/src/local/local-container.ts +++ b/packages/@eventual/core-runtime/src/local/local-container.ts @@ -6,7 +6,6 @@ import { EntityStreamRemoveItem, LogLevel, } from "@eventual/core"; -import { EntityClient } from "../clients/entity-client.js"; import { EventClient } from "../clients/event-client.js"; import { ExecutionQueueClient } from "../clients/execution-queue-client.js"; import { LogsClient } from "../clients/logs-client.js"; @@ -39,6 +38,10 @@ import { createTransactionWorker, } from "../handlers/transaction-worker.js"; import { LogAgent } from "../log-agent.js"; +import { + EntityProvider, + GlobalEntityProvider, +} from "../providers/entity-provider.js"; import { InMemoryExecutorProvider } from "../providers/executor-provider.js"; import { GlobalSubscriptionProvider, @@ -125,9 +128,10 @@ export class LocalContainer { public executionStore: ExecutionStore; public taskStore: TaskStore; - public workflowProvider: WorkflowProvider; - public taskProvider: TaskProvider; + public entityProvider: EntityProvider; public subscriptionProvider: SubscriptionProvider; + public taskProvider: TaskProvider; + public workflowProvider: WorkflowProvider; constructor( private localConnector: LocalEnvConnector, @@ -152,16 +156,17 @@ export class LocalContainer { this.taskStore = new LocalTaskStore(); this.subscriptionProvider = props.subscriptionProvider ?? new GlobalSubscriptionProvider(); + this.entityProvider = new GlobalEntityProvider(); const entityStore = new LocalEntityStore({ + entityProvider: this.entityProvider, localConnector: this.localConnector, }); - const entityClient = new EntityClient(entityStore); const bucketStore = new LocalBucketStore({ localConnector: this.localConnector, }); this.bucketHandlerWorker = createBucketNotificationHandlerWorker({ bucketStore, - entityClient, + entityStore, serviceClient: undefined, serviceSpec: undefined, serviceName: props.serviceName, @@ -171,7 +176,7 @@ export class LocalContainer { subscriptionProvider: this.subscriptionProvider, serviceClient: undefined, bucketStore, - entityClient, + entityStore, serviceName: props.serviceName, serviceSpec: undefined, serviceUrl: props.serviceUrl, @@ -186,7 +191,7 @@ export class LocalContainer { this.taskWorker = createTaskWorker({ bucketStore, - entityClient, + entityStore, eventClient: this.eventClient, executionQueueClient: this.executionQueueClient, logAgent, @@ -207,7 +212,7 @@ export class LocalContainer { this.entityStreamWorker = createEntityStreamWorker({ bucketStore, - entityClient, + entityStore, serviceClient: undefined, serviceName: props.serviceName, serviceSpec: undefined, @@ -216,6 +221,7 @@ export class LocalContainer { this.transactionWorker = createTransactionWorker({ entityStore, + entityProvider: this.entityProvider, eventClient: this.eventClient, executionQueueClient: this.executionQueueClient, serviceName: props.serviceName, @@ -226,7 +232,7 @@ export class LocalContainer { this.orchestrator = createOrchestrator({ callExecutor: new WorkflowCallExecutor({ bucketStore, - entityClient, + entityStore, eventClient: this.eventClient, executionQueueClient: this.executionQueueClient, taskClient: this.taskClient, @@ -277,7 +283,7 @@ export class LocalContainer { // must register commands before the command worker is loaded! this.commandWorker = createCommandWorker({ - entityClient, + entityStore, bucketStore, serviceClient: undefined, serviceName: props.serviceName, diff --git a/packages/@eventual/core-runtime/src/local/local-environment.ts b/packages/@eventual/core-runtime/src/local/local-environment.ts index 3ccb1a338..b99fc40fb 100644 --- a/packages/@eventual/core-runtime/src/local/local-environment.ts +++ b/packages/@eventual/core-runtime/src/local/local-environment.ts @@ -151,7 +151,15 @@ export class LocalEnvironment { entityStreamItems.forEach((i) => { const streamNames = [...entities().values()] .flatMap((d) => d.streams) - .filter((s) => entityStreamMatchesItem(i, s)) + .filter((s) => { + const entity = this.localContainer.entityProvider.getEntity( + i.entityName + ); + if (!entity) { + return false; + } + return entityStreamMatchesItem(entity, i, s); + }) .map((s) => s.name); streamNames.forEach((streamName) => { this.localContainer.entityStreamWorker({ diff --git a/packages/@eventual/core-runtime/src/local/stores/entity-store.ts b/packages/@eventual/core-runtime/src/local/stores/entity-store.ts index ea71de0f9..8a4000c49 100644 --- a/packages/@eventual/core-runtime/src/local/stores/entity-store.ts +++ b/packages/@eventual/core-runtime/src/local/stores/entity-store.ts @@ -1,19 +1,25 @@ import { - CompositeKey, + Entity, + Attributes, EntityConsistencyOptions, - EntityListKeysResult, - EntityListRequest, - EntityListResult, + KeyValue, + EntityQueryOptions, + EntityQueryResult, EntitySetOptions, - EntityTransactItem, + EntityWithMetadata, + TransactionCancelled, + UnexpectedVersion, } from "@eventual/core"; import { assertNever } from "@eventual/core/internal"; +import { EntityProvider } from "../../providers/entity-provider.js"; import { EntityStore, - EntityWithMetadata, - TransactionCancelledResult, - UnexpectedVersionResult, - normalizeCompositeKey, + NormalizedEntityCompositeKey, + NormalizedEntityCompositeKeyComplete, + NormalizedEntityKeyCompletePart, + NormalizedEntityTransactItem, + convertNormalizedEntityKeyToMap, + isCompleteKey, } from "../../stores/entity-store.js"; import { deserializeCompositeKey, serializeCompositeKey } from "../../utils.js"; import { LocalEnvConnector } from "../local-container.js"; @@ -21,33 +27,36 @@ import { paginateItems } from "./pagination.js"; export interface LocalEntityStoreProps { localConnector: LocalEnvConnector; + entityProvider: EntityProvider; } -export class LocalEntityStore implements EntityStore { +export class LocalEntityStore extends EntityStore { private entities: Record< string, - Record>> + Map> > = {}; - constructor(private props: LocalEntityStoreProps) {} + constructor(private props: LocalEntityStoreProps) { + super(props.entityProvider); + } - public async getEntityValue( - name: string, - _key: string | CompositeKey - ): Promise | undefined> { - const { key, namespace } = normalizeCompositeKey(_key); - return this.getNamespaceMap(name, namespace).get(key); + protected override async _getWithMetadata( + entity: Entity, + key: NormalizedEntityCompositeKeyComplete + ): Promise { + return this.getPartitionMap(entity, key.partition).get( + key.sort?.keyValue ?? "default" + ); } - public async setEntityValue( - name: string, - _key: string | CompositeKey, + protected override async _set( entity: Entity, + value: Attributes, + key: NormalizedEntityCompositeKeyComplete, options?: EntitySetOptions ): Promise<{ version: number }> { - const { key, namespace } = normalizeCompositeKey(_key); - const { version = 0, entity: oldValue } = - (await this.getEntityValue(name, _key)) ?? {}; + const { version = 0, value: oldValue } = + (await this._getWithMetadata(entity, key)) ?? {}; if ( options?.expectedVersion !== undefined && options.expectedVersion !== version @@ -58,95 +67,104 @@ export class LocalEntityStore implements EntityStore { } const newVersion = options?.incrementVersion === false ? version : version + 1; - this.getNamespaceMap(name, namespace).set(key, { - entity, - version: newVersion, - }); + this.getPartitionMap(entity, key.partition).set( + key.sort?.keyValue ?? "default", + { + value, + version: newVersion, + } + ); this.props.localConnector.pushWorkflowTask({ - entityName: name, - key, - namespace, + entityName: entity.name, + key: convertNormalizedEntityKeyToMap(key), operation: version === 0 ? ("insert" as const) : ("modify" as const), - newValue: entity, + newValue: value, newVersion, oldValue, oldVersion: version, }); - return { version: newVersion }; } - public async deleteEntityValue( - name: string, - _key: string | CompositeKey, - options?: EntityConsistencyOptions - ): Promise { - const { key, namespace } = normalizeCompositeKey(_key); - const item = await this.getEntityValue(name, _key); - + protected override async _delete( + entity: Entity, + key: NormalizedEntityCompositeKeyComplete, + options?: EntityConsistencyOptions | undefined + ): Promise { + const item = await this._getWithMetadata(entity, key); if (item) { if (options?.expectedVersion !== undefined) { if (options.expectedVersion !== item.version) { - return { unexpectedVersion: true }; + throw new UnexpectedVersion("Unexpected Version"); } } - this.getNamespaceMap(name, namespace).delete(key); + if (!isCompleteKey(key)) { + throw new Error("Entity key cannot be partial for delete"); + } + + this.getPartitionMap(entity, key.partition).delete( + key.sort?.keyValue ?? "default" + ); this.props.localConnector.pushWorkflowTask({ - entityName: name, - key, - namespace, + entityName: entity.name, + key: convertNormalizedEntityKeyToMap(key), operation: "remove" as const, - oldValue: item.entity, + oldValue: item.value, oldVersion: item.version, }); } } - public async listEntityEntries( - name: string, - request: EntityListRequest - ): Promise> { - const { items, nextToken } = this.orderedEntries(name, request); + protected override async _query( + entity: Entity, + queryKey: NormalizedEntityCompositeKey, + options?: EntityQueryOptions + ): Promise { + const partition = this.getPartitionMap(entity, queryKey.partition); + const entries = partition ? [...partition.entries()] : []; - // values should be sorted - return { - entries: items?.map(([key, value]) => ({ - key, - entity: value.entity, - version: value.version, - })), - nextToken, - }; - } + const { items, nextToken } = paginateItems( + entries, + (a, b) => + typeof a[0] === "string" + ? a[0].localeCompare(b[0] as string) + : typeof a[0] === "number" + ? a[0] - (b[0] as number) + : 0, + undefined, + undefined, + options?.limit, + options?.nextToken + ); - public async listEntityKeys( - name: string, - request: EntityListRequest - ): Promise { - const { items, nextToken } = this.orderedEntries(name, request); + // values should be sorted return { - keys: items?.map(([key]) => key), + entries: items?.map( + ([, value]) => + ({ + value: value.value, + version: value.version, + } satisfies EntityWithMetadata) + ), nextToken, }; } - public async transactWrite( - items: EntityTransactItem[] - ): Promise { + protected override async _transactWrite( + items: NormalizedEntityTransactItem[] + ): Promise { const keysAndVersions = Object.fromEntries( - items.map( - (i) => - [ - serializeCompositeKey(i.entity, i.operation.key), - i.operation.operation === "condition" - ? i.operation.version - : i.operation.options?.expectedVersion, - ] as const - ) + items.map((item) => { + return [ + serializeCompositeKey(item.entity.name, item.key), + item.operation === "condition" + ? item.version + : item.options?.expectedVersion, + ] as const; + }) ); - /** * Evaluate the expected versions against the current state and return the results. * @@ -158,75 +176,58 @@ export class LocalEntityStore implements EntityStore { if (expectedVersion === undefined) { return true; } - const [name, key] = deserializeCompositeKey(sKey); - const { version } = (await this.getEntityValue(name, key)) ?? { + const [entityName, key] = deserializeCompositeKey(sKey); + const { version } = (await this.getWithMetadata(entityName, key)) ?? { version: 0, }; return version === expectedVersion; }) ); - if (consistencyResults.some((r) => !r)) { - return { - reasons: consistencyResults.map((r) => - r ? undefined : { unexpectedVersion: true } - ), - }; + throw new TransactionCancelled( + consistencyResults.map((r) => + r ? undefined : new UnexpectedVersion("Unexpected Version") + ) + ); } - /** * After ensuring that all of the expected versions are accurate, actually perform the writes. * Here we assume that the write operations are synchronous and that * the state of the condition checks will not be invalided. */ await Promise.all( - items.map(async (i) => { - if (i.operation.operation === "set") { - return await this.setEntityValue( - i.entity, - i.operation.key, - i.operation.value, - i.operation.options + items.map(async (item) => { + if (item.operation === "set") { + return await this._set( + item.entity, + item.value, + item.key, + item.options ); - } else if (i.operation.operation === "delete") { - return await this.deleteEntityValue( - i.entity, - i.operation.key, - i.operation.options - ); - } else if (i.operation.operation === "condition") { + } else if (item.operation === "delete") { + return await this._delete(item.entity, item.key, item.options); + } else if (item.operation === "condition") { // no op return; } - return assertNever(i.operation); + return assertNever(item); }) ); } - private orderedEntries(name: string, listRequest: EntityListRequest) { - const namespace = this.getNamespaceMap(name, listRequest.namespace); - const entries = namespace ? [...namespace.entries()] : []; - - const result = paginateItems( - entries, - (a, b) => a[0].localeCompare(b[0]), - listRequest.prefix - ? ([key]) => key.startsWith(listRequest.prefix!) - : undefined, - undefined, - listRequest.limit, - listRequest.nextToken - ); - - return result; - } - - private getNamespaceMap(name: string, namespace?: string) { - const entity = (this.entities[name] ??= {}); - const namespaceMap = (entity[namespace ?? "default"] ??= new Map< - string, - EntityWithMetadata + private getPartitionMap( + entity: Entity, + partitionKey: NormalizedEntityKeyCompletePart + ) { + const _entity = (this.entities[entity.name] ??= new Map< + KeyValue, + Map >()); - return namespaceMap; + let partitionMap = _entity.get(partitionKey.keyValue); + if (!partitionMap) { + partitionMap = new Map(); + _entity.set(partitionKey.keyValue, partitionMap); + } + return partitionMap; } } diff --git a/packages/@eventual/core-runtime/src/providers/entity-provider.ts b/packages/@eventual/core-runtime/src/providers/entity-provider.ts new file mode 100644 index 000000000..3733e7f8f --- /dev/null +++ b/packages/@eventual/core-runtime/src/providers/entity-provider.ts @@ -0,0 +1,21 @@ +import type { Entity } from "@eventual/core"; +import { entities } from "@eventual/core/internal"; +import type { WorkflowExecutor } from "../workflow-executor.js"; + +export interface EntityProvider { + /** + * Returns an executor which may already be started. + * + * Use {@link WorkflowExecutor}.isStarted to determine if it is already started. + */ + getEntity(entityName: string): Entity | undefined; +} + +/** + * An executor provider that works with an out of memory store. + */ +export class GlobalEntityProvider implements EntityProvider { + public getEntity(entityName: string): Entity | undefined { + return entities().get(entityName); + } +} diff --git a/packages/@eventual/core-runtime/src/providers/index.ts b/packages/@eventual/core-runtime/src/providers/index.ts index 6791d29b4..ef2029583 100644 --- a/packages/@eventual/core-runtime/src/providers/index.ts +++ b/packages/@eventual/core-runtime/src/providers/index.ts @@ -1,3 +1,4 @@ +export * from "./entity-provider.js"; export * from "./executor-provider.js"; export * from "./subscription-provider.js"; export * from "./task-provider.js"; diff --git a/packages/@eventual/core-runtime/src/providers/workflow-provider.ts b/packages/@eventual/core-runtime/src/providers/workflow-provider.ts index e8477fbe5..a1de74d33 100644 --- a/packages/@eventual/core-runtime/src/providers/workflow-provider.ts +++ b/packages/@eventual/core-runtime/src/providers/workflow-provider.ts @@ -3,7 +3,6 @@ import { workflows, ServiceSpec } from "@eventual/core/internal"; export interface WorkflowProvider extends WorkflowSpecProvider { lookupWorkflow(workflowName: string): Workflow | undefined; - getWorkflowNames(): string[]; } export interface WorkflowSpecProvider { @@ -20,11 +19,9 @@ export class GlobalWorkflowProvider implements WorkflowProvider { public lookupWorkflow(workflowName: string): Workflow | undefined { return workflows().get(workflowName); } - public getWorkflowNames(): string[] { return [...workflows().keys()]; } - public workflowExists(workflowName: string): boolean { return !!this.lookupWorkflow(workflowName); } diff --git a/packages/@eventual/core-runtime/src/stores/entity-store.ts b/packages/@eventual/core-runtime/src/stores/entity-store.ts index 65825ef09..50001d77f 100644 --- a/packages/@eventual/core-runtime/src/stores/entity-store.ts +++ b/packages/@eventual/core-runtime/src/stores/entity-store.ts @@ -1,77 +1,308 @@ import type { + Entity, + Attributes, CompositeKey, EntityConsistencyOptions, - EntityListKeysResult, - EntityListRequest, - EntityListResult, + KeyMap, + KeyValue, + QueryKey, + EntityQueryOptions, + EntityQueryResult, EntitySetOptions, EntityTransactItem, + EntityWithMetadata, } from "@eventual/core"; +import type { + EntityHook, + KeyDefinition, + KeyDefinitionPart, +} from "@eventual/core/internal"; +import { EntityProvider } from "../providers/entity-provider.js"; + +export abstract class EntityStore implements EntityHook { + constructor(private entityProvider: EntityProvider) {} + + public async get(entityName: string, key: CompositeKey): Promise { + return (await this.getWithMetadata(entityName, key))?.value; + } + + public async getWithMetadata( + entityName: string, + key: CompositeKey + ): Promise { + const entity = this.getEntity(entityName); + const normalizedCompositeKey = normalizeCompositeKey(entity, key); + + if (!isCompleteKey(normalizedCompositeKey)) { + throw new Error("Key cannot be partial for get or getWithMetadata."); + } -export interface EntityStore { - getEntityValue( - name: string, - key: string | CompositeKey - ): Promise | undefined>; - setEntityValue( - name: string, - key: string | CompositeKey, + return this._getWithMetadata(entity, normalizedCompositeKey); + } + + protected abstract _getWithMetadata( entity: Entity, + key: NormalizedEntityCompositeKeyComplete + ): Promise; + + public set( + entityName: string, + value: Attributes, options?: EntitySetOptions - ): Promise<{ version: number } | UnexpectedVersionResult>; - deleteEntityValue( - name: string, - key: string | CompositeKey, - options?: EntityConsistencyOptions - ): Promise; - listEntityEntries( - name: string, - request: EntityListRequest - ): Promise>; - listEntityKeys( - name: string, - request: EntityListRequest - ): Promise; - transactWrite( - items: EntityTransactItem[] - ): Promise; + ): Promise<{ version: number }> { + const entity = this.getEntity(entityName); + const normalizedKey = normalizeCompositeKey(entity, value); + + if (!isCompleteKey(normalizedKey)) { + throw new Error("Key cannot be partial for set."); + } + + return this._set(entity, value, normalizedKey, options); + } + + protected abstract _set( + entity: Entity, + value: Attributes, + key: NormalizedEntityCompositeKeyComplete, + options?: EntitySetOptions + ): Promise<{ version: number }>; + + public delete( + entityName: string, + key: CompositeKey, + options?: EntityConsistencyOptions | undefined + ): Promise { + const entity = this.getEntity(entityName); + const normalizedKey = normalizeCompositeKey(entity, key); + + if (!isCompleteKey(normalizedKey)) { + throw new Error("Key cannot be partial for delete."); + } + + return this._delete(entity, normalizedKey, options); + } + + protected abstract _delete( + entity: Entity, + key: NormalizedEntityCompositeKeyComplete, + options?: EntityConsistencyOptions | undefined + ): Promise; + + public query( + entityName: string, + queryKey: QueryKey, + options?: EntityQueryOptions | undefined + ): Promise { + const entity = this.getEntity(entityName); + const normalizedKey = normalizeCompositeKey(entity, queryKey); + + if (!isCompleteKeyPart(normalizedKey.partition)) { + throw new Error("Entity partition key cannot be partial for query"); + } + + return this._query( + entity, + normalizedKey as NormalizedEntityCompositeKey, + options + ); + } + + protected abstract _query( + entity: Entity, + queryKey: NormalizedEntityCompositeKey, + options: EntityQueryOptions | undefined + ): Promise; + + public async transactWrite(items: EntityTransactItem[]): Promise { + return this._transactWrite( + items.map((item): NormalizedEntityTransactItem => { + const entity = + typeof item.entity === "string" + ? this.getEntity(item.entity) + : item.entity; + const keyValue = item.operation === "set" ? item.value : item.key; + const key = normalizeCompositeKey(entity, keyValue); + if (!isCompleteKey(key)) { + throw new Error( + "Entity key cannot be partial for set, delete, or condition operations." + ); + } + + return item.operation === "set" + ? { + operation: "set", + entity, + key, + value: item.value, + options: item.options, + } + : item.operation === "delete" + ? { + operation: "delete", + entity, + key, + options: item.options, + } + : { + operation: "condition", + entity, + key, + version: item.version, + }; + }) + ); + } + + protected abstract _transactWrite( + items: NormalizedEntityTransactItem[] + ): Promise; + + protected getEntity(entityName: string) { + const entity = this.entityProvider.getEntity(entityName); + + if (!entity) { + throw new Error(`Entity ${entityName} was not found.`); + } + return entity; + } } -export interface EntityWithMetadata { +export type NormalizedEntityTransactItem = { entity: Entity; - version: number; + key: NormalizedEntityCompositeKeyComplete; +} & ( + | { + operation: "set"; + value: Attributes; + options?: EntitySetOptions; + } + | { + operation: "delete"; + options?: EntityConsistencyOptions; + } + | { + operation: "condition"; + version?: number; + } +); + +export interface NormalizedEntityKeyPartBase extends KeyDefinitionPart { + parts: { field: string; value: KeyValue }[]; } -export interface UnexpectedVersionResult { - unexpectedVersion: true; +export type NormalizedEntityKeyPart = + | NormalizedEntityKeyPartialPart + | NormalizedEntityKeyCompletePart; + +export interface NormalizedEntityKeyCompletePart + extends NormalizedEntityKeyPartBase { + keyValue: string | number; + partialValue: false; } -export interface TransactionCancelledResult { - reasons: (UnexpectedVersionResult | undefined)[]; +export interface NormalizedEntityKeyPartialPart + extends NormalizedEntityKeyPartBase { + keyValue?: string | number; + partialValue: true; } -export interface TransactionConflictResult { - transactionConflict: true; +export function isCompleteKeyPart( + key: NormalizedEntityKeyPart +): key is NormalizedEntityKeyCompletePart { + return !key.partialValue; } -export function isUnexpectedVersionResult( - value: any -): value is UnexpectedVersionResult { - return value && "unexpectedVersion" in value; +export function isCompleteKey( + key: NormalizedEntityCompositeKey +): key is NormalizedEntityCompositeKey< + NormalizedEntityKeyCompletePart, + NormalizedEntityKeyCompletePart +> { + return ( + isCompleteKeyPart(key.partition) && + (!key.sort || isCompleteKeyPart(key.sort)) + ); } -export function isTransactionCancelledResult( - value: any -): value is TransactionCancelledResult { - return value && "reasons" in value; +export interface NormalizedEntityCompositeKey< + Partition extends NormalizedEntityKeyPart = NormalizedEntityKeyPart, + Sort extends NormalizedEntityKeyPart = NormalizedEntityKeyPart +> { + partition: Partition; + sort?: Sort; } -export function isTransactionConflictResult( - value: any -): value is TransactionConflictResult { - return value && "transactionConflict" in value; +export type NormalizedEntityCompositeKeyComplete = NormalizedEntityCompositeKey< + NormalizedEntityKeyCompletePart, + NormalizedEntityKeyCompletePart +>; + +/** + * Generate properties for an entity key given the key definition and key values. + */ +export function normalizeCompositeKey( + entity: E | KeyDefinition, + key: Partial +): NormalizedEntityCompositeKey { + const keyDef = "partition" in entity ? entity : entity.key; + + const partitionCompositeKey = formatNormalizedPart(keyDef.partition, (p, i) => + Array.isArray(key) ? key[i] : (key as KeyMap)[p] + ); + + const sortCompositeKey = keyDef.sort + ? formatNormalizedPart(keyDef.sort, (p, i) => + Array.isArray(key) + ? key[keyDef.partition.attributes.length + i] + : (key as KeyMap)[p] + ) + : undefined; + + return sortCompositeKey + ? { + partition: partitionCompositeKey, + sort: sortCompositeKey, + } + : { + partition: partitionCompositeKey, + }; +} + +function formatNormalizedPart( + keyPart: KeyDefinitionPart, + valueRetriever: (field: string, index: number) => string | number +): NormalizedEntityKeyPart { + const parts = keyPart.attributes.map((p, i) => ({ + field: p, + value: valueRetriever(p, i), + })); + + const missingValueIndex = parts.findIndex((p) => p.value === undefined); + + return { + type: keyPart.type, + attributes: keyPart.attributes, + parts, + keyAttribute: keyPart.keyAttribute, + keyValue: (keyPart.type === "number" + ? parts[0]?.value + : (missingValueIndex === -1 ? parts : parts.slice(0, missingValueIndex)) + .map((p) => p.value) + .join("#")) as any, + partialValue: missingValueIndex !== -1, + }; } -export function normalizeCompositeKey(key: string | CompositeKey) { - return typeof key === "string" ? { key, namespace: undefined } : key; +export function convertNormalizedEntityKeyToMap( + key: NormalizedEntityCompositeKey +): KeyMap { + console.log("input key", JSON.stringify(key)); + const generatedKey = Object.fromEntries([ + ...key.partition.parts.map(({ field, value }) => [field, value]), + ...(key.sort + ? key.sort.parts.map(({ field, value }) => [field, value]) + : []), + ]); + console.log("generated key", JSON.stringify(generatedKey)); + return generatedKey; } diff --git a/packages/@eventual/core-runtime/src/system-commands.ts b/packages/@eventual/core-runtime/src/system-commands.ts index 92ca97d41..ad4a02985 100644 --- a/packages/@eventual/core-runtime/src/system-commands.ts +++ b/packages/@eventual/core-runtime/src/system-commands.ts @@ -1,8 +1,9 @@ import { AnyCommand, api, command, HttpResponse } from "@eventual/core"; import { assertNever, - EVENTUAL_SYSTEM_COMMAND_NAMESPACE, + emitEventsRequestSchema, EventualService, + EVENTUAL_SYSTEM_COMMAND_NAMESPACE, executeTransactionRequestSchema, extendsError, isSendTaskFailureRequest, @@ -10,7 +11,6 @@ import { isSendTaskSuccessRequest, listExecutionEventsRequestSchema, listExecutionsRequestSchema, - emitEventsRequestSchema, sendSignalRequestSchema, sendTaskUpdateSchema, startExecutionRequestSchema, @@ -20,8 +20,8 @@ import { z } from "zod"; import type { EventClient } from "./clients/event-client.js"; import type { ExecutionQueueClient } from "./clients/execution-queue-client.js"; import type { TaskClient } from "./clients/task-client.js"; +import type { TransactionClient } from "./clients/transaction-client.js"; import type { WorkflowClient } from "./clients/workflow-client.js"; -import type { TransactionClient } from "./index.js"; import type { WorkflowSpecProvider } from "./providers/workflow-provider.js"; import type { ExecutionHistoryStateStore } from "./stores/execution-history-state-store.js"; import type { ExecutionHistoryStore } from "./stores/execution-history-store.js"; diff --git a/packages/@eventual/core-runtime/src/transaction-executor.ts b/packages/@eventual/core-runtime/src/transaction-executor.ts index f2aca6fb9..ae0cf75e4 100644 --- a/packages/@eventual/core-runtime/src/transaction-executor.ts +++ b/packages/@eventual/core-runtime/src/transaction-executor.ts @@ -1,15 +1,21 @@ -import type { - CompositeKey, +import { Entity, + KeyMap, + EntityTransactConditionalOperation, + EntityTransactDeleteOperation, + EntityTransactSetOperation, EntityTransactItem, + EntityWithMetadata, + TransactionCancelled, + TransactionConflict, TransactionContext, TransactionFunction, + UnexpectedVersion, } from "@eventual/core"; import { assertNever, EmitEventsCall, - EntityDeleteOperation, - EntitySetOperation, + EntityOperation, EventualCallHook, EventualPromise, EventualPromiseSymbol, @@ -26,14 +32,13 @@ import { import type { EventClient } from "./clients/event-client.js"; import type { ExecutionQueueClient } from "./clients/execution-queue-client.js"; import { enterEventualCallHookScope } from "./eventual-hook.js"; +import type { EntityProvider } from "./providers/entity-provider.js"; import { isResolved } from "./result.js"; import { + convertNormalizedEntityKeyToMap, EntityStore, - EntityWithMetadata, - isTransactionCancelledResult, - isTransactionConflictResult, - isUnexpectedVersionResult, normalizeCompositeKey, + NormalizedEntityCompositeKey, } from "./stores/entity-store.js"; import { serializeCompositeKey } from "./utils.js"; @@ -84,8 +89,14 @@ export interface TransactionExecutor { ): Promise>; } +interface TransactionFailedItem { + entityName: string; + key: KeyMap; +} + export function createTransactionExecutor( entityStore: EntityStore, + entityProvider: EntityProvider, executionQueueClient: ExecutionQueueClient, eventClient: EventClient ): TransactionExecutor { @@ -95,42 +106,41 @@ export function createTransactionExecutor( transactionContext: TransactionContext, retries = 3 ) { - // retry the transaction until it completes, there is an explicit conflict, or we run out of retries. - do { - const result = await executeTransactionOnce(); - if ("output" in result) { - return { result: Result.resolved(result.output) }; - } else if (result.canRetry) { - continue; - } else { - return { - result: Result.failed( - new Error("Failed after an explicit conflict.") - ), - }; - } - } while (retries--); + try { + // retry the transaction until it completes, there is an explicit conflict, or we run out of retries. + do { + const result = await executeTransactionOnce(); + if ("output" in result) { + return { result: Result.resolved(result.output) }; + } else if (result.canRetry) { + continue; + } else { + return { + result: Result.failed( + new Error("Failed after an explicit conflict.") + ), + }; + } + } while (retries--); - return { - result: Result.failed(new Error("Failed after too many retires.")), - }; + return { + result: Result.failed(new Error("Failed after too many retires.")), + }; + } catch (err) { + return { + result: Result.failed(err), + }; + } async function executeTransactionOnce(): Promise< | { output: Output } | { canRetry: boolean; - failedItems: { - entityName: string; - key: string; - namespace?: string; - }[]; + failedItems: TransactionFailedItem[]; } > { // a map of the keys of all mutable entity calls that have been made to the request - const entityCalls = new Map< - string, - EntitySetOperation | EntityDeleteOperation - >(); + const entityCalls = new Map>(); // store all of the event and signal calls to execute after the transaction completes const eventCalls: (EmitEventsCall | SendSignalCall)[] = []; // a map of the keys of all get operations or mutation operations to check during the transaction. @@ -139,30 +149,37 @@ export function createTransactionExecutor( string, { entityName: string; - key: string | CompositeKey; - value: EntityWithMetadata | undefined; + key: NormalizedEntityCompositeKey; + value: EntityWithMetadata | undefined; } >(); const eventualCallHook: EventualCallHook = { - registerEventualCall: (eventual) => { + registerEventualCall: (eventual): any => { if (isEntityCall(eventual)) { if ( isEntityOperationOfType("set", eventual) || isEntityOperationOfType("delete", eventual) ) { return createEventualPromise< - Awaited["delete"] | Entity["set"]>> + Awaited> >(async () => { - const entity = await resolveEntity(eventual.name, eventual.key); - const normalizedKey = serializeCompositeKey( - eventual.name, - eventual.key + const entity = getEntity(eventual.entityName); + // should either by the key or the value object, which can be used as the key + const key = eventual.params[0]; + const normalizedKey = normalizeCompositeKey(entity, key); + const entityValue = await resolveEntity( + entity.name, + normalizedKey + ); + const serializedKey = serializeCompositeKey( + entity.name, + normalizedKey ); - entityCalls.set(normalizedKey, eventual); + entityCalls.set(serializedKey, eventual); return isEntityOperationOfType("set", eventual) - ? { version: (entity?.version ?? 0) + 1 } + ? { version: (entityValue?.version ?? 0) + 1 } : undefined; }); } else if ( @@ -170,10 +187,15 @@ export function createTransactionExecutor( isEntityOperationOfType("getWithMetadata", eventual) ) { return createEventualPromise(async () => { - const value = await resolveEntity(eventual.name, eventual.key); + const entity = getEntity(eventual.entityName); + const key = eventual.params[0]; + const value = await resolveEntity( + entity.name, + normalizeCompositeKey(entity, key) + ); if (isEntityOperationOfType("get", eventual)) { - return value?.entity; + return value?.value; } else if ( isEntityOperationOfType("getWithMetadata", eventual) ) { @@ -235,124 +257,158 @@ export function createTransactionExecutor( /** * Build the transaction items that contain mutations with assertions or just assertions. */ - const transactionItems: EntityTransactItem[] = [ + const transactionItems: EntityTransactItem[] = [ ...retrievedEntities.entries(), - ].map(([normalizedKey, { entityName, key, value }], i) => { - const call = entityCalls.get(normalizedKey); + ].map(([serializedKey, { entityName, key, value }], i) => { + const call = entityCalls.get(serializedKey); const retrievedVersion = value?.version ?? 0; if (call) { + const [, options] = call.params; // if the user provided a version that was not the same that was retrieved // we will consider the transaction not retry-able on failure. // for example, if an entity is set with an expected version of 1, // but the current version at set time is 2, this condition /// will never be true. if ( - call.options?.expectedVersion !== undefined && - call.options?.expectedVersion !== retrievedVersion + options?.expectedVersion !== undefined && + options?.expectedVersion !== retrievedVersion ) { versionOverridesIndices.add(i); - return { entity: entityName, operation: call }; } - return { - entity: entityName, - operation: { - ...call, - options: { - ...call.options, - expectedVersion: retrievedVersion, - }, - }, - }; + + return call.operation === "set" + ? ({ + entity: entityName, + operation: "set", + value: call.params[0], + options: { + ...options, + expectedVersion: options?.expectedVersion ?? retrievedVersion, + }, + } satisfies EntityTransactSetOperation) + : ({ + entity: entityName, + operation: "delete", + key: call.params[0], + options: { + ...options, + expectedVersion: options?.expectedVersion ?? retrievedVersion, + }, + } satisfies EntityTransactDeleteOperation); } else { // values that are retrieved only, will be checked using a condition return { entity: entityName, - operation: { - operation: "condition", - key, - version: retrievedVersion, - }, - }; + operation: "condition", + key: convertNormalizedEntityKeyToMap(key), + version: retrievedVersion, + } satisfies EntityTransactConditionalOperation; } }); console.log(JSON.stringify(transactionItems, undefined, 4)); - /** - * Run the transaction - */ - const result = - transactionItems.length > 0 - ? await entityStore.transactWrite(transactionItems) - : undefined; + try { + /** + * Run the transaction + */ + const result = + transactionItems.length > 0 + ? await entityStore.transactWrite(transactionItems) + : undefined; + + console.log(JSON.stringify(result, undefined, 4)); + } catch (err) { + /** + * If the transaction failed, check if it is retryable or not. + */ - console.log(JSON.stringify(result, undefined, 4)); + if (err instanceof TransactionCancelled) { + const retry = !err.reasons.some((r, i) => + r instanceof UnexpectedVersion + ? versionOverridesIndices.has(i) + : false + ); + return { + canRetry: retry, + failedItems: err.reasons + .map((r, i) => { + if (r instanceof UnexpectedVersion) { + const x: EntityTransactItem = transactionItems[i]!; + const entity = + typeof x.entity === "string" + ? getEntity(x.entity) + : x.entity; + // normalize the key to extract only the key fields. + const key = normalizeCompositeKey( + entity, + x.operation === "set" ? x.value : x.key + ); + return { + entityName: entity.name, + // convert back to a map to send to the caller + key: convertNormalizedEntityKeyToMap(key), + } satisfies TransactionFailedItem; + } + return undefined; + }) + .filter((i): i is Exclude => !!i), + }; + } else if (err instanceof TransactionConflict) { + return { canRetry: true, failedItems: [] }; + } + } /** - * If the transaction failed, check if it is retryable or not. + * If the transaction succeeded, emit events and send signals. + * TODO: move the side effects to a transactional dynamo update. */ - if (isTransactionCancelledResult(result)) { - const retry = !result.reasons.some((r, i) => - isUnexpectedVersionResult(r) ? versionOverridesIndices.has(i) : false - ); - return { - canRetry: retry, - failedItems: result.reasons - .map((r, i) => { - if (isUnexpectedVersionResult(r)) { - const x = transactionItems[i]!; - const { key, namespace } = normalizeCompositeKey( - x.operation.key - ); - return { entityName: x.entity, key, namespace }; - } - return undefined; - }) - .filter((i): i is Exclude => !!i), - }; - } else if (isTransactionConflictResult(result)) { - return { canRetry: true, failedItems: [] }; - } else { - /** - * If the transaction succeeded, emit events and send signals. - * TODO: move the side effects to a transactional dynamo update. - */ - await Promise.allSettled( - eventCalls.map(async (call) => { - if (isEmitEventsCall(call)) { - await eventClient.emitEvents(...call.events); - } else if (call) { - // shouldn't happen - if (call.target.type === SignalTargetType.ChildExecution) { - return; - } - await executionQueueClient.sendSignal({ - execution: call.target.executionId, - signal: call.signalId, - payload: call.payload, - id: call.id, - }); + await Promise.allSettled( + eventCalls.map(async (call) => { + if (isEmitEventsCall(call)) { + await eventClient.emitEvents(...call.events); + } else if (call) { + // shouldn't happen + if (call.target.type === SignalTargetType.ChildExecution) { + return; } - }) - ); - } + await executionQueueClient.sendSignal({ + execution: call.target.executionId, + signal: call.signalId, + payload: call.payload, + id: call.id, + }); + } + }) + ); return { output }; + function getEntity(entityName: string) { + const entity = entityProvider.getEntity(entityName); + if (!entity) { + throw new Error(`Entity ${entityName} was not found.`); + } + return entity; + } + function resolveEntity( entityName: string, - key: string | CompositeKey - ): EventualPromise | undefined> { - const normalizedKey = serializeCompositeKey(entityName, key); - if (retrievedEntities.has(normalizedKey)) { + key: NormalizedEntityCompositeKey + ): EventualPromise { + const serializedKey = serializeCompositeKey(entityName, key); + if (retrievedEntities.has(serializedKey)) { return createResolvedEventualPromise( - Result.resolved(retrievedEntities.get(normalizedKey)?.value) + Result.resolved(retrievedEntities.get(serializedKey)?.value) ); } else { return createEventualPromise(async () => { - const value = await entityStore.getEntityValue(entityName, key); - retrievedEntities.set(normalizedKey, { + const value = await entityStore.getWithMetadata( + entityName, + convertNormalizedEntityKeyToMap(key) + ); + retrievedEntities.set(serializedKey, { entityName, key, value, diff --git a/packages/@eventual/core-runtime/src/utils.ts b/packages/@eventual/core-runtime/src/utils.ts index 40b0b2e0c..4c6091af9 100644 --- a/packages/@eventual/core-runtime/src/utils.ts +++ b/packages/@eventual/core-runtime/src/utils.ts @@ -1,13 +1,20 @@ -import { +import type { BucketNotificationEvent, - CompositeKey, + Entity, + Attributes, + CompositeKeyPart, + KeyTuple, EntityStreamItem, } from "@eventual/core"; -import { +import type { BucketNotificationHandlerSpec, EntityStreamSpec, } from "@eventual/core/internal"; -import { normalizeCompositeKey } from "./stores/entity-store.js"; +import { + NormalizedEntityCompositeKey, + NormalizedEntityKeyPart, + normalizeCompositeKey, +} from "./stores/entity-store.js"; export async function promiseAllSettledPartitioned( items: T[], @@ -60,17 +67,16 @@ export function getLazy( export function serializeCompositeKey( entityName: string, - _key: string | CompositeKey + key: NormalizedEntityCompositeKey ) { - const { key, namespace } = normalizeCompositeKey(_key); - return `${entityName}|${namespace ?? ""}|${key}`; + return `${entityName}|${key.partition.keyValue}|${key.sort?.keyValue ?? ""}`; } export function deserializeCompositeKey( sKey: string -): [string, string | CompositeKey] { - const [name, namespace, key] = sKey.split("|") as [string, string, string]; - return [name, namespace ? { key, namespace } : key]; +): [string, KeyTuple] { + const [name, partition, sort] = sKey.split("|") as [string, string, string]; + return [name, sort ? [partition, sort] : [partition]]; } export function isEntityStreamItem(value: any): value is EntityStreamItem { @@ -83,25 +89,48 @@ export function isBucketNotificationEvent( return "bucketName" in value && "event" in value; } -export function entityStreamMatchesItem( - item: EntityStreamItem, - streamSpec: EntityStreamSpec +export function entityStreamMatchesItem< + Attr extends Attributes, + const Partition extends CompositeKeyPart, + const Sort extends CompositeKeyPart | undefined +>( + entity: Entity, + item: EntityStreamItem, + streamSpec: EntityStreamSpec ) { + const { partition, sort } = normalizeCompositeKey(entity, item.key); + const normalizedQueryKeys = + streamSpec.options?.queryKeys?.map((key) => + normalizeCompositeKey(entity, key) + ) ?? []; return ( streamSpec.entityName === item.entityName && (!streamSpec.options?.operations || streamSpec.options.operations.includes(item.operation)) && - (!streamSpec.options?.namespaces || - (item.namespace && - streamSpec.options.namespaces.includes(item.namespace))) && - (!streamSpec.options?.namespacePrefixes || - (item.namespace && - streamSpec.options.namespacePrefixes.some((p) => - item.namespace?.startsWith(p) - ))) + (normalizedQueryKeys.length === 0 || + normalizedQueryKeys.some( + (k) => + // if the query key exists, it will have at least a partial partition key + compareNormalizedEntityKeyPart(partition, k.partition) && + // if there is a sort part to the query key, there must be a sort key to the value + (!k.sort || (sort && compareNormalizedEntityKeyPart(sort, k.sort))) + )) ); } +function compareNormalizedEntityKeyPart( + value: NormalizedEntityKeyPart, + matcher: NormalizedEntityKeyPart +) { + return matcher.partialValue + ? // the matcher is a partial value and both matcher and value are string + typeof value.keyValue === "string" && + typeof matcher.keyValue === "string" && + value.keyValue.startsWith(matcher.keyValue) + : // matcher is not partial, compare the two + matcher.keyValue === value.keyValue; +} + export function bucketHandlerMatchesEvent( item: BucketNotificationEvent, streamSpec: BucketNotificationHandlerSpec diff --git a/packages/@eventual/core-runtime/src/workflow-call-executor.ts b/packages/@eventual/core-runtime/src/workflow-call-executor.ts index 7f99cdb1a..65db829c1 100644 --- a/packages/@eventual/core-runtime/src/workflow-call-executor.ts +++ b/packages/@eventual/core-runtime/src/workflow-call-executor.ts @@ -1,4 +1,4 @@ -import { ExecutionID, Workflow } from "@eventual/core"; +import type { ExecutionID, Workflow } from "@eventual/core"; import { assertNever, AwaitTimerCall, @@ -44,23 +44,23 @@ import { WorkflowEventType, } from "@eventual/core/internal"; import stream from "stream"; -import { EntityClient } from "./clients/entity-client.js"; -import { EventClient } from "./clients/event-client.js"; -import { ExecutionQueueClient } from "./clients/execution-queue-client.js"; -import { TaskClient, TaskWorkerRequest } from "./clients/task-client.js"; -import { TimerClient } from "./clients/timer-client.js"; -import { TransactionClient } from "./clients/transaction-client.js"; -import { WorkflowClient } from "./clients/workflow-client.js"; +import type { EventClient } from "./clients/event-client.js"; +import type { ExecutionQueueClient } from "./clients/execution-queue-client.js"; +import type { TaskClient, TaskWorkerRequest } from "./clients/task-client.js"; +import type { TimerClient } from "./clients/timer-client.js"; +import type { TransactionClient } from "./clients/transaction-client.js"; +import type { WorkflowClient } from "./clients/workflow-client.js"; import { formatChildExecutionName, formatExecutionId } from "./execution.js"; -import { BucketStore } from "./index.js"; import { normalizeError } from "./result.js"; import { computeScheduleDate } from "./schedule.js"; +import type { BucketStore } from "./stores/bucket-store.js"; +import type { EntityStore } from "./stores/entity-store.js"; import { createEvent } from "./workflow-events.js"; -import { WorkflowCall } from "./workflow-executor.js"; +import type { WorkflowCall } from "./workflow-executor.js"; interface WorkflowCallExecutorProps { bucketStore: BucketStore; - entityClient: EntityClient; + entityStore: EntityStore; eventClient: EventClient; executionQueueClient: ExecutionQueueClient; taskClient: TaskClient; @@ -274,7 +274,7 @@ export class WorkflowCallExecutor { operation: call.operation, name: isEntityOperationOfType("transact", call) ? undefined - : call.name, + : call.entityName, result, seq, }, @@ -290,7 +290,7 @@ export class WorkflowCallExecutor { seq, name: isEntityOperationOfType("transact", call) ? undefined - : call.name, + : call.entityName, operation: call.operation, ...normalizeError(err), }, @@ -310,26 +310,13 @@ export class WorkflowCallExecutor { async function invokeEntityOperation(operation: EntityOperation) { if (isEntityOperationOfType("transact", operation)) { - return self.props.entityClient.transactWrite(operation.items); + return self.props.entityStore.transactWrite(operation.items); } - const entity = await self.props.entityClient.getEntity(operation.name); - if (!entity) { - throw new Error(`Entity ${operation.name} does not exist`); - } - if (isEntityOperationOfType("get", operation)) { - return entity.get(operation.key); - } else if (isEntityOperationOfType("getWithMetadata", operation)) { - return entity.getWithMetadata(operation.key); - } else if (isEntityOperationOfType("set", operation)) { - return entity.set(operation.key, operation.value, operation.options); - } else if (isEntityOperationOfType("delete", operation)) { - return entity.delete(operation.key, operation.options); - } else if (isEntityOperationOfType("list", operation)) { - return entity.list(operation.request); - } else if (isEntityOperationOfType("listKeys", operation)) { - return entity.listKeys(operation.request); - } - return assertNever(operation); + return self.props.entityStore[operation.operation]( + operation.entityName, + // @ts-ignore + ...operation.params + ); } } diff --git a/packages/@eventual/core-runtime/test/command-executor.test.ts b/packages/@eventual/core-runtime/test/command-executor.test.ts index 5746b6a0a..8ae88f1aa 100644 --- a/packages/@eventual/core-runtime/test/command-executor.test.ts +++ b/packages/@eventual/core-runtime/test/command-executor.test.ts @@ -1,5 +1,4 @@ import { - Entity, EventEnvelope, Schedule, SendSignalRequest, @@ -7,7 +6,6 @@ import { } from "@eventual/core"; import { ChildWorkflowScheduled, - EntityMethods, EntityRequest, EventsEmitted, SignalSent, @@ -18,7 +16,6 @@ import { WorkflowEventType, } from "@eventual/core/internal"; import { jest } from "@jest/globals"; -import { EntityClient } from "../src/clients/entity-client.js"; import { EventClient } from "../src/clients/event-client.js"; import { ExecutionQueueClient } from "../src/clients/execution-queue-client.js"; import { TaskClient } from "../src/clients/task-client.js"; @@ -29,17 +26,18 @@ import { import { TransactionClient } from "../src/clients/transaction-client.js"; import { WorkflowClient } from "../src/clients/workflow-client.js"; import { - INTERNAL_EXECUTION_ID_PREFIX, formatChildExecutionName, formatExecutionId, + INTERNAL_EXECUTION_ID_PREFIX, } from "../src/execution.js"; import { BucketStore } from "../src/stores/bucket-store.js"; +import { EntityStore } from "../src/stores/entity-store.js"; import { WorkflowCallExecutor } from "../src/workflow-call-executor.js"; import { awaitTimerCall, childWorkflowCall, - entityRequestCall, emitEventCall, + entityRequestCall, sendSignalCall, taskCall, } from "./call-util.js"; @@ -61,17 +59,13 @@ const mockExecutionQueueClient = { jest.fn() as ExecutionQueueClient["submitExecutionEvents"], sendSignal: jest.fn() as ExecutionQueueClient["sendSignal"], } satisfies Partial as ExecutionQueueClient; -const mockEntity = { - get: jest.fn() as Entity["get"], - getWithMetadata: jest.fn() as Entity["getWithMetadata"], - set: jest.fn() as Entity["set"], - delete: jest.fn() as Entity["delete"], - list: jest.fn() as Entity["list"], - listKeys: jest.fn() as Entity["listKeys"], -} satisfies EntityMethods; -const mockEntityClient = { - getEntity: jest.fn() as EntityClient["getEntity"], -} satisfies Partial as EntityClient; +const mockEntityStore = { + get: jest.fn() as EntityStore["get"], + getWithMetadata: jest.fn() as EntityStore["getWithMetadata"], + set: jest.fn() as EntityStore["set"], + delete: jest.fn() as EntityStore["delete"], + query: jest.fn() as EntityStore["query"], +} satisfies Partial as EntityStore; const mockTransactionClient = { executeTransaction: jest.fn() as TransactionClient["executeTransaction"], } satisfies Partial as TransactionClient; @@ -79,7 +73,7 @@ const mockBucketStore = {} satisfies Partial as BucketStore; const testExecutor = new WorkflowCallExecutor({ bucketStore: mockBucketStore, - entityClient: mockEntityClient, + entityStore: mockEntityStore, eventClient: mockEventClient, executionQueueClient: mockExecutionQueueClient, taskClient: mockTaskClient, @@ -95,10 +89,6 @@ const executionId = "execId/123"; const baseTime = new Date(); -beforeEach(() => { - (mockEntityClient.getEntity as jest.Mock).mockResolvedValue(mockEntity); -}); - afterEach(() => { jest.resetAllMocks(); }); @@ -278,17 +268,19 @@ describe("entity request", () => { const event = await testExecutor.executeCall( workflow, executionId, - entityRequestCall({ name: "ent", operation: "get", key: "key" }, 0), + entityRequestCall( + { entityName: "ent", operation: "get", params: [["key"]] }, + 0 + ), baseTime ); - expect(mockEntityClient.getEntity).toHaveBeenCalledWith("ent"); - expect(mockEntity.get).toHaveBeenCalledWith("key"); + expect(mockEntityStore.get).toHaveBeenCalledWith("ent", ["key"]); expect(event).toMatchObject({ seq: 0, type: WorkflowEventType.EntityRequest, - operation: { name: "ent", operation: "get", key: "key" }, + operation: { entityName: "ent", operation: "get", params: [["key"]] }, timestamp: expect.stringContaining("Z"), }); }); @@ -298,23 +290,28 @@ describe("entity request", () => { workflow, executionId, entityRequestCall( - { name: "ent", operation: "set", key: "key", value: "some value" }, + { + entityName: "ent", + operation: "set", + params: [{ key: "key", value: "some value" }], + }, 0 ), baseTime ); - expect(mockEntityClient.getEntity).toHaveBeenCalledWith("ent"); - expect(mockEntity.set).toHaveBeenCalledWith("key", "some value", undefined); + expect(mockEntityStore.set).toHaveBeenCalledWith("ent", { + key: "key", + value: "some value", + }); expect(event).toMatchObject({ seq: 0, type: WorkflowEventType.EntityRequest, operation: { - name: "ent", + entityName: "ent", operation: "set", - key: "key", - value: "some value", + params: [{ key: "key", value: "some value" }], }, timestamp: expect.stringContaining("Z"), }); @@ -324,55 +321,48 @@ describe("entity request", () => { const event = await testExecutor.executeCall( workflow, executionId, - entityRequestCall({ name: "ent", operation: "delete", key: "key" }, 0), - baseTime - ); - - expect(mockEntityClient.getEntity).toHaveBeenCalledWith("ent"); - expect(mockEntity.delete).toHaveBeenCalledWith("key", undefined); - - expect(event).toMatchObject({ - seq: 0, - type: WorkflowEventType.EntityRequest, - operation: { name: "ent", operation: "delete", key: "key" }, - timestamp: expect.stringContaining("Z"), - }); - }); - - test("list", async () => { - const event = await testExecutor.executeCall( - workflow, - executionId, - entityRequestCall({ name: "ent", operation: "list", request: {} }, 0), + entityRequestCall( + { entityName: "ent", operation: "delete", params: [["key"]] }, + 0 + ), baseTime ); - expect(mockEntityClient.getEntity).toHaveBeenCalledWith("ent"); - expect(mockEntity.list).toHaveBeenCalledWith({}); + expect(mockEntityStore.delete).toHaveBeenCalledWith("ent", ["key"]); expect(event).toMatchObject({ seq: 0, type: WorkflowEventType.EntityRequest, - operation: { name: "ent", operation: "list", request: {} }, + operation: { entityName: "ent", operation: "delete", params: [["key"]] }, timestamp: expect.stringContaining("Z"), }); }); - test("listKeys", async () => { + test("query", async () => { const event = await testExecutor.executeCall( workflow, executionId, - entityRequestCall({ name: "ent", operation: "listKeys", request: {} }, 0), + entityRequestCall( + { + entityName: "ent", + operation: "query", + params: [{ id: "part" }], + }, + 0 + ), baseTime ); - expect(mockEntityClient.getEntity).toHaveBeenCalledWith("ent"); - expect(mockEntity.listKeys).toHaveBeenCalledWith({}); + expect(mockEntityStore.query).toHaveBeenCalledWith("ent", { id: "part" }); expect(event).toMatchObject({ seq: 0, type: WorkflowEventType.EntityRequest, - operation: { name: "ent", operation: "listKeys", request: {} }, + operation: { + entityName: "ent", + operation: "query", + params: [{ id: "part" }], + }, timestamp: expect.stringContaining("Z"), }); }); diff --git a/packages/@eventual/core-runtime/test/transaction-executor.test.ts b/packages/@eventual/core-runtime/test/transaction-executor.test.ts index 16c764810..c0e5b5646 100644 --- a/packages/@eventual/core-runtime/test/transaction-executor.test.ts +++ b/packages/@eventual/core-runtime/test/transaction-executor.test.ts @@ -1,24 +1,38 @@ -import { entity as _entity, event, TransactionContext } from "@eventual/core"; -import { entities, registerEntityHook, Result } from "@eventual/core/internal"; +import { + Attributes, + CompositeKeyPart, + EntityOptions, + TransactionContext, + entity as _entity, + event, +} from "@eventual/core"; +import { Result, entities, registerEntityHook } from "@eventual/core/internal"; import { jest } from "@jest/globals"; -import { EntityClient } from "../src/clients/entity-client.js"; +import { z } from "zod"; import { EventClient } from "../src/clients/event-client.js"; import { ExecutionQueueClient } from "../src/clients/execution-queue-client.js"; import { NoOpLocalEnvConnector } from "../src/local/local-container.js"; import { LocalEntityStore } from "../src/local/stores/entity-store.js"; +import { GlobalEntityProvider } from "../src/providers/entity-provider.js"; import { EntityStore } from "../src/stores/entity-store.js"; import { - createTransactionExecutor, TransactionExecutor, TransactionResult, + createTransactionExecutor, } from "../src/transaction-executor.js"; const entity = (() => { let n = 0; - return () => { + return < + Attr extends Attributes, + const Partition extends CompositeKeyPart = CompositeKeyPart, + const Sort extends CompositeKeyPart | undefined = undefined + >( + options: EntityOptions + ) => { // eslint-disable-next-line no-empty while (entities().has(`ent${++n}`)) {} - return _entity(`ent${n}`); + return _entity(`ent${n}`, options); }; })(); @@ -32,6 +46,7 @@ const mockEventClient = { let store: EntityStore; let executor: TransactionExecutor; +const entityProvider = new GlobalEntityProvider(); const event1 = event("event1"); beforeEach(() => { @@ -39,12 +54,14 @@ beforeEach(() => { store = new LocalEntityStore({ localConnector: NoOpLocalEnvConnector, + entityProvider, }); - registerEntityHook(new EntityClient(store)); + registerEntityHook(store); executor = createTransactionExecutor( store, + entityProvider, mockExecutionQueueClient, mockEventClient ); @@ -56,26 +73,31 @@ const context: TransactionContext = { }, }; +const simpleSchema = { key: z.string(), value: z.number() }; + test("just get", async () => { - const d1 = entity(); + const d1 = entity({ + attributes: { key: z.string(), value: z.number() }, + partition: ["key"], + }); const result = await executor( () => { - return d1.get("1"); + return d1.get({ key: "1" }); }, undefined, context ); - expect(result).toMatchObject>({ + expect(result).toEqual>({ result: Result.resolved(undefined), }); }); test("just set", async () => { - const d1 = entity(); + const d1 = entity({ partition: ["key"], attributes: simpleSchema }); const result = await executor( () => { - return d1.set("1", 1); + return d1.set({ key: "1", value: 1 }); }, undefined, context @@ -85,20 +107,20 @@ test("just set", async () => { result: Result.resolved({ version: 1 }), }); - await expect(store.getEntityValue(d1.name, "1")).resolves.toEqual({ - entity: 1, + await expect(store.getWithMetadata(d1.name, { key: "1" })).resolves.toEqual({ + value: { key: "1", value: 1 }, version: 1, }); }); test("just delete", async () => { - const d1 = entity(); + const d1 = entity({ attributes: simpleSchema, partition: ["key"] }); - await store.setEntityValue(d1.name, "1", 0); + await store.set(d1.name, { key: "1", value: 0 }); const result = await executor( () => { - return d1.delete("1"); + return d1.delete(["1"]); }, undefined, context @@ -108,17 +130,20 @@ test("just delete", async () => { result: Result.resolved(undefined), }); - await expect(store.getEntityValue(d1.name, "1")).resolves.toBeUndefined(); + await expect(store.get(d1.name, { key: "1" })).resolves.toBeUndefined(); }); test("multiple operations", async () => { - const d1 = entity(); - const d2 = entity(); + const d1 = entity({ attributes: simpleSchema, partition: ["key"] }); + const d2 = entity({ + attributes: simpleSchema, + partition: ["value"], + }); const result = await executor( async () => { - await d1.set("1", 1); - await d2.set("1", "a"); + await d1.set({ key: "1", value: 1 }); + await d2.set({ key: "1", value: 1 }); }, undefined, context @@ -128,27 +153,30 @@ test("multiple operations", async () => { result: Result.resolved(undefined), }); - await expect(store.getEntityValue(d1.name, "1")).resolves.toEqual({ - entity: 1, + await expect(store.getWithMetadata(d1.name, { key: "1" })).resolves.toEqual({ + value: { key: "1", value: 1 }, version: 1, }); - await expect(store.getEntityValue(d2.name, "1")).resolves.toEqual({ - entity: "a", + await expect(store.getWithMetadata(d2.name, [1])).resolves.toEqual({ + value: { key: "1", value: 1 }, version: 1, }); }); test("multiple operations fail", async () => { - const d1 = entity(); - const d2 = entity(); + const d1 = entity({ attributes: simpleSchema, partition: ["key"] }); + const d2 = entity({ + attributes: simpleSchema, + partition: ["value"], + }); - await store.setEntityValue(d1.name, "1", 0); + await store.set(d1.name, { key: "1", value: 0 }); const result = await executor( async () => { - await d1.set("1", 1, { expectedVersion: 3 }); - await d2.set("1", "a"); + await d1.set({ key: "1", value: 1 }, { expectedVersion: 3 }); + await d2.set({ key: "1", value: 1 }); }, undefined, context @@ -158,27 +186,27 @@ test("multiple operations fail", async () => { result: Result.failed(Error("Failed after an explicit conflict.")), }); - await expect(store.getEntityValue(d1.name, "1")).resolves.toEqual({ - entity: 0, + await expect(store.getWithMetadata(d1.name, ["1"])).resolves.toEqual({ + value: { key: "1", value: 0 }, version: 1, }); - await expect(store.getEntityValue(d2.name, "1")).resolves.toBeUndefined(); + await expect(store.get(d2.name, [1])).resolves.toBeUndefined(); }); test("retry when retrieved data changes version", async () => { - const d1 = entity(); + const d1 = entity({ attributes: simpleSchema, partition: ["key"] }); - await store.setEntityValue(d1.name, "1", 0); + await store.set(d1.name, { key: "1", value: 0 }); const result = await executor( async () => { - const v = await d1.get("1"); + const v = await d1.get(["1"]); // this isn't kosher... normally - if (v === 0) { - await store.setEntityValue(d1.name, "1", v! + 1); + if (v?.value === 0) { + await store.set(d1.name, { key: "1", value: v.value + 1 }); } - await d1.set("1", v! + 1); + await d1.set({ key: "1", value: v!.value + 1 }); }, undefined, context @@ -188,25 +216,25 @@ test("retry when retrieved data changes version", async () => { result: Result.resolved(undefined), }); - await expect(store.getEntityValue(d1.name, "1")).resolves.toEqual({ - entity: 2, + await expect(store.getWithMetadata(d1.name, ["1"])).resolves.toEqual({ + value: { key: "1", value: 2 }, version: 3, }); }); test("retry when retrieved data changes version multiple times", async () => { - const d1 = entity(); + const d1 = entity({ attributes: simpleSchema, partition: ["key"] }); - await store.setEntityValue(d1.name, "1", 0); + await store.set(d1.name, { key: "1", value: 0 }); const result = await executor( async () => { - const v = (await d1.get("1")) ?? 0; + const { value } = (await d1.get(["1"])) ?? { value: 0 }; // this isn't kosher... normally - if (v < 2) { - await store.setEntityValue(d1.name, "1", v + 1); + if (value < 2) { + await store.set(d1.name, { key: "1", value: value + 1 }); } - await d1.set("1", v + 1); + await d1.set({ key: "1", value: value + 1 }); }, undefined, context @@ -216,19 +244,19 @@ test("retry when retrieved data changes version multiple times", async () => { result: Result.resolved(undefined), }); - await expect(store.getEntityValue(d1.name, "1")).resolves.toEqual({ - entity: 3, + await expect(store.getWithMetadata(d1.name, ["1"])).resolves.toEqual({ + value: { key: "1", value: 3 }, version: 4, }); }); test("emit events on success", async () => { - const d1 = entity(); + const d1 = entity({ attributes: simpleSchema, partition: ["key"] }); const result = await executor( async () => { event1.emit({ n: 1 }); - await d1.set("1", 1); + await d1.set({ key: "1", value: 1 }); event1.emit({ n: 1 }); }, undefined, @@ -243,27 +271,30 @@ test("emit events on success", async () => { }); test("emit events after retry", async () => { - const d1 = entity(); + const d1 = entity({ + attributes: simpleSchema, + partition: ["key"], + }); - await store.setEntityValue(d1.name, "1", 0); + await store.set(d1.name, { key: "1", value: 0 }); const result = await executor( async () => { event1.emit({ n: 1 }); - const v = await d1.get("1"); + const v = await d1.get(["1"]); event1.emit({ n: v }); // this isn't kosher... normally - if (v === 0) { - await store.setEntityValue(d1.name, "1", v! + 1); + if (v?.value === 0) { + await store.set(d1.name, { key: "1", value: v!.value + 1 }); } - await d1.set("1", v! + 1); + await d1.set({ key: "1", value: v!.value + 1 }); event1.emit({ n: 1 }); }, undefined, context ); - expect(result).toMatchObject>({ + expect(result).toEqual>({ result: Result.resolved(undefined), }); @@ -271,14 +302,14 @@ test("emit events after retry", async () => { }); test("events not emitted on failure", async () => { - const d1 = entity(); + const d1 = entity({ attributes: simpleSchema, partition: ["key"] }); - await store.setEntityValue(d1.name, "1", 0); + await store.set(d1.name, { key: "1", value: 0 }); const result = await executor( async () => { event1.emit({ n: 1 }); - await d1.set("1", 1, { expectedVersion: 1000 }); + await d1.set({ key: "1", value: 1 }, { expectedVersion: 1000 }); event1.emit({ n: 1 }); }, undefined, diff --git a/packages/@eventual/core/src/entity.ts b/packages/@eventual/core/src/entity.ts deleted file mode 100644 index b865d0080..000000000 --- a/packages/@eventual/core/src/entity.ts +++ /dev/null @@ -1,443 +0,0 @@ -import type { z } from "zod"; -import { - createEventualCall, - EntityCall, - EntityDeleteOperation, - EntitySetOperation, - EventualCallKind, -} from "./internal/calls.js"; -import { getEntityHook } from "./internal/entity-hook.js"; -import { entities } from "./internal/global.js"; -import { - EntitySpec, - EntityStreamOptions, - EntityStreamSpec, - isSourceLocation, - SourceLocation, -} from "./internal/service-spec.js"; -import type { ServiceContext } from "./service.js"; - -export interface CompositeKey { - namespace: string; - key: string; -} - -export interface EntityListResult { - entries?: { key: string; entity: Entity; version: number }[]; - /** - * Returned when there are more values than the limit allowed to return. - */ - nextToken?: string; -} - -export interface EntityListKeysResult { - /** - * Keys that match the provided prefix. If using composite keys, this will only be the key part (not the namespace). - */ - keys?: string[]; - /** - * Returned when there are more values than the limit allowed to return. - */ - nextToken?: string; -} - -export interface EntityListRequest { - /** - * Namespace to retrieve values for. - * - * @default - retrieve values with no namespace. - */ - namespace?: string; - /** - * Key prefix to retrieve values or keys for. - * Values are only retrieved for a single name + namespace pair (including no namespace). - */ - prefix?: string; - /** - * Number of items to retrieve - * @default 100 - */ - limit?: number; - nextToken?: string; -} - -export interface EntityConsistencyOptions { - /** - * The expected version of the entity in the entity. - * - * Used to support consistent writes and deletes. - * A value of 0 will only pass if the item is new. - */ - expectedVersion?: number; -} - -export interface EntitySetOptions extends EntityConsistencyOptions { - /** - * Whether or not to update the version on change. - * If this is the first time the value has been set, it will be set to 1. - * - * @default true - version will be incremented. - */ - incrementVersion?: boolean; -} - -export interface EntityStreamContext { - /** - * Information about the containing service. - */ - service: ServiceContext; -} - -export interface EntityStreamHandler { - /** - * Provides the keys, new value - */ - (item: EntityStreamItem, context: EntityStreamContext): - | Promise - | void - | false; -} - -export interface EntityStreamItemBase { - streamName: string; - entityName: string; - namespace?: string; - key: string; -} - -export type EntityStreamItem = - | EntityStreamInsertItem - | EntityStreamModifyItem - | EntityStreamRemoveItem; - -export interface EntityStreamInsertItem extends EntityStreamItemBase { - newValue: Entity; - newVersion: number; - operation: "insert"; -} - -export interface EntityStreamModifyItem extends EntityStreamItemBase { - operation: "modify"; - newValue: Entity; - newVersion: number; - oldValue?: Entity; - oldVersion?: number; -} - -export interface EntityStreamRemoveItem extends EntityStreamItemBase { - operation: "remove"; - oldValue?: Entity; - oldVersion?: number; -} - -export interface EntityStream extends EntityStreamSpec { - kind: "EntityStream"; - handler: EntityStreamHandler; - sourceLocation?: SourceLocation; -} - -export interface Entity extends Omit { - kind: "Entity"; - schema?: z.Schema; - streams: EntityStream[]; - /** - * Get a value. - * If your values use composite keys, the namespace must be provided. - * - * @param key - key or {@link CompositeKey} of the value to retrieve. - */ - get(key: string | CompositeKey): Promise; - /** - * Get a value and metadata like version. - * If your values use composite keys, the namespace must be provided. - * - * @param key - key or {@link CompositeKey} of the value to retrieve. - */ - getWithMetadata( - key: string | CompositeKey - ): Promise<{ entity: E; version: number } | undefined>; - /** - * Sets or updates a value within an entity and optionally a namespace. - * - * Values with namespaces are considered distinct from value without a namespace or within different namespaces. - * Values and keys can only be listed within a single namespace. - */ - set( - key: string | CompositeKey, - entity: E, - options?: EntitySetOptions - ): Promise<{ version: number }>; - /** - * Deletes a single entry within an entity and namespace. - */ - delete( - key: string | CompositeKey, - options?: EntityConsistencyOptions - ): Promise; - /** - * List entries that match a prefix within an entity and namespace. - * - * If namespace is not provided, only values which do not use composite keys will be returned. - */ - list(request: EntityListRequest): Promise>; - /** - * List keys that match a prefix within an entity and namespace. - * - * If namespace is not provided, only values which do not use composite keys will be returned. - */ - listKeys(request: EntityListRequest): Promise; - stream( - name: string, - options: EntityStreamOptions, - handler: EntityStreamHandler - ): EntityStream; - stream(name: string, handler: EntityStreamHandler): EntityStream; -} - -export interface EntityTransactItem< - E = any, - D extends string | Entity = string | Entity -> { - entity: D; - operation: - | Omit, "name"> - | Omit - | Omit; -} - -/** - * Used in transactions, cancels the transaction if the key's version does not match. - */ -export interface EntityConditionalOperation { - operation: "condition"; - name: string; - key: string | CompositeKey; - version?: number; -} - -export const Entity = { - transactWrite: []>( - items: Items - ): Promise => { - return getEventualCallHook().registerEventualCall( - createEventualCall>(EventualCallKind.EntityCall, { - operation: "transact", - items, - }), - async () => { - return getEntityHook().transactWrite(items); - } - ); - }, -}; - -export function entity(name: string, schema?: z.Schema): Entity { - if (entities().has(name)) { - throw new Error(`entity with name '${name}' already exists`); - } - - /** - * Used to maintain a limited number of streams on the entity. - */ - const streams: EntityStream[] = []; - - const entity: Entity = { - kind: "Entity", - name, - schema, - streams, - get: (key: string | CompositeKey) => { - return getEventualCallHook().registerEventualCall( - createEventualCall>(EventualCallKind.EntityCall, { - name, - operation: "get", - key, - }), - async () => { - return (await getEntity()).get(key); - } - ); - }, - getWithMetadata: (key: string | CompositeKey) => { - return getEventualCallHook().registerEventualCall( - createEventualCall>( - EventualCallKind.EntityCall, - { - name, - operation: "getWithMetadata", - key, - } - ), - async () => { - return (await getEntity()).getWithMetadata(key); - } - ); - }, - set: ( - key: string | CompositeKey, - entity: E, - options?: EntitySetOptions - ) => { - return getEventualCallHook().registerEventualCall( - createEventualCall>(EventualCallKind.EntityCall, { - name, - operation: "set", - key, - options, - value: entity, - }), - async () => { - return (await getEntity()).set(key, entity, options); - } - ); - }, - delete: (key, options) => { - return getEventualCallHook().registerEventualCall( - createEventualCall>(EventualCallKind.EntityCall, { - name, - operation: "delete", - key, - options, - }), - async () => { - return (await getEntity()).delete(key, options); - } - ); - }, - list: (request) => { - return getEventualCallHook().registerEventualCall( - createEventualCall>(EventualCallKind.EntityCall, { - name, - operation: "list", - request, - }), - async () => { - return (await getEntity()).list(request); - } - ); - }, - listKeys: (request) => { - return getEventualCallHook().registerEventualCall( - createEventualCall>( - EventualCallKind.EntityCall, - { - name, - operation: "listKeys", - request, - } - ), - async () => { - return (await getEntity()).listKeys(request); - } - ); - }, - stream: ( - ...args: - | [name: string, handler: EntityStreamHandler] - | [ - name: string, - options: EntityStreamOptions, - handler: EntityStreamHandler - ] - | [ - sourceLocation: SourceLocation, - name: string, - handler: EntityStreamHandler - ] - | [ - sourceLocation: SourceLocation, - name: string, - options: EntityStreamOptions, - handler: EntityStreamHandler - ] - ) => { - const [sourceLocation, streamName, options, handler] = - args.length === 2 - ? [, args[0], , args[1]] - : args.length === 4 - ? args - : isSourceLocation(args[0]) && typeof args[1] === "string" - ? [args[0], args[1] as string, , args[2]] - : [, args[0] as string, args[1] as EntityStreamOptions, args[2]]; - - if (streams.length > 1) { - throw new Error("Only two streams are allowed per entity."); - } - - const entityStream: EntityStream = { - kind: "EntityStream", - handler, - name: streamName, - entityName: name, - options, - sourceLocation, - }; - - streams.push(entityStream); - - return entityStream; - }, - }; - - entities().set(name, entity); - - return entity; - - async function getEntity() { - const entityHook = getEntityHook(); - const entity = await entityHook.getEntity(name); - if (!entity) { - throw new Error(`Entity ${name} does not exist.`); - } - return entity; - } -} - -export function entityStream( - ...args: - | [name: string, entity: Entity, handler: EntityStreamHandler] - | [ - name: string, - entity: Entity, - options: EntityStreamOptions, - handler: EntityStreamHandler - ] - | [ - sourceLocation: SourceLocation, - name: string, - entity: Entity, - handler: EntityStreamHandler - ] - | [ - sourceLocation: SourceLocation, - name: string, - entity: Entity, - options: EntityStreamOptions, - handler: EntityStreamHandler - ] -) { - const [sourceLocation, name, entity, options, handler] = - args.length === 3 - ? [, args[0], args[1], , args[2]] - : args.length === 5 - ? args - : isSourceLocation(args[0]) - ? [args[0], args[1] as string, args[2] as Entity, , args[3]] - : [ - , - args[0] as string, - args[1] as Entity, - args[2] as EntityStreamOptions, - args[3], - ]; - - return sourceLocation - ? options - ? // @ts-ignore - entity.stream(sourceLocation, name, options, handler) - : // @ts-ignore - entity.stream(sourceLocation, name, handler) - : options - ? entity.stream(name, options, handler) - : entity.stream(name, handler); -} diff --git a/packages/@eventual/core/src/entity/entity.ts b/packages/@eventual/core/src/entity/entity.ts new file mode 100644 index 000000000..735195b25 --- /dev/null +++ b/packages/@eventual/core/src/entity/entity.ts @@ -0,0 +1,457 @@ +import { z } from "zod"; +import { + createEventualCall, + EntityCall, + EventualCallKind, +} from "../internal/calls.js"; +import { getEntityHook } from "../internal/entity-hook.js"; +import { computeKeyDefinition, KeyDefinition } from "../internal/entity.js"; +import { entities } from "../internal/global.js"; +import { + EntitySpec, + EntityStreamOptions, + isSourceLocation, + SourceLocation, +} from "../internal/service-spec.js"; +import type { CompositeKey, CompositeKeyPart, QueryKey } from "./key.js"; +import type { EntityStream, EntityStreamHandler } from "./stream.js"; + +export type AttributeBinaryValue = + | ArrayBuffer + | Blob + | Buffer + | DataView + | File + | Int8Array + | Uint8Array + | Uint8ClampedArray + | Int16Array + | Uint16Array + | Int32Array + | Uint32Array + | Float32Array + | Float64Array + | BigInt64Array + | BigUint64Array; + +export type AttributeValue = + | Attributes + | string + | number + | boolean + | AttributeBinaryValue + | Set + | AttributeValue[]; + +export interface Attributes { + [key: string]: AttributeValue; +} + +/** + * Turns a {@link Attributes} type into a Zod {@link z.ZodRawShape}. + */ +export type EntityZodShape = { + [key in keyof Attr]: z.ZodType; +}; + +/** + * An eventual entity. + * + * @see entity + */ +export interface Entity< + Attr extends Attributes = any, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> extends Omit { + kind: "Entity"; + key: KeyDefinition; + attributes: z.ZodObject>; + streams: EntityStream[]; + /** + * Get a value. + * If your values use composite keys, the namespace must be provided. + * + * @param key - key or {@link CompositeKey} of the value to retrieve. + */ + get(key: CompositeKey): Promise; + /** + * Get a value and metadata like version. + * If your values use composite keys, the namespace must be provided. + * + * @param key - key or {@link CompositeKey} of the value to retrieve. + */ + getWithMetadata( + key: CompositeKey + ): Promise | undefined>; + /** + * Sets or updates a value within an entity and optionally a namespace. + * + * Values with namespaces are considered distinct from value without a namespace or within different namespaces. + * Values and keys can only be listed within a single namespace. + */ + set(entity: Attr, options?: EntitySetOptions): Promise<{ version: number }>; + /** + * Deletes a single entry within an entity and namespace. + */ + delete( + key: CompositeKey, + options?: EntityConsistencyOptions + ): Promise; + /** + * List entries that match a prefix within an entity and namespace. + * + * If namespace is not provided, only values which do not use composite keys will be returned. + */ + query( + key: QueryKey, + request?: EntityQueryOptions + ): Promise>; + stream( + name: string, + options: EntityStreamOptions, + handler: EntityStreamHandler + ): EntityStream; + stream( + name: string, + handler: EntityStreamHandler + ): EntityStream; +} + +export const Entity = { + transactWrite: (items: EntityTransactItem[]): Promise => { + return getEventualCallHook().registerEventualCall( + createEventualCall>(EventualCallKind.EntityCall, { + operation: "transact", + items, + }), + async () => { + return getEntityHook().transactWrite(items); + } + ); + }, +}; + +export interface EntityOptions< + Attr extends Attributes, + Partition extends CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = undefined +> { + attributes: z.ZodObject> | EntityZodShape; + partition: Partition; + sort?: Sort; +} + +/** + * Creates an entity which holds data. + * + * An entity's keys are made up of one or more attributes in the entity. + * When an entity's key is made up of more than one attribute, it is considered to be a composite key. + * + * Each attribute of the composite key is considered to be either a partition key or a sort key, which we consider a composite key part. + * Each entity is required to at least have one partition key attribute, but may have may partition and or sort key attributes. + * To retrieve a single value with an entity, the entire composite key must be used, until the query operation is used to return multiple entities (within a partition). + * + * A partition key separates data within an entity. When using the Query operation, data can only be queried within + * a single partition. + * + * A sort key determines the order of the values when running a query. It also allows for ranges of values to be queried + * using only some of the sort key attributes (in order). + * + * ```ts + * // lets take an example where we have posts for a user, separated by forum. + * const userComments = entity("userComments", { + * attributes: { + * forum: z.string(), + * userId: z.string(), + * postId: z.string(), + * commentId: z.string(), + * message: z.string() + * }, + * partition: ["forum", "userId"], + * sort: ["postId", "id"], + * }); + * + * // add a new post comment + * await userComments.set({ + * forum: "games", + * userId: "1", + * postId: "100", + * commentId: "abc", + * message: "I love games" + * }); + * + * // get all comments for a user in a forum + * await userComments.query({ + * forum: "games", // required in the query + * userId: "1", // required in the query + * }); + * + * // get all comments for a user in a forum and a post + * await userComments.query({ + * forum: "games", // required in the query + * userId: "1", // required in the query + * post: "100", // optional in the query + * }); + * + * // get a single post + * await userComments.get({ + * forum: "games", + * userId: "1", + * postId: "100", + * commentId: "abc" + * }); + * ``` + */ +export function entity< + Attr extends Attributes, + const Partition extends CompositeKeyPart, + const Sort extends CompositeKeyPart | undefined = undefined +>( + name: string, + options: EntityOptions +): Entity { + if (entities().has(name)) { + throw new Error(`entity with name '${name}' already exists`); + } + + /** + * Used to maintain a limited number of streams on the entity. + */ + const streams: EntityStream[] = []; + + const attributes = + options.attributes instanceof z.ZodObject + ? options.attributes + : z.object(options.attributes); + + const entity: Entity = { + // @ts-ignore + __entityBrand: undefined, + kind: "Entity", + name, + key: computeKeyDefinition(attributes, options.partition, options.sort), + attributes, + streams, + get: (...args) => { + return getEventualCallHook().registerEventualCall( + createEventualCall>(EventualCallKind.EntityCall, { + operation: "get", + entityName: name, + params: args, + }), + async () => { + return getEntityHook().get(name, ...args) as Promise; + } + ); + }, + getWithMetadata: (...args) => { + return getEventualCallHook().registerEventualCall( + createEventualCall>( + EventualCallKind.EntityCall, + { + operation: "getWithMetadata", + entityName: name, + params: args, + } + ), + async () => { + return getEntityHook().getWithMetadata(name, ...args); + } + ); + }, + set: (...args) => { + return getEventualCallHook().registerEventualCall( + createEventualCall>(EventualCallKind.EntityCall, { + entityName: name, + operation: "set", + params: args, + }), + async () => { + return getEntityHook().set(name, ...args); + } + ); + }, + delete: (...args) => { + return getEventualCallHook().registerEventualCall( + createEventualCall>(EventualCallKind.EntityCall, { + entityName: name, + operation: "delete", + params: args, + }), + async () => { + return getEntityHook().delete(name, ...args); + } + ); + }, + query: (...args) => { + return getEventualCallHook().registerEventualCall( + createEventualCall>(EventualCallKind.EntityCall, { + entityName: name, + operation: "query", + params: args, + }), + async () => { + return getEntityHook().query(name, ...args); + } + ); + }, + stream: ( + ...args: + | [name: string, handler: EntityStreamHandler] + | [ + name: string, + options: EntityStreamOptions, + handler: EntityStreamHandler + ] + | [ + sourceLocation: SourceLocation, + name: string, + handler: EntityStreamHandler + ] + | [ + sourceLocation: SourceLocation, + name: string, + options: EntityStreamOptions, + handler: EntityStreamHandler + ] + ) => { + const [sourceLocation, streamName, options, handler] = + args.length === 2 + ? [, args[0], , args[1]] + : args.length === 4 + ? args + : isSourceLocation(args[0]) && typeof args[1] === "string" + ? [args[0], args[1] as string, , args[2]] + : [ + , + args[0] as string, + args[1] as EntityStreamOptions, + args[2], + ]; + + if (streams.length > 1) { + throw new Error("Only two streams are allowed per entity."); + } + + const entityStream: EntityStream = { + kind: "EntityStream", + handler, + name: streamName, + entityName: name, + options, + sourceLocation, + }; + + streams.push(entityStream); + + return entityStream; + }, + }; + + entities().set(name, entity as any); + + return entity; +} + +export interface EntityQueryResult { + entries?: EntityWithMetadata[]; + /** + * Returned when there are more values than the limit allowed to return. + */ + nextToken?: string; +} + +export interface EntityQueryOptions { + /** + * Number of items to retrieve + * @default 100 + */ + limit?: number; + nextToken?: string; +} + +export interface EntityConsistencyOptions { + /** + * The expected version of the entity in the entity. + * + * Used to support consistent writes and deletes. + * A value of 0 will only pass if the item is new. + */ + expectedVersion?: number; +} + +export interface EntitySetOptions extends EntityConsistencyOptions { + /** + * Whether or not to update the version on change. + * If this is the first time the value has been set, it will be set to 1. + * + * @default true - version will be incremented. + */ + incrementVersion?: boolean; +} + +export interface EntityWithMetadata { + value: Attr; + version: number; +} + +interface EntityTransactItemBase< + Attr extends Attributes, + Partition extends CompositeKeyPart, + Sort extends CompositeKeyPart | undefined +> { + entity: Entity | string; +} + +export type EntityTransactItem< + Attr extends Attributes = any, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> = + | EntityTransactSetOperation + | EntityTransactDeleteOperation + | EntityTransactConditionalOperation; + +export interface EntityTransactSetOperation< + Attr extends Attributes = any, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> extends EntityTransactItemBase { + operation: "set"; + value: Attr; + options?: EntitySetOptions; +} + +export interface EntityTransactDeleteOperation< + Attr extends Attributes = any, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> extends EntityTransactItemBase { + operation: "delete"; + key: CompositeKey; + options?: EntitySetOptions; +} + +/** + * Used in transactions, cancels the transaction if the key's version does not match. + */ +export interface EntityTransactConditionalOperation< + Attr extends Attributes = any, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> extends EntityTransactItemBase { + operation: "condition"; + key: CompositeKey; + version?: number; +} diff --git a/packages/@eventual/core/src/entity/index.ts b/packages/@eventual/core/src/entity/index.ts new file mode 100644 index 000000000..03f448212 --- /dev/null +++ b/packages/@eventual/core/src/entity/index.ts @@ -0,0 +1,3 @@ +export * from "./entity.js"; +export * from "./key.js"; +export * from "./stream.js"; diff --git a/packages/@eventual/core/src/entity/key.ts b/packages/@eventual/core/src/entity/key.ts new file mode 100644 index 000000000..fe07abad6 --- /dev/null +++ b/packages/@eventual/core/src/entity/key.ts @@ -0,0 +1,157 @@ +import type { Attributes } from "./entity.js"; + +export type KeyValue = string | number; + +/** + * Composite Key - Whole key used to get and set an entity, made up of partition and sort key parts containing one or more attribute. + * Key Part - partition or sort key of the composite key, each made up of one or more key attribute. + * Key Attribute - A single attribute used as a segment of a key part. + */ + +/** + * Any attribute name considered to be a valid key attribute. + */ +export type KeyAttribute = { + [K in keyof Attr]: K extends string + ? // only include attributes that extend string or number + Attr[K] extends KeyValue + ? K + : never + : never; +}[keyof Attr]; + +/** + * A part of the composite key, either the partition or sort key. + */ +export type CompositeKeyPart = readonly [ + KeyAttribute, + ...KeyAttribute[] +]; + +/** + * All attributes of the composite key as an object. + * + * ```ts + * { + * partitionAttribute1: "", + * partitionAttribute2: "", + * sortAttribute1: "", + * sortAttribute2: "" + * } + * ``` + */ +export type KeyMap< + Attr extends Attributes = any, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> = { + [k in Partition[number]]: Attr[k]; +} & (Sort extends CompositeKeyPart + ? { + [k in Sort[number]]: Attr[k]; + } + : // eslint-disable-next-line + {}); + +export type KeyPartialTuple< + Attr extends Attributes, + Attrs extends readonly (keyof Attr)[] +> = Attrs extends [] + ? readonly [] + : Attrs extends readonly [ + infer Head extends keyof Attr, + ...infer Rest extends readonly (keyof Attr)[] + ] + ? readonly [Attr[Head], ...KeyPartialTuple] + : readonly []; + +/** + * All attributes of the composite key as a in order tuple. + * + * ```ts + * [partitionAttribute1, partitionAttribute2, sortAttribute1, sortAttribute2] + * ``` + */ +export type KeyTuple< + Attr extends Attributes, + Partition extends CompositeKeyPart, + Sort extends CompositeKeyPart | undefined +> = Sort extends undefined + ? KeyPartialTuple + : readonly [ + ...KeyPartialTuple, + ...KeyPartialTuple> + ]; + +/** + * All attributes in either the partition key and the sort key (when present). + */ +export type CompositeKey< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> = KeyMap | KeyTuple; + +export type ProgressiveQueryKey< + Attr extends Attributes, + Sort extends readonly (keyof Attr)[], + Accum extends object = object +> = Sort extends readonly [] + ? Accum + : Sort extends readonly [ + infer k extends keyof Attr, + ...infer ks extends readonly (keyof Attr)[] + ] + ? + | Accum + | ProgressiveQueryKey< + Attr, + ks, + Accum & { + [sk in k]: Attr[sk]; + } + > + : never; + +/** + * A partial key that can be used to query an entity. + * + * ```ts + * entity.query({ part1: "val", part2: "val2", sort1: "val" }); + * ``` + * + * TODO: support expressions like between and starts with on sort properties + */ +export type QueryKey< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> = + | ({ + [pk in Partition[number]]: Attr[pk]; + } & (Sort extends undefined + ? // eslint-disable-next-line + {} + : ProgressiveQueryKey>)) + | Partial>; + +/** + * A stream query can contain partial sort keys and partial partition keys. + */ +export type StreamQueryKey< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> = ProgressiveQueryKey & + (Sort extends undefined + ? // eslint-disable-next-line + {} + : ProgressiveQueryKey>); diff --git a/packages/@eventual/core/src/entity/stream.ts b/packages/@eventual/core/src/entity/stream.ts new file mode 100644 index 000000000..3c4acad08 --- /dev/null +++ b/packages/@eventual/core/src/entity/stream.ts @@ -0,0 +1,166 @@ +import { + EntityStreamOptions, + EntityStreamSpec, + SourceLocation, + isSourceLocation, +} from "../internal/service-spec.js"; +import type { ServiceContext } from "../service.js"; +import type { Entity, Attributes } from "./entity.js"; +import type { CompositeKeyPart, KeyMap } from "./key.js"; + +export interface EntityStreamContext { + /** + * Information about the containing service. + */ + service: ServiceContext; +} + +export interface EntityStreamHandler< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> { + /** + * Provides the keys, new value + */ + ( + item: EntityStreamItem, + context: EntityStreamContext + ): Promise | void | false; +} + +export interface EntityStreamItemBase< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> { + streamName: string; + entityName: string; + key: KeyMap; +} + +export type EntityStreamItem< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> = + | EntityStreamInsertItem + | EntityStreamModifyItem + | EntityStreamRemoveItem; + +export interface EntityStreamInsertItem< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> extends EntityStreamItemBase { + newValue: Attr; + newVersion: number; + operation: "insert"; +} + +export interface EntityStreamModifyItem< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> extends EntityStreamItemBase { + operation: "modify"; + newValue: Attr; + newVersion: number; + oldValue?: Attr; + oldVersion?: number; +} + +export interface EntityStreamRemoveItem< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> extends EntityStreamItemBase { + operation: "remove"; + oldValue?: Attr; + oldVersion?: number; +} + +export interface EntityStream< + Attr extends Attributes, + Partition extends CompositeKeyPart, + Sort extends CompositeKeyPart | undefined +> extends EntityStreamSpec { + kind: "EntityStream"; + handler: EntityStreamHandler; + sourceLocation?: SourceLocation; +} + +export function entityStream< + Attr extends Attributes, + const Partition extends CompositeKeyPart, + const Sort extends CompositeKeyPart | undefined +>( + ...args: + | [ + name: string, + entity: Entity, + handler: EntityStreamHandler + ] + | [ + name: string, + entity: Entity, + options: EntityStreamOptions, + handler: EntityStreamHandler + ] + | [ + sourceLocation: SourceLocation, + name: string, + entity: Entity, + handler: EntityStreamHandler + ] + | [ + sourceLocation: SourceLocation, + name: string, + entity: Entity, + options: EntityStreamOptions, + handler: EntityStreamHandler + ] +) { + const [sourceLocation, name, entity, options, handler] = + args.length === 3 + ? [, args[0], args[1], , args[2]] + : args.length === 5 + ? args + : isSourceLocation(args[0]) + ? [ + args[0], + args[1] as string, + args[2] as Entity, + , + args[3], + ] + : [ + , + args[0] as string, + args[1] as Entity, + args[2] as EntityStreamOptions, + args[3], + ]; + + return sourceLocation + ? options + ? // @ts-ignore + entity.stream(sourceLocation, name, options, handler) + : // @ts-ignore + entity.stream(sourceLocation, name, handler) + : options + ? entity.stream(name, options, handler) + : entity.stream(name, handler); +} diff --git a/packages/@eventual/core/src/error.ts b/packages/@eventual/core/src/error.ts index 02dd67d1c..7d3f82fda 100644 --- a/packages/@eventual/core/src/error.ts +++ b/packages/@eventual/core/src/error.ts @@ -124,8 +124,20 @@ export class UnexpectedVersion extends Error { * * Returns reasons in the same order as the input items. */ -export class TransactionCancelled extends Error { +export class TransactionCancelled extends EventualError { constructor(public reasons: (UnexpectedVersion | undefined)[]) { - super("Transactions Cancelled, see reasons"); + super("TransactionCancelled", "Transaction cancelled, see reasons"); + } +} + +/** + * Thrown when a transaction conflict with another conflict or write operation. + */ +export class TransactionConflict extends EventualError { + constructor() { + super( + "TransactionConflict", + "Transaction conflicted with another operation" + ); } } diff --git a/packages/@eventual/core/src/index.ts b/packages/@eventual/core/src/index.ts index bbbc3dea3..93742f78d 100644 --- a/packages/@eventual/core/src/index.ts +++ b/packages/@eventual/core/src/index.ts @@ -1,7 +1,7 @@ export * from "./await-time.js"; export * from "./bucket.js"; export * from "./condition.js"; -export * from "./entity.js"; +export * from "./entity/index.js"; export * from "./error.js"; export * from "./event.js"; export * from "./execution.js"; diff --git a/packages/@eventual/core/src/internal/calls.ts b/packages/@eventual/core/src/internal/calls.ts index 34c462cf2..76e6eda74 100644 --- a/packages/@eventual/core/src/internal/calls.ts +++ b/packages/@eventual/core/src/internal/calls.ts @@ -1,27 +1,22 @@ import type { Bucket } from "../bucket.js"; import type { ConditionPredicate } from "../condition.js"; -import type { - CompositeKey, - EntityConsistencyOptions, - EntityListRequest, - EntitySetOptions, - EntityTransactItem, -} from "../entity.js"; +import type { Entity, EntityTransactItem } from "../entity/entity.js"; import type { EventEnvelope } from "../event.js"; import type { DurationSchedule, Schedule } from "../schedule.js"; import type { WorkflowExecutionOptions } from "../workflow.js"; import type { BucketMethod } from "./bucket-hook.js"; +import type { EntityMethod } from "./entity-hook.js"; import type { SignalTarget } from "./signal.js"; export type EventualCall = | AwaitTimerCall | BucketCall + | ChildWorkflowCall | ConditionCall + | EmitEventsCall | EntityCall - | ChildWorkflowCall | ExpectSignalCall | InvokeTransactionCall - | EmitEventsCall | RegisterSignalHandlerCall | SendSignalCall | TaskCall; @@ -107,62 +102,25 @@ export type EntityCall< export function isEntityOperationOfType< OpType extends EntityOperation["operation"] ->( - operation: OpType, - call: EntityOperation -): call is EntityOperation & { operation: OpType } { +>(operation: OpType, call: EntityOperation): call is EntityOperation { return call.operation === operation; } -export interface EntityOperationBase { - name: string; -} - -export type EntityOperation = - | EntityDeleteOperation - | EntityGetOperation - | EntityGetWithMetadataOperation - | EntityListOperation - | EntityListKeysOperation - | EntitySetOperation - | EntityTransactOperation; - -export interface EntityGetOperation extends EntityOperationBase { - operation: "get"; - key: string | CompositeKey; -} - -export interface EntityGetWithMetadataOperation extends EntityOperationBase { - operation: "getWithMetadata"; - key: string | CompositeKey; -} - -export interface EntityDeleteOperation extends EntityOperationBase { - operation: "delete"; - key: string | CompositeKey; - options?: EntityConsistencyOptions; -} - -export interface EntitySetOperation extends EntityOperationBase { - operation: "set"; - key: string | CompositeKey; - value: E; - options?: EntitySetOptions; -} - -export interface EntityListOperation extends EntityOperationBase { - operation: "list"; - request: EntityListRequest; -} - -export interface EntityListKeysOperation extends EntityOperationBase { - operation: "listKeys"; - request: EntityListRequest; -} +export type EntityOperation< + Op extends EntityMethod | EntityTransactOperation["operation"] = + | EntityMethod + | EntityTransactOperation["operation"] +> = Op extends EntityMethod + ? { + operation: Op; + entityName: string; + params: Parameters; + } + : EntityTransactOperation; export interface EntityTransactOperation { operation: "transact"; - items: EntityTransactItem[]; + items: EntityTransactItem[]; } export function isBucketCall(a: any): a is BucketCall { diff --git a/packages/@eventual/core/src/internal/entity-hook.ts b/packages/@eventual/core/src/internal/entity-hook.ts index 0726dd9a5..9897de75a 100644 --- a/packages/@eventual/core/src/internal/entity-hook.ts +++ b/packages/@eventual/core/src/internal/entity-hook.ts @@ -1,19 +1,15 @@ -import { z } from "zod"; -import type { Entity, EntityTransactItem } from "../entity.js"; +import type { Entity, EntityTransactItem } from "../entity/entity.js"; declare global { // eslint-disable-next-line no-var var eventualEntityHook: EntityHook | undefined; } -export interface EntityDefinition { - name: string; - schema: z.Schema; -} - -export type EntityMethods = Pick< - Entity, - "get" | "getWithMetadata" | "delete" | "set" | "list" | "listKeys" +export type EntityMethod = Exclude< + { + [k in keyof Entity]: [Entity[k]] extends [Function] ? k : never; + }[keyof Entity], + "partition" | "sort" | "stream" | undefined >; /** @@ -21,10 +17,14 @@ export type EntityMethods = Pick< * * Does not handle the workflow case. That is handled by the {@link entity} function in core. */ -export interface EntityHook { - getEntity(name: string): Promise | undefined>; - transactWrite(items: EntityTransactItem[]): Promise; -} +export type EntityHook = { + [K in EntityMethod]: ( + entityName: string, + ...args: Parameters + ) => ReturnType; +} & { + transactWrite(items: EntityTransactItem[]): Promise; +}; export function getEntityHook() { const hook = globalThis.eventualEntityHook; diff --git a/packages/@eventual/core/src/internal/entity.ts b/packages/@eventual/core/src/internal/entity.ts new file mode 100644 index 000000000..16285893d --- /dev/null +++ b/packages/@eventual/core/src/internal/entity.ts @@ -0,0 +1,54 @@ +import { z } from "zod"; +import { Attributes, EntityZodShape } from "../entity/entity.js"; +import { CompositeKeyPart } from "../entity/key.js"; + +export interface KeyDefinitionPart { + type: "number" | "string"; + keyAttribute: string; + attributes: readonly string[]; +} + +export interface KeyDefinition { + partition: KeyDefinitionPart; + sort?: KeyDefinitionPart; +} + +export function computeKeyDefinition( + attributes: z.ZodObject>, + partition: CompositeKeyPart, + sort?: CompositeKeyPart +): KeyDefinition { + const entityZodShape = attributes.shape; + + return { + partition: formatKeyDefinitionPart(partition), + sort: sort ? formatKeyDefinitionPart(sort) : undefined, + }; + + function formatKeyDefinitionPart( + keyAttributes: CompositeKeyPart + ): KeyDefinitionPart { + const [head, ...tail] = keyAttributes; + + if (!head) { + throw new Error( + "Entity Key Part must contain at least one segment. Sort Key may be undefined." + ); + } + + // the value will be a number if there is a single part to the composite key part and the value is already a number. + // else a string will be formatted + const type = + tail.length === 0 && entityZodShape[head] instanceof z.ZodNumber + ? "number" + : "string"; + + const attribute = keyAttributes.join("|"); + + return { + type, + keyAttribute: attribute, + attributes: keyAttributes, + }; + } +} diff --git a/packages/@eventual/core/src/internal/global.ts b/packages/@eventual/core/src/internal/global.ts index 299c1f128..c5b8b217c 100644 --- a/packages/@eventual/core/src/internal/global.ts +++ b/packages/@eventual/core/src/internal/global.ts @@ -1,6 +1,6 @@ import type { AsyncLocalStorage } from "async_hooks"; import type { Bucket } from "../bucket.js"; -import type { Entity } from "../entity.js"; +import type { Entity } from "../entity/entity.js"; import type { Event } from "../event.js"; import type { AnyCommand } from "../http/command.js"; import type { EventualServiceClient } from "../service-client.js"; @@ -33,7 +33,7 @@ declare global { /** * A simple key value store that work efficiently within eventual. */ - entities?: Map>; + entities?: Map; /** * A data bucket within eventual. */ @@ -80,8 +80,8 @@ export const events = (): Map => export const subscriptions = (): Subscription[] => (globalThis._eventual.subscriptions ??= []); -export const entities = (): Map> => - (globalThis._eventual.entities ??= new Map>()); +export const entities = (): Map => + (globalThis._eventual.entities ??= new Map()); export const buckets = (): Map => (globalThis._eventual.buckets ??= new Map()); diff --git a/packages/@eventual/core/src/internal/index.ts b/packages/@eventual/core/src/internal/index.ts index 990afda07..037657d52 100644 --- a/packages/@eventual/core/src/internal/index.ts +++ b/packages/@eventual/core/src/internal/index.ts @@ -1,6 +1,7 @@ export * from "./bucket-hook.js"; export * from "./calls.js"; export * from "./entity-hook.js"; +export * from "./entity.js"; export * from "./eventual-hook.js"; export * from "./eventual-service.js"; export * from "./flags.js"; diff --git a/packages/@eventual/core/src/internal/service-spec.ts b/packages/@eventual/core/src/internal/service-spec.ts index e8806534e..c0c9a9110 100644 --- a/packages/@eventual/core/src/internal/service-spec.ts +++ b/packages/@eventual/core/src/internal/service-spec.ts @@ -1,4 +1,6 @@ import type openapi from "openapi3-ts"; +import { Attributes } from "../entity/entity.js"; +import { CompositeKeyPart, StreamQueryKey } from "../entity/key.js"; import type { FunctionRuntimeProps } from "../function-props.js"; import type { HttpMethod } from "../http-method.js"; import type { RestParams } from "../http/command.js"; @@ -7,6 +9,7 @@ import type { SubscriptionFilter, SubscriptionRuntimeProps, } from "../subscription.js"; +import { KeyDefinition } from "./entity.js"; import type { TaskSpec } from "./task.js"; /** @@ -163,16 +166,23 @@ export interface BucketNotificationHandlerSpec { export interface EntitySpec { name: string; + key: KeyDefinition; /** * An Optional schema for the entity within an entity. */ - schema?: openapi.SchemaObject; + attributes: openapi.SchemaObject; streams: EntityStreamSpec[]; } export type EntityStreamOperation = "insert" | "modify" | "remove"; -export interface EntityStreamOptions extends FunctionRuntimeProps { +export interface EntityStreamOptions< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> extends FunctionRuntimeProps { /** * A list of operations to be send to the stream. * @@ -184,23 +194,21 @@ export interface EntityStreamOptions extends FunctionRuntimeProps { */ includeOld?: boolean; /** - * A subset of namespaces to include in the stream. - * - * If neither `namespaces` or `namespacePrefixes` are provided, all namespaces will be sent. - */ - namespaces?: string[]; - /** - * One or more namespace prefixes to match. - * - * If neither `namespaces` or `namespacePrefixes` are provided, all namespaces will be sent. + * One or more key queries that will be included in the stream. */ - namespacePrefixes?: string[]; + queryKeys?: StreamQueryKey[]; } -export interface EntityStreamSpec { +export interface EntityStreamSpec< + Attr extends Attributes = Attributes, + Partition extends CompositeKeyPart = CompositeKeyPart, + Sort extends CompositeKeyPart | undefined = + | CompositeKeyPart + | undefined +> { name: string; entityName: string; - options?: EntityStreamOptions; + options?: EntityStreamOptions; sourceLocation?: SourceLocation; } diff --git a/packages/@eventual/core/src/service-client.ts b/packages/@eventual/core/src/service-client.ts index 58d7c3c6d..97a79888c 100644 --- a/packages/@eventual/core/src/service-client.ts +++ b/packages/@eventual/core/src/service-client.ts @@ -173,7 +173,7 @@ export interface ExecuteTransactionRequest< } export type ExecuteTransactionResponse = - | { succeeded: false } + | { succeeded: false; error: string; message: string } | { output: TransactionOutput; succeeded: true; diff --git a/packages/@eventual/core/src/transaction.ts b/packages/@eventual/core/src/transaction.ts index 270ae0c77..197f58e4f 100644 --- a/packages/@eventual/core/src/transaction.ts +++ b/packages/@eventual/core/src/transaction.ts @@ -1,8 +1,8 @@ -import { TransactionCancelled } from "./error.js"; +import { EventualError } from "./error.js"; import { + createEventualCall, EventualCallKind, InvokeTransactionCall, - createEventualCall, } from "./internal/calls.js"; import { getServiceClient, transactions } from "./internal/global.js"; import { TransactionSpec } from "./internal/service-spec.js"; @@ -63,10 +63,9 @@ export function transaction( if (response.succeeded) { return response.output; + } else { + throw new EventualError(response.error, response.message); } - - // todo: return reason? - throw new TransactionCancelled([]); } ); }) as any; diff --git a/packages/@eventual/testing/src/environment.ts b/packages/@eventual/testing/src/environment.ts index f2a6ed8b3..a2942c587 100644 --- a/packages/@eventual/testing/src/environment.ts +++ b/packages/@eventual/testing/src/environment.ts @@ -407,9 +407,15 @@ export class TestEnvironment extends RuntimeServiceClient { } }), entityStreamItems.flatMap((i) => { + const entity = this.localContainer.entityProvider.getEntity( + i.entityName + ); + if (!entity) { + return []; + } const streamNames = [...entities().values()] .flatMap((d) => d.streams) - .filter((s) => entityStreamMatchesItem(i, s)) + .filter((s) => entityStreamMatchesItem(entity, i, s)) .map((s) => s.name); return streamNames.map((streamName) => { return this.localContainer.entityStreamWorker({ diff --git a/packages/@eventual/testing/test/env.test.ts b/packages/@eventual/testing/test/env.test.ts index 9019fc9f5..481553ce8 100644 --- a/packages/@eventual/testing/test/env.test.ts +++ b/packages/@eventual/testing/test/env.test.ts @@ -1,18 +1,17 @@ import type { SQSClient } from "@aws-sdk/client-sqs"; import { - CompositeKey, Entity, + entity, EventPayloadType, EventualError, Execution, ExecutionStatus, SubscriptionHandler, + task as _task, TaskHandler, Timeout, - WorkflowHandler, - task as _task, workflow as _workflow, - entity, + WorkflowHandler, } from "@eventual/core"; import { tasks, workflows } from "@eventual/core/internal"; import { jest } from "@jest/globals"; @@ -1292,18 +1291,27 @@ describe("time", () => { }); }); -const myEntity = entity<{ n: number }>("testEntity1", z.any()); +const myEntity = entity("testEntity1", { + attributes: { n: z.number(), id: z.string() }, + partition: ["id"], +}); + +const myEntityWithSort = entity("testEntity2", { + attributes: { n: z.number(), id: z.string(), part: z.string() }, + partition: ["part"], + sort: ["id"], +}); describe("entity", () => { test("workflow and task uses get and set", async () => { const entityTask = task(async (_, { execution: { id } }) => { - await myEntity.set(id, { n: ((await myEntity.get(id))?.n ?? 0) + 1 }); + await myEntity.set({ id, n: ((await myEntity.get([id]))?.n ?? 0) + 1 }); }); const wf = workflow(async (_, { execution: { id } }) => { - await myEntity.set(id, { n: 1 }); + await myEntity.set({ id, n: 1 }); await entityTask(); - const value = await myEntity.get(id); - myEntity.delete(id); + const value = await myEntity.get([id]); + myEntity.delete({ id }); return value; }); @@ -1317,27 +1325,28 @@ describe("entity", () => { await expect(execution.getStatus()).resolves.toMatchObject< Partial> >({ - result: { n: 2 }, + result: { n: 2, id: execution.executionId }, status: ExecutionStatus.SUCCEEDED, }); }); - test("workflow and task uses get and set with namespaces", async () => { - const entityTask = task( - async (namespace: string, { execution: { id } }) => { - const key = { namespace, key: id }; - await myEntity.set(key, { n: ((await myEntity.get(key))?.n ?? 0) + 1 }); - } - ); + test("workflow and task uses get and set with partitions and sort keys", async () => { + const entityTask = task(async (part: string, { execution: { id } }) => { + const key = { part, id }; + await myEntityWithSort.set({ + ...key, + n: ((await myEntityWithSort.get(key))?.n ?? 0) + 1, + }); + }); const wf = workflow(async (_, { execution: { id } }) => { - await myEntity.set({ namespace: "1", key: id }, { n: 1 }); - await myEntity.set({ namespace: "2", key: id }, { n: 100 }); + await myEntityWithSort.set({ part: "1", id, n: 1 }); + await myEntityWithSort.set({ part: "2", id, n: 100 }); await entityTask("1"); await entityTask("2"); - const value = await myEntity.get({ namespace: "1", key: id }); - const value2 = await myEntity.get({ namespace: "2", key: id }); - await myEntity.delete({ namespace: "1", key: id }); - await myEntity.delete({ namespace: "2", key: id }); + const value = await myEntityWithSort.get({ part: "1", id }); + const value2 = await myEntityWithSort.get({ part: "2", id }); + await myEntityWithSort.delete({ part: "1", id }); + await myEntityWithSort.delete({ part: "2", id }); return value!.n + value2!.n; }); @@ -1358,37 +1367,41 @@ describe("entity", () => { test("version", async () => { const wf = workflow(async (_, { execution: { id } }) => { - const key: CompositeKey = { namespace: "versionTest", key: id }; + const key = { part: "versionTest", id }; // set - version 1 - value 1 - const { version } = await myEntity.set(key, { n: 1 }); + const { version } = await myEntityWithSort.set({ ...key, n: 1 }); // set - version 2 - value 2 - const { version: version2 } = await myEntity.set( - key, - { n: 2 }, + const { version: version2 } = await myEntityWithSort.set( + { ...key, n: 2 }, { expectedVersion: version } ); try { // try set to 3, fail - await myEntity.set(key, { n: 3 }, { expectedVersion: version }); + await myEntityWithSort.set( + { ...key, n: 3 }, + { expectedVersion: version } + ); } catch { // set - version 2 (unchanged) - value 3 - await myEntity.set( - key, - { n: 4 }, + await myEntityWithSort.set( + { ...key, n: 4 }, { expectedVersion: version2, incrementVersion: false } ); } try { // try delete and fail - await myEntity.delete(key, { expectedVersion: version }); + await myEntityWithSort.delete(key, { expectedVersion: version }); } catch { // set - version 3 - value 5 - await myEntity.set(key, { n: 5 }, { expectedVersion: version2 }); + await myEntityWithSort.set( + { ...key, n: 5 }, + { expectedVersion: version2 } + ); } - const value = await myEntity.getWithMetadata(key); + const value = await myEntityWithSort.getWithMetadata(key); // delete at version 3 - myEntity.delete(key, { expectedVersion: value!.version }); + myEntityWithSort.delete(key, { expectedVersion: value!.version }); return value; }); @@ -1402,7 +1415,10 @@ describe("entity", () => { await expect(execution.getStatus()).resolves.toMatchObject< Partial> >({ - result: { entity: { n: 5 }, version: 3 }, + result: { + value: { n: 5, id: execution.executionId, part: "versionTest" }, + version: 3, + }, status: ExecutionStatus.SUCCEEDED, }); }); @@ -1412,51 +1428,53 @@ describe("entity", () => { async ( { version, - namespace, + partition, value, - }: { version: number; namespace?: string; value: number }, + }: { version: number; partition?: string; value: number }, { execution: { id } } ) => { return Entity.transactWrite([ + partition + ? { + operation: "set", + value: { part: partition, id, n: value }, + options: { expectedVersion: version }, + entity: myEntityWithSort, + } + : { + operation: "set", + value: { id, n: value }, + options: { expectedVersion: version }, + entity: myEntity, + }, { - operation: { - operation: "set", - key: namespace ? { key: id, namespace } : id, - value: { n: value }, - options: { expectedVersion: version }, - }, - entity: myEntity, - }, - { - operation: { - operation: "set", - key: { key: id, namespace: "3" }, - value: { n: value }, - options: { expectedVersion: version }, - }, - entity: myEntity, + operation: "set", + value: { part: "3", id, n: value }, + options: { expectedVersion: version }, + entity: myEntityWithSort, }, ]); } ); const wf = workflow(async (_, { execution: { id } }) => { - const { version: version1 } = await myEntity.set(id, { n: 1 }); - const { version: version2 } = await myEntity.set( - { key: id, namespace: "2" }, - { n: 1 } - ); - await myEntity.set({ key: id, namespace: "3" }, { n: 1 }); + const { version: version1 } = await myEntity.set({ id, n: 1 }); + const { version: version2 } = await myEntityWithSort.set({ + id, + part: "2", + n: 1, + }); + await myEntityWithSort.set({ id, part: "3", n: 1 }); await testTask({ version: version1, value: 2 }); try { - await testTask({ version: version2 + 1, namespace: "2", value: 3 }); + await testTask({ version: version2 + 1, partition: "2", value: 3 }); } catch {} return Promise.all([ - myEntity.get(id), - myEntity.get({ key: id, namespace: "2" }), - myEntity.get({ key: id, namespace: "3" }), + myEntity.get([id]), + myEntityWithSort.get({ id, part: "2" }), + myEntityWithSort.get({ id, part: "3" }), ]); }); @@ -1470,7 +1488,11 @@ describe("entity", () => { await expect(execution.getStatus()).resolves.toMatchObject< Partial> >({ - result: [{ n: 2 }, { n: 1 }, { n: 2 }], + result: [ + { n: 2, id: execution.executionId }, + { part: "2", n: 1, id: execution.executionId }, + { part: "3", n: 2, id: execution.executionId }, + ], status: ExecutionStatus.SUCCEEDED, }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 74642bfe5..6ff8a0d08 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -180,7 +180,7 @@ importers: version: link:../../packages/@eventual/aws-cdk '@serverless-stack/cli': specifier: ^1.18.4 - version: 1.18.4(constructs@10.1.154) + version: 1.18.4(constructs@10.2.17) '@serverless-stack/core': specifier: ^1.18.4 version: 1.18.4 @@ -192,7 +192,7 @@ importers: version: 1.0.1 aws-cdk-lib: specifier: 2.50.0 - version: 2.50.0(constructs@10.1.154) + version: 2.50.0(constructs@10.2.17) chalk: specifier: ^5.2.0 version: 5.2.0 @@ -468,6 +468,9 @@ importers: '@aws-sdk/s3-request-presigner': specifier: ^3.254.0 version: 3.319.0 + '@aws-sdk/util-dynamodb': + specifier: ^3.254.0 + version: 3.319.0 '@eventual/aws-client': specifier: workspace:^ version: link:../aws-client @@ -990,6 +993,17 @@ packages: constructs: 10.1.154 dev: true + /@aws-cdk/aws-apigatewayv2-alpha@2.50.0-alpha.0(aws-cdk-lib@2.50.0)(constructs@10.2.17): + resolution: {integrity: sha512-dttWDqy+nTg/fD9y0egvj7/zdnOVEo0qyGsep1RV+p16R3F4ObMKyPVIg15fz57tK//Gp/i1QgXsZaSqbcWHOg==} + engines: {node: '>= 14.15.0'} + peerDependencies: + aws-cdk-lib: ^2.50.0 + constructs: ^10.0.0 + dependencies: + aws-cdk-lib: 2.50.0(constructs@10.2.17) + constructs: 10.2.17 + dev: true + /@aws-cdk/aws-apigatewayv2-authorizers-alpha@2.50.0-alpha.0(@aws-cdk/aws-apigatewayv2-alpha@2.50.0-alpha.0)(aws-cdk-lib@2.50.0)(constructs@10.1.154): resolution: {integrity: sha512-lMXnSpUSOYtCxoAxauNkGJZLsKMonHgd9rzlFUK2zxE7aC1lVwb4qYX4X9WJdvIExkFOHSZQzOTKM6SZqusssw==} engines: {node: '>= 14.15.0'} @@ -5198,7 +5212,7 @@ packages: resolution: {integrity: sha512-QyKIWEnKQFnYu2ey+SAAm1A5xjzJLJJj3bhIZd3QKyXKKjaJ0hlxam/OsWSltxTNbcyH1jRJjC6Cxv31usv0Ag==} engines: {node: ^14.17.0 || >=16.0.0} dependencies: - chalk: 4.1.2 + chalk: 4.1.0 execa: 5.0.0 strong-log-transformer: 2.1.0 dev: true @@ -5366,7 +5380,7 @@ packages: resolution: {integrity: sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==} dependencies: '@gar/promisify': 1.1.3 - semver: 7.5.0 + semver: 7.3.8 dev: true /@npmcli/fs@2.1.2: @@ -5813,15 +5827,15 @@ packages: - supports-color dev: true - /@serverless-stack/cli@1.18.4(constructs@10.1.154): + /@serverless-stack/cli@1.18.4(constructs@10.2.17): resolution: {integrity: sha512-eEG3brlbF/ptIo/s69Hcrn185CVkLWHpmtOmere7+lMPkmy1vxNhWIUuic+LNG0yweK+sg4uMVipREyvwblNDQ==} hasBin: true dependencies: - '@aws-cdk/aws-apigatewayv2-alpha': 2.50.0-alpha.0(aws-cdk-lib@2.50.0)(constructs@10.1.154) + '@aws-cdk/aws-apigatewayv2-alpha': 2.50.0-alpha.0(aws-cdk-lib@2.50.0)(constructs@10.2.17) '@serverless-stack/core': 1.18.4 '@serverless-stack/resources': 1.18.4 aws-cdk: 2.50.0 - aws-cdk-lib: 2.50.0(constructs@10.1.154) + aws-cdk-lib: 2.50.0(constructs@10.2.17) aws-sdk: 2.1314.0 body-parser: 1.20.1 chalk: 4.1.2 @@ -7079,6 +7093,34 @@ packages: - semver - yaml + /aws-cdk-lib@2.50.0(constructs@10.2.17): + resolution: {integrity: sha512-deDbZTI7oyu3rqUyqjwhP6tnUO8MD70lE98yR65xiYty4yXBpsWKbeH3s1wNLpLAWS3hWJYyMtjZ4ZfC35NtVg==} + engines: {node: '>= 14.15.0'} + peerDependencies: + constructs: ^10.0.0 + dependencies: + '@balena/dockerignore': 1.0.2 + case: 1.6.3 + constructs: 10.2.17 + fs-extra: 9.1.0 + ignore: 5.2.4 + jsonschema: 1.4.1 + minimatch: 3.1.2 + punycode: 2.3.0 + semver: 7.3.8 + yaml: 1.10.2 + dev: true + bundledDependencies: + - '@balena/dockerignore' + - case + - fs-extra + - ignore + - jsonschema + - minimatch + - punycode + - semver + - yaml + /aws-cdk@2.50.0: resolution: {integrity: sha512-55vmKTf2DZRqioumVfXn+S0H9oAbpRK3HFHY8EjZ5ykR5tq2+XiMWEZkYduX2HJhVAeHJJIS6h+Okk3smZjeqw==} engines: {node: '>= 14.15.0'} @@ -7353,7 +7395,7 @@ packages: resolution: {integrity: sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==} dependencies: base64-js: 1.5.1 - ieee754: 1.2.1 + ieee754: 1.1.13 isarray: 1.0.0 /buffer@5.7.1: @@ -7808,7 +7850,7 @@ packages: json-schema-typed: 7.0.3 onetime: 5.1.2 pkg-up: 3.1.0 - semver: 7.5.0 + semver: 7.3.8 dev: true /config-chain@1.1.12: @@ -7826,6 +7868,11 @@ packages: resolution: {integrity: sha512-JStQT84+NhsfamESRExZoGzpq/f/gpq9xpzgtQNOzungs42Gy8kxjfU378MnVoRqwCHwk0vLN37HZjgH5tJo2A==} engines: {node: '>= 14.17.0'} + /constructs@10.2.17: + resolution: {integrity: sha512-D3a/+iKMkBj8Elf1NIl1jBNIrK07Pg7ICBe5QEgtEKYEZOuHQvlCK9PK1f87SQ+GhtOnwSyvP+q+Pq5zBLu5kg==} + engines: {node: '>= 14.17.0'} + dev: true + /content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -8130,7 +8177,7 @@ packages: engines: {node: '>=10'} dependencies: globby: 11.1.0 - graceful-fs: 4.2.11 + graceful-fs: 4.2.10 is-glob: 4.0.3 is-path-cwd: 2.2.0 is-path-inside: 3.0.3 @@ -9739,7 +9786,7 @@ packages: resolution: {integrity: sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==} engines: {node: '>=14.14'} dependencies: - graceful-fs: 4.2.11 + graceful-fs: 4.2.10 jsonfile: 6.1.0 universalify: 2.0.0 dev: true @@ -9758,7 +9805,7 @@ packages: engines: {node: '>=10'} dependencies: at-least-node: 1.0.0 - graceful-fs: 4.2.10 + graceful-fs: 4.2.11 jsonfile: 6.1.0 universalify: 2.0.0 @@ -9993,7 +10040,7 @@ packages: fs.realpath: 1.0.0 inflight: 1.0.6 inherits: 2.0.4 - minimatch: 3.1.2 + minimatch: 3.0.5 once: 1.4.0 path-is-absolute: 1.0.1 dev: true @@ -10066,6 +10113,7 @@ packages: /graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} + dev: true /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -10794,7 +10842,7 @@ packages: async: 3.2.4 chalk: 4.1.2 filelist: 1.0.4 - minimatch: 3.1.2 + minimatch: 3.0.5 dev: true /jest-changed-files@29.4.2: @@ -11659,7 +11707,7 @@ packages: resolution: {integrity: sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==} engines: {node: '>=4'} dependencies: - graceful-fs: 4.2.11 + graceful-fs: 4.2.10 parse-json: 4.0.0 pify: 3.0.0 strip-bom: 3.0.0 @@ -11669,7 +11717,7 @@ packages: resolution: {integrity: sha512-gUD/epcRms75Cw8RT1pUdHugZYM5ce64ucs2GEISABwkRsOQr0q2wm/MV2TKThycIe5e0ytRweW2RZxclogCdQ==} engines: {node: '>=8'} dependencies: - graceful-fs: 4.2.11 + graceful-fs: 4.2.10 parse-json: 5.2.0 strip-bom: 4.0.0 type-fest: 0.6.0 @@ -12232,7 +12280,7 @@ packages: array-differ: 3.0.0 array-union: 2.1.0 arrify: 2.0.1 - minimatch: 3.1.2 + minimatch: 3.0.5 dev: true /mute-stream@0.0.8: @@ -12318,7 +12366,7 @@ packages: nopt: 5.0.0 npmlog: 4.1.2 rimraf: 3.0.2 - semver: 7.5.0 + semver: 7.3.8 tar: 6.1.14 which: 2.0.2 transitivePeerDependencies: @@ -12333,7 +12381,7 @@ packages: dependencies: env-paths: 2.2.1 glob: 7.2.3 - graceful-fs: 4.2.11 + graceful-fs: 4.2.10 make-fetch-happen: 10.2.1 nopt: 6.0.0 npmlog: 6.0.2 @@ -15252,7 +15300,7 @@ packages: /write-file-atomic@2.4.3: resolution: {integrity: sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==} dependencies: - graceful-fs: 4.2.11 + graceful-fs: 4.2.10 imurmurhash: 0.1.4 signal-exit: 3.0.7 dev: true @@ -15286,7 +15334,7 @@ packages: engines: {node: '>=6'} dependencies: detect-indent: 5.0.0 - graceful-fs: 4.2.11 + graceful-fs: 4.2.10 make-dir: 2.1.0 pify: 4.0.1 sort-keys: 2.0.0