diff --git a/.vscode/settings.json b/.vscode/settings.json index 526db4d36..8fc0e22b7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -40,6 +40,7 @@ "editor.insertSpaces": true, "editor.formatOnSave": true, "eslint.format.enable": false, + "prettier.prettierPath": "./node_modules/prettier", "typescript.tsdk": "node_modules/typescript/lib", "typescript.preferences.importModuleSpecifier": "relative" } diff --git a/apps/tests/aws-runtime/test/test-service.ts b/apps/tests/aws-runtime/test/test-service.ts index 1f480d8b3..a2f92266a 100644 --- a/apps/tests/aws-runtime/test/test-service.ts +++ b/apps/tests/aws-runtime/test/test-service.ts @@ -32,7 +32,7 @@ import { transaction, workflow, } from "@eventual/core"; -import type openapi from "openapi3-ts"; +import openapi from "openapi3-ts"; import { Readable } from "stream"; import z from "zod"; import { AsyncWriterTestEvent } from "./async-writer-handler.js"; @@ -555,6 +555,7 @@ export const counter = entity("counter5", { z.literal("different"), z.literal("default"), z.literal("another"), + z.literal("another2"), ]), id: z.string(), optional: z.string().optional(), @@ -670,39 +671,54 @@ export const entityIndexTask = task( n: 1000, optional: "hello", }); - return await Promise.all([ - allCounters.query({ id }).then((q) => - q.entries?.map((e) => ({ - n: e.value.n, - namespace: e.value.namespace, - })) - ), - allCountersByN.query({ id }).then((q) => - q.entries?.map((e) => ({ - n: e.value.n, - namespace: e.value.namespace, - })) - ), - countersByNamespace.query({ id }).then((q) => - q.entries?.map((e) => ({ - n: e.value.n, - namespace: e.value.namespace, - })) - ), - countersByN.query({ namespace: "another", id }).then((q) => - q.entries?.map((e) => ({ - n: e.value.n, - namespace: e.value.namespace, - })) + await counter.set({ + namespace: "another2", + id, + n: 0, + optional: "hello", + }); + const queries = { + all: allCounters.query({ id }), + byN: allCountersByN.query({ id }), + byNamespace: countersByNamespace.query({ id }), + filterByNamespace: countersByN.query({ namespace: "another", id }), + betweenN: allCountersByN.query({ + id, + n: { $between: [2, 100] }, + }), + greaterThanN: allCountersByN.query({ + id, + n: { $gt: 1 }, + }), + reverse: countersByNamespace.query( + { id, namespace: { $beginsWith: "d" } }, + { direction: "DESC" } ), // sparse indices only include records with the given field - countersByOptional2.query({ id }).then((q) => - q.entries?.map((e) => ({ - n: e.value.n, - namespace: e.value.namespace, - })) - ), - ]); + sparse: countersByOptional2.query({ id }), + // between using a multi-attribute key + inlineBetween: countersByOptional2.query({ + id, + $between: [{ optional: "h" }, { optional: "hello", n: 0 }], + }), + }; + + return Object.fromEntries( + await Promise.all( + Object.entries(queries).map( + async ([name, q]) => + [ + name, + ( + await q + ).entries?.map((e) => ({ + n: e.value.n, + namespace: e.value.namespace, + })), + ] as const + ) + ) + ) as Record; } ); diff --git a/apps/tests/aws-runtime/test/tester.test.ts b/apps/tests/aws-runtime/test/tester.test.ts index 34056dda7..e3394a187 100644 --- a/apps/tests/aws-runtime/test/tester.test.ts +++ b/apps/tests/aws-runtime/test/tester.test.ts @@ -120,8 +120,8 @@ eventualRuntimeTestHarness( testCompletion("awsSdkCalls", createAndDestroyWorkflow, "done"); testCompletion("ent", entityWorkflow, [ - [ - expect.arrayContaining([ + { + all: expect.arrayContaining([ { namespace: "different", n: 1, @@ -134,8 +134,16 @@ eventualRuntimeTestHarness( namespace: "default", n: 6, }, + { + namespace: "another2", + n: 0, + }, ]), - [ + byN: [ + { + namespace: "another2", + n: 0, + }, { namespace: "different", n: 1, @@ -149,11 +157,15 @@ eventualRuntimeTestHarness( n: 1000, }, ], - [ + byNamespace: [ { namespace: "another", n: 1000, }, + { + namespace: "another2", + n: 0, + }, { namespace: "default", n: 6, @@ -163,19 +175,55 @@ eventualRuntimeTestHarness( n: 1, }, ], - [ + filterByNamespace: [ { namespace: "another", n: 1000, }, ], - [ + betweenN: [ + { + namespace: "default", + n: 6, + }, + ], + greaterThanN: [ + { + namespace: "default", + n: 6, + }, { namespace: "another", n: 1000, }, ], - ], + reverse: [ + { + namespace: "different", + n: 1, + }, + { + namespace: "default", + n: 6, + }, + ], + sparse: [ + { + namespace: "another2", + n: 0, + }, + { + namespace: "another", + n: 1000, + }, + ], + inlineBetween: [ + { + namespace: "another2", + n: 0, + }, + ], + }, { n: 7 }, [ [1, 1], diff --git a/packages/@eventual/aws-runtime/src/stores/entity-store.ts b/packages/@eventual/aws-runtime/src/stores/entity-store.ts index f8e0d3685..45927a3e3 100644 --- a/packages/@eventual/aws-runtime/src/stores/entity-store.ts +++ b/packages/@eventual/aws-runtime/src/stores/entity-store.ts @@ -25,6 +25,7 @@ import { EntityScanOptions, EntitySetOptions, EntityWithMetadata, + KeyValue, TransactionCancelled, TransactionConflict, UnexpectedVersion, @@ -34,15 +35,26 @@ import { EntityProvider, EntityStore, getLazy, + isNormalizedEntityQueryKeyConditionPart, LazyValue, NormalizedEntityCompositeKey, NormalizedEntityCompositeKeyComplete, - NormalizedEntityKeyCompletePart, + NormalizedEntityCompositeQueryKey, + NormalizedEntityQueryKeyPart, NormalizedEntityTransactItem, removeGeneratedKeyAttributes, removeKeyAttributes, } from "@eventual/core-runtime"; -import { assertNever } from "@eventual/core/internal"; +import { + assertNever, + isBeginsWithQueryKeyCondition, + isBetweenQueryKeyCondition, + isGreaterThanEqualsQueryKeyCondition, + isGreaterThanQueryKeyCondition, + isLessThanEqualsQueryKeyCondition, + isLessThanQueryKeyCondition, + KeyDefinitionPart, +} from "@eventual/core/internal"; import { entityServiceTableName, isAwsErrorOfType, @@ -187,7 +199,7 @@ export class AWSEntityStore extends EntityStore { protected override async _query( entity: Entity | EntityIndex, - queryKey: NormalizedEntityCompositeKey, + queryKey: NormalizedEntityCompositeQueryKey, options?: EntityQueryOptions ): Promise { const [_entity, _index] = @@ -195,11 +207,19 @@ export class AWSEntityStore extends EntityStore { ? [entity, undefined] : [this.getEntity(entity.entityName), entity]; + const partitionCondition = `${formatAttributeNameMapKey( + queryKey.partition.keyAttribute + )}=:pk`; + + const { + expression: sortExpression, + attribute: sortAttribute, + attributeValueMap: sortAttributeValueMap, + } = getSortKeyExpressionAndAttribute(queryKey.sort) ?? {}; + const allAttributes = new Set([ queryKey.partition.keyAttribute, - ...(queryKey.sort && queryKey.sort.keyValue !== undefined - ? [queryKey.sort.keyAttribute] - : []), + ...(sortAttribute ? [sortAttribute] : []), ]); const result = await queryPageWithToken< @@ -218,39 +238,15 @@ export class AWSEntityStore extends EntityStore { IndexName: _index?.name, ConsistentRead: options?.consistentRead, ScanIndexForward: !options?.direction || options?.direction === "ASC", // default is ASC, ascending - KeyConditionExpression: - queryKey.sort && - (queryKey.sort.keyValue !== undefined || - queryKey.sort.keyValue === "") - ? queryKey.sort.partialValue - ? `${formatAttributeNameMapKey( - queryKey.partition.keyAttribute - )}=:pk AND begins_with(${formatAttributeNameMapKey( - queryKey.sort.keyAttribute - )}, :sk)` - : `${formatAttributeNameMapKey( - queryKey.partition.keyAttribute - )}=:pk AND ${formatAttributeNameMapKey( - queryKey.sort.keyAttribute - )}=:sk` - : `${formatAttributeNameMapKey( - queryKey.partition.keyAttribute - )}=:pk`, + KeyConditionExpression: sortExpression + ? [partitionCondition, sortExpression].join(" AND ") + : partitionCondition, ExpressionAttributeValues: { - ":pk": - typeof queryKey.partition.keyValue === "number" - ? { N: queryKey.partition.keyValue.toString() } - : { S: queryKey.partition.keyValue }, - ...(queryKey.sort && - (queryKey.sort.keyValue !== undefined || - queryKey.sort.keyValue === "") - ? { - ":sk": - typeof queryKey.sort.keyValue === "string" - ? { S: queryKey.sort.keyValue } - : { N: queryKey.sort.keyValue.toString() }, - } - : {}), + ":pk": keyPartAttributeValue( + queryKey.partition, + queryKey.partition.keyValue + ), + ...sortAttributeValueMap, }, ExpressionAttributeNames: Object.fromEntries( [...allAttributes]?.map((f) => [formatAttributeNameMapKey(f), f]) @@ -464,6 +460,7 @@ export class AWSEntityStore extends EntityStore { } private entityKey(key: NormalizedEntityCompositeKey) { + console.debug("Key", JSON.stringify(key)); const marshalledKey = marshall( { [key.partition.keyAttribute]: key.partition.keyValue, @@ -471,6 +468,7 @@ export class AWSEntityStore extends EntityStore { }, { removeUndefinedValues: true } ); + console.debug("Marshalled Key", JSON.stringify(marshalledKey)); return marshalledKey; } @@ -502,3 +500,90 @@ function formatAttributeValueMapKey(key: string) { function formatAttributeMapKey(key: string, prefix: string) { return `${prefix}${key.replaceAll(/[|.\- ]/g, "_")}`; } + +function getSortKeyExpressionAndAttribute( + keyPart?: NormalizedEntityQueryKeyPart +): + | undefined + | { + expression: string; + attribute: string; + attributeValueMap: Record; + } { + // sort key is undefined, return undefined + if (!keyPart) { + return undefined; + } + + const attributeNameKey = formatAttributeNameMapKey(keyPart.keyAttribute); + + // if the key part is a condition key part + if (isNormalizedEntityQueryKeyConditionPart(keyPart)) { + if (isBetweenQueryKeyCondition(keyPart.condition)) { + return { + attribute: keyPart.keyAttribute, + expression: `${attributeNameKey} BETWEEN :skLeft AND :skRight`, + attributeValueMap: { + ":skLeft": keyPartAttributeValue( + keyPart, + keyPart.condition.$between[0] + ), + ":skRight": keyPartAttributeValue( + keyPart, + keyPart.condition.$between[1] + ), + }, + }; + } else { + const [value, expression] = isBeginsWithQueryKeyCondition( + keyPart.condition + ) + ? [ + keyPart.condition.$beginsWith, + `begins_with(${attributeNameKey}, :sk)`, + ] + : isLessThanQueryKeyCondition(keyPart.condition) + ? [keyPart.condition.$lt, `${attributeNameKey} < :sk`] + : isLessThanEqualsQueryKeyCondition(keyPart.condition) + ? [keyPart.condition.$lte, `${attributeNameKey} <= :sk`] + : isGreaterThanQueryKeyCondition(keyPart.condition) + ? [keyPart.condition.$gt, `${attributeNameKey} > :sk`] + : isGreaterThanEqualsQueryKeyCondition(keyPart.condition) + ? [keyPart.condition.$gte, `${attributeNameKey} >= :sk`] + : assertNever(keyPart.condition); + + return { + expression, + attribute: keyPart.keyAttribute, + attributeValueMap: { + ":sk": keyPartAttributeValue(keyPart, value), + }, + }; + } + } else if (keyPart.keyValue === undefined) { + // if the key value is undefined (no key part attributes given), return undefined. + return undefined; + } + // finally, format the prefix or exact match use cases based on if the key part attributes are partial or not. + return { + expression: keyPart.partialValue + ? `begins_with(${attributeNameKey}, :sk)` + : `${attributeNameKey}=:sk`, + attribute: keyPart.keyAttribute, + attributeValueMap: { + ":sk": keyPartAttributeValue(keyPart, keyPart.keyValue), + }, + }; +} + +/** + * Given a key part, format the value based on the key part's type. + */ +function keyPartAttributeValue( + part: KeyDefinitionPart, + value: KeyValue +): AttributeValue { + return part.type === "string" + ? { S: value.toString() } + : { N: value.toString() }; +} diff --git a/packages/@eventual/core-runtime/src/local/stores/entity-store.ts b/packages/@eventual/core-runtime/src/local/stores/entity-store.ts index 6abaf72cb..23d83c25b 100644 --- a/packages/@eventual/core-runtime/src/local/stores/entity-store.ts +++ b/packages/@eventual/core-runtime/src/local/stores/entity-store.ts @@ -12,16 +12,27 @@ import { TransactionCancelled, UnexpectedVersion, } from "@eventual/core"; -import { assertNever } from "@eventual/core/internal"; +import { + KeyDefinition, + assertNever, + isBeginsWithQueryKeyCondition, + isBetweenQueryKeyCondition, + isGreaterThanEqualsQueryKeyCondition, + isGreaterThanQueryKeyCondition, + isLessThanEqualsQueryKeyCondition, + isLessThanQueryKeyCondition, +} from "@eventual/core/internal"; import { EntityProvider } from "../../providers/entity-provider.js"; import { EntityStore, - NormalizedEntityCompositeKey, NormalizedEntityCompositeKeyComplete, + NormalizedEntityCompositeQueryKey, NormalizedEntityKeyCompletePart, + NormalizedEntityQueryKeyPart, NormalizedEntityTransactItem, convertNormalizedEntityKeyToMap, isCompleteKey, + isNormalizedEntityQueryKeyConditionPart, normalizeCompositeKey, } from "../../stores/entity-store.js"; import { deserializeCompositeKey, serializeCompositeKey } from "../../utils.js"; @@ -119,14 +130,20 @@ export class LocalEntityStore extends EntityStore { protected override async _query( entity: Entity | EntityIndex, - queryKey: NormalizedEntityCompositeKey, + queryKey: NormalizedEntityCompositeQueryKey, options?: EntityQueryOptions ): Promise { const partition = this.getPartitionMap(entity, queryKey.partition); const entries = partition ? [...partition.entries()] : []; + const sortKeyPart = queryKey.sort; + const sortFilteredEntries = sortKeyPart + ? entries.filter((e) => + filterEntryBySortKey(entity.key, sortKeyPart, e[1].value) + ) + : entries; const { items, nextToken } = paginateItems( - entries, + sortFilteredEntries, (a, b) => typeof a[0] === "string" ? a[0].localeCompare(b[0] as string) @@ -383,3 +400,51 @@ function deletePartitionEntry( } return false; } + +function filterEntryBySortKey( + keyDef: KeyDefinition, + querySortKey: NormalizedEntityQueryKeyPart, + entry: Attributes +) { + const entryKey = normalizeCompositeKey(keyDef, entry); + const entrySortKeyValue = entryKey.sort?.keyValue; + + // neither of these should happen, but discard any incomplete values. + // the item should contain the complete key, including a defined value + // unless the item is in a sparse index (items don't include the index key attributes), + // in which case it should not be placed in the index at all + if (!isCompleteKey(entryKey) || entrySortKeyValue === undefined) { + return false; + } else if (isNormalizedEntityQueryKeyConditionPart(querySortKey)) { + if (isBetweenQueryKeyCondition(querySortKey.condition)) { + return ( + entrySortKeyValue >= querySortKey.condition.$between[0] && + entrySortKeyValue <= querySortKey.condition.$between[1] + ); + } else if (isBeginsWithQueryKeyCondition(querySortKey.condition)) { + return typeof entrySortKeyValue === "string" + ? entrySortKeyValue.startsWith(querySortKey.condition.$beginsWith) + : false; + } else if (isLessThanQueryKeyCondition(querySortKey.condition)) { + return entrySortKeyValue < querySortKey.condition.$lt; + } else if (isLessThanEqualsQueryKeyCondition(querySortKey.condition)) { + return entrySortKeyValue <= querySortKey.condition.$lte; + } else if (isGreaterThanQueryKeyCondition(querySortKey.condition)) { + return entrySortKeyValue > querySortKey.condition.$gt; + } else if (isGreaterThanEqualsQueryKeyCondition(querySortKey.condition)) { + return entrySortKeyValue >= querySortKey.condition.$gte; + } + + assertNever(querySortKey.condition); + } else if (querySortKey.keyValue === undefined) { + return true; + } else if ( + querySortKey.partialValue && + typeof entrySortKeyValue === "string" && + typeof querySortKey.keyValue === "string" + ) { + return entrySortKeyValue.startsWith(querySortKey.keyValue); + } else { + return entrySortKeyValue === querySortKey.keyValue; + } +} diff --git a/packages/@eventual/core-runtime/src/stores/entity-store.ts b/packages/@eventual/core-runtime/src/stores/entity-store.ts index ff13111d8..6ebc065b7 100644 --- a/packages/@eventual/core-runtime/src/stores/entity-store.ts +++ b/packages/@eventual/core-runtime/src/stores/entity-store.ts @@ -1,5 +1,7 @@ -import type { +import { Attributes, + BetweenProgressiveKeyCondition, + BetweenQueryKeyCondition, CompositeKey, Entity, EntityConsistencyOptions, @@ -14,11 +16,21 @@ import type { KeyMap, KeyValue, QueryKey, + QueryKeyCondition, + QueryKeyMap, } from "@eventual/core"; -import type { +import { EntityHook, KeyDefinition, KeyDefinitionPart, + assertNever, + isBeginsWithQueryKeyCondition, + isBetweenQueryKeyCondition, + isGreaterThanEqualsQueryKeyCondition, + isGreaterThanQueryKeyCondition, + isLessThanEqualsQueryKeyCondition, + isLessThanQueryKeyCondition, + keyHasInlineBetween, } from "@eventual/core/internal"; import { EntityProvider } from "../providers/entity-provider.js"; @@ -95,11 +107,11 @@ export abstract class EntityStore implements EntityHook { public query( entityName: string, - queryKey: QueryKey, + queryKey: QueryKey, options?: EntityQueryOptions | undefined ): Promise { const entity = this.getEntity(entityName); - const normalizedKey = normalizeCompositeKey(entity, queryKey); + const normalizedKey = normalizeCompositeQueryKey(entity, queryKey); if (!isCompleteKeyPart(normalizedKey.partition)) { throw new Error("Entity partition key cannot be partial for query"); @@ -115,7 +127,7 @@ export abstract class EntityStore implements EntityHook { public queryIndex( entityName: string, indexName: string, - queryKey: QueryKey, + queryKey: QueryKey, options?: EntityQueryOptions | undefined ): Promise { const index = this.getEntity(entityName).indices.find( @@ -128,7 +140,7 @@ export abstract class EntityStore implements EntityHook { ); } - const normalizedKey = normalizeCompositeKey(index.key, queryKey); + const normalizedKey = normalizeCompositeQueryKey(index.key, queryKey); if (!isCompleteKeyPart(normalizedKey.partition)) { throw new Error( @@ -136,16 +148,12 @@ export abstract class EntityStore implements EntityHook { ); } - return this._query( - index, - normalizedKey as NormalizedEntityCompositeKey, - options - ); + return this._query(index, normalizedKey, options); } protected abstract _query( entity: Entity | EntityIndex, - queryKey: NormalizedEntityCompositeKey, + queryKey: NormalizedEntityCompositeQueryKey, options: EntityQueryOptions | undefined ): Promise; @@ -254,26 +262,63 @@ export type NormalizedEntityTransactItem = { } ); -export interface NormalizedEntityKeyPartBase extends KeyDefinitionPart { - parts: { field: string; value: KeyValue }[]; +export interface NormalizedEntityKeyPartBase extends KeyDefinitionPart { + parts: { field: string; value?: Value }[]; } export type NormalizedEntityKeyPart = | NormalizedEntityKeyPartialPart | NormalizedEntityKeyCompletePart; -export interface NormalizedEntityKeyCompletePart - extends NormalizedEntityKeyPartBase { - keyValue: string | number; +export type NormalizedEntityQueryKeyPart = + | NormalizedEntityKeyPartialPart + | NormalizedEntityKeyCompletePart + | NormalizedEntityQueryKeyConditionPart; + +export interface NormalizedEntityKeyCompletePart + extends NormalizedEntityKeyPartBase { + keyValue: KeyValue; partialValue: false; } -export interface NormalizedEntityKeyPartialPart - extends NormalizedEntityKeyPartBase { - keyValue?: string | number; +export interface NormalizedEntityKeyPartialPart + extends NormalizedEntityKeyPartBase { + keyValue?: KeyValue; partialValue: true; } +/** + * A query key that has been normalized. + */ +export interface NormalizedEntityQueryKeyConditionPart + extends NormalizedEntityKeyPartBase { + /** + * The condition given by the user, but updated with the given key prefix if present. + * + * ```ts + * { + * "sortA": "A", + * "sortB": { between: ["B", "C"] } + * } + * ``` + * + * Outputs as: + * + * ```ts + * { + * condition: { between: ["A#B", "A#C"] } + * } + * ``` + */ + condition: QueryKeyCondition; +} + +export function isNormalizedEntityQueryKeyConditionPart( + key: NormalizedEntityQueryKeyPart +): key is NormalizedEntityQueryKeyConditionPart { + return !!(key as NormalizedEntityQueryKeyConditionPart).condition; +} + export function isCompleteKeyPart( key: NormalizedEntityKeyPart ): key is NormalizedEntityKeyCompletePart { @@ -294,17 +339,30 @@ export function isCompleteKey( export interface NormalizedEntityCompositeKey< Partition extends NormalizedEntityKeyPart = NormalizedEntityKeyPart, - Sort extends NormalizedEntityKeyPart = NormalizedEntityKeyPart + Sort extends + | NormalizedEntityKeyPart + | NormalizedEntityQueryKeyPart = NormalizedEntityKeyPart > { partition: Partition; sort?: Sort; } +/** + * A composite key that is complete. Both the partition and sort keys have all attributes present. + */ export type NormalizedEntityCompositeKeyComplete = NormalizedEntityCompositeKey< NormalizedEntityKeyCompletePart, NormalizedEntityKeyCompletePart >; +/** + * A composite key that can be used for queries. The partition key must be complete, but the sort key can be partial or a condition. + */ +export type NormalizedEntityCompositeQueryKey = NormalizedEntityCompositeKey< + NormalizedEntityKeyCompletePart, + NormalizedEntityQueryKeyPart +>; + /** * Generate properties for an entity key given the key definition and key values. */ @@ -314,15 +372,39 @@ export function normalizeCompositeKey( ): NormalizedEntityCompositeKey { const keyDef = "partition" in entity ? entity : entity.key; - const partitionCompositeKey = formatNormalizedPart(keyDef.partition, (p, i) => - Array.isArray(key) ? key[i] : (key as KeyMap)[p] - ); + const partitionCompositeKey = formatNormalizePartitionKeyPart(keyDef, key); + + const sortCompositeKey = formatNormalizeSortKeyPart(keyDef, key); + + return sortCompositeKey + ? { + partition: partitionCompositeKey, + sort: sortCompositeKey, + } + : { + partition: partitionCompositeKey, + }; +} + +/** + * Generate properties for an entity query key given the key definition and key values or conditions. + */ +export function normalizeCompositeQueryKey( + entity: E | KeyDefinition, + key: QueryKey +): NormalizedEntityCompositeQueryKey { + const keyDef = "partition" in entity ? entity : entity.key; + + const partitionCompositeKey = formatNormalizePartitionKeyPart(keyDef, key); + + if (partitionCompositeKey.partialValue) { + throw new Error("Query key partition part cannot be partial"); + } const sortCompositeKey = keyDef.sort - ? formatNormalizedPart(keyDef.sort, (p, i) => - Array.isArray(key) - ? key[keyDef.partition.attributes.length + i] - : (key as KeyMap)[p] + ? formatNormalizedQueryPart( + keyDef as KeyDefinition & { sort: KeyDefinitionPart }, + key ) : undefined; @@ -338,7 +420,7 @@ export function normalizeCompositeKey( function formatNormalizedPart( keyPart: KeyDefinitionPart, - valueRetriever: (field: string, index: number) => string | number + valueRetriever: (field: string, index: number) => KeyValue | undefined ): NormalizedEntityKeyPart { const parts = keyPart.attributes.map((p, i) => ({ field: p, @@ -366,6 +448,169 @@ function formatNormalizedPart( }; } +function formatNormalizePartitionKeyPart( + keyDef: KeyDefinition, + key: Partial | QueryKey +) { + return formatNormalizedPart(keyDef.partition, (p, i) => { + const value = Array.isArray(key) ? key[i] : (key as KeyMap)[p]; + if (typeof value === "object") { + throw new Error(`Partition Key value must be a string or number: ${p}`); + } + return value; + }); +} + +function formatNormalizeSortKeyPart( + keyDef: KeyDefinition, + key: Partial | QueryKey +) { + if (!keyDef.sort) { + return undefined; + } + return formatNormalizedPart(keyDef.sort, (p, i) => + Array.isArray(key) + ? key[keyDef.partition.attributes.length + i] + : (key as KeyMap)[p] + ); +} + +function formatNormalizedQueryPart( + keyDef: KeyDefinition & { sort: KeyDefinitionPart }, + key: QueryKey +): NormalizedEntityQueryKeyPart { + const keyPart = keyDef.sort; + const parts = keyPart.attributes.map((p, i) => ({ + field: p, + value: Array.isArray(key) + ? key[keyDef.partition.attributes.length + i] + : (key as QueryKeyMap)[p], + })); + + const queryConditionIndex = parts.findIndex( + (p) => typeof p.value === "object" + ); + const missingValueIndex = parts.findIndex((p) => p.value === undefined); + + const isPartial = missingValueIndex > -1; + const hasInlineBetweenCondition = keyHasInlineBetween(key); + const hasFieldCondition = queryConditionIndex > -1; + + /** + * The key condition must be the last given value and there be at most one condition. + * + * [condition] - valid + * [value] - valid + * [value, missing] - valid + * [value, condition] - valid + * [value, condition, missing] - valid + * [value, condition, value | condition] - invalid + * [missing, value | condition] - invalid + */ + if ( + hasFieldCondition && + ((isPartial && queryConditionIndex > missingValueIndex) || + queryConditionIndex !== keyPart.attributes.length - 1) + ) { + throw new Error( + "Query Key condition must be the final value provided key attribute." + ); + } + + if (hasInlineBetweenCondition && hasFieldCondition) { + throw new Error( + "Query key cannot contain a key condition and an inline $between condition." + ); + } + + const hasCondition = hasFieldCondition || hasInlineBetweenCondition; + + const lastIndex = isPartial + ? hasFieldCondition + ? Math.min(missingValueIndex, queryConditionIndex) + : missingValueIndex + : hasFieldCondition + ? queryConditionIndex + : parts.length; + + const keyValuePrefix = + lastIndex === 0 // if there are no present values, return undefined + ? undefined + : keyPart.type === "number" + ? (parts[0]!.value as KeyValue) + : parts + .slice(0, lastIndex) + .map((p) => p.value) + .join("#"); + + if (hasCondition) { + const condition = hasInlineBetweenCondition + ? // turns the inline between into a normal between condition. We'll append the key value prefix next. + generateBetweenConditionFromInlineBetween( + key, + keyDef, + // there should not be both a condition key and the inline $between condition + isPartial ? missingValueIndex : 0 + ) + : (parts[queryConditionIndex]?.value as QueryKeyCondition); + + return { + type: keyPart.type, + attributes: keyPart.attributes, + parts, + // this can never be a number, either there is a condition on a > 0 index attribute or the number is the first + keyAttribute: keyPart.keyAttribute, + condition: isBetweenQueryKeyCondition(condition) + ? { + $between: [ + formatKeyPrefixConditionValue( + condition.$between[0], + keyValuePrefix + ), + formatKeyPrefixConditionValue( + condition.$between[1], + keyValuePrefix + ), + ], + } + : isBeginsWithQueryKeyCondition(condition) + ? { + $beginsWith: formatKeyPrefixConditionValue( + condition.$beginsWith, + keyValuePrefix + ) as string, + } + : isLessThanQueryKeyCondition(condition) + ? { + $lt: formatKeyPrefixConditionValue(condition.$lt, keyValuePrefix), + } + : isLessThanEqualsQueryKeyCondition(condition) + ? { + $lte: formatKeyPrefixConditionValue(condition.$lte, keyValuePrefix), + } + : isGreaterThanQueryKeyCondition(condition) + ? { + $gt: formatKeyPrefixConditionValue(condition.$gt, keyValuePrefix), + } + : isGreaterThanEqualsQueryKeyCondition(condition) + ? { + $gte: formatKeyPrefixConditionValue(condition.$gte, keyValuePrefix), + } + : assertNever(condition), + }; + } + + return { + type: keyPart.type, + attributes: keyPart.attributes, + parts, + keyAttribute: keyPart.keyAttribute, + keyValue: keyValuePrefix, + // since true is more permissive (allows undefined), hack the types here to allow any value + partialValue: (missingValueIndex !== -1) as true, + }; +} + export function convertNormalizedEntityKeyToMap( key: NormalizedEntityCompositeKey ): KeyMap { @@ -454,3 +699,39 @@ export function removeKeyAttributes( ); } } + +function formatKeyPrefixConditionValue( + conditionValue: KeyValue, + prefixValue?: KeyValue +) { + return prefixValue !== undefined + ? `${prefixValue.toString()}#${conditionValue.toString()}` + : conditionValue; +} + +function generateBetweenConditionFromInlineBetween( + condition: BetweenProgressiveKeyCondition, + keyDefinition: KeyDefinition & { sort: KeyDefinitionPart }, + sortKeyOffset: number +): BetweenQueryKeyCondition { + const betweenKeyPart: KeyDefinitionPart = { + attributes: keyDefinition.sort.attributes.slice(sortKeyOffset), + keyAttribute: keyDefinition.sort.keyAttribute, + type: keyDefinition.sort.type, + }; + const left = formatNormalizedPart(betweenKeyPart, (p) => { + return (condition.$between[0] as KeyMap)[p]; + }); + const right = formatNormalizedPart(betweenKeyPart, (p) => { + return (condition.$between[1] as KeyMap)[p]; + }); + if ( + keyDefinition.sort.type === "number" && + (left.keyValue === undefined || right.keyValue === undefined) + ) { + throw new Error( + "Between conditions cannot be empty when the field type is number." + ); + } + return { $between: [left.keyValue ?? "", right.keyValue ?? ""] }; +} diff --git a/packages/@eventual/core-runtime/test/entity-store.test.ts b/packages/@eventual/core-runtime/test/entity-store.test.ts new file mode 100644 index 000000000..ef39b5e11 --- /dev/null +++ b/packages/@eventual/core-runtime/test/entity-store.test.ts @@ -0,0 +1,200 @@ +import { + NormalizedEntityCompositeQueryKey, + normalizeCompositeQueryKey, +} from "../src/stores/entity-store.js"; + +describe("normalizeCompositeQueryKey", () => { + test("should normalize composite query key", () => { + expect( + normalizeCompositeQueryKey( + { + partition: { + attributes: ["part"], + keyAttribute: "part", + type: "string", + }, + sort: { attributes: ["sort"], keyAttribute: "sort", type: "string" }, + }, + { + part: "a", + sort: "b", + } + ) + ).toEqual({ + partition: { + attributes: ["part"], + keyAttribute: "part", + keyValue: "a", + partialValue: false, + parts: [{ field: "part", value: "a" }], + type: "string", + }, + sort: { + attributes: ["sort"], + keyAttribute: "sort", + keyValue: "b", + partialValue: false, + parts: [{ field: "sort", value: "b" }], + type: "string", + }, + }); + }); + + test("should normalize composite query key without a sort key", () => { + expect( + normalizeCompositeQueryKey( + { + partition: { + attributes: ["part"], + keyAttribute: "part", + type: "string", + }, + }, + { + part: "a", + sort: "b", + } + ) + ).toEqual({ + partition: { + attributes: ["part"], + keyAttribute: "part", + keyValue: "a", + partialValue: false, + parts: [{ field: "part", value: "a" }], + type: "string", + }, + }); + }); + + test("should normalize composite query key with a multi-attribute partition key", () => { + expect( + normalizeCompositeQueryKey( + { + partition: { + attributes: ["part", "part2"], + keyAttribute: "part|part2", + type: "string", + }, + }, + { + part: "a", + part2: 1, + sort: "b", + } + ) + ).toEqual({ + partition: { + attributes: ["part", "part2"], + keyAttribute: "part|part2", + keyValue: "a#1", + partialValue: false, + parts: [ + { field: "part", value: "a" }, + { field: "part2", value: 1 }, + ], + type: "string", + }, + }); + }); + + test("should fail to normalize composite query key with a partial partition key", () => { + expect(() => + normalizeCompositeQueryKey( + { + partition: { + attributes: ["part", "part2"], + keyAttribute: "part|part2", + type: "string", + }, + }, + { + part: "a", + sort: "b", + } + ) + ).toThrow(new Error("Query key partition part cannot be partial")); + }); + + test("should normalize composite key with sort conditions", () => { + expect( + normalizeCompositeQueryKey( + { + partition: { + attributes: ["part"], + keyAttribute: "part", + type: "string", + }, + sort: { + attributes: ["sort"], + keyAttribute: "sort", + type: "string", + }, + }, + { + part: "a", + sort: { $beginsWith: "b" } as any, + } + ) + ).toEqual({ + partition: { + attributes: ["part"], + keyAttribute: "part", + keyValue: "a", + partialValue: false, + parts: [{ field: "part", value: "a" }], + type: "string", + }, + sort: { + attributes: ["sort"], + keyAttribute: "sort", + condition: { $beginsWith: "b" }, + parts: [{ field: "sort", value: { $beginsWith: "b" } }], + type: "string", + }, + }); + }); + + test("should normalize composite key with multi-attribute sort conditions", () => { + expect( + normalizeCompositeQueryKey( + { + partition: { + attributes: ["part"], + keyAttribute: "part", + type: "string", + }, + sort: { + attributes: ["sort", "sort2"], + keyAttribute: "sort|sort2", + type: "string", + }, + }, + { + part: "a", + sort: "b", + sort2: { $beginsWith: "c" } as any, + } + ) + ).toEqual({ + partition: { + attributes: ["part"], + keyAttribute: "part", + keyValue: "a", + partialValue: false, + parts: [{ field: "part", value: "a" }], + type: "string", + }, + sort: { + attributes: ["sort", "sort2"], + keyAttribute: "sort|sort2", + condition: { $beginsWith: "b#c" }, + parts: [ + { field: "sort", value: "b" }, + { field: "sort2", value: { $beginsWith: "c" } }, + ], + type: "string", + }, + }); + }); +}); diff --git a/packages/@eventual/core/src/entity/key.ts b/packages/@eventual/core/src/entity/key.ts index bc2c247be..58622c967 100644 --- a/packages/@eventual/core/src/entity/key.ts +++ b/packages/@eventual/core/src/entity/key.ts @@ -1,4 +1,7 @@ +/* eslint-disable @typescript-eslint/ban-types */ + import type { Attributes } from "./entity.js"; +import type t from "type-fest"; export type KeyValue = string | number | bigint; @@ -49,7 +52,7 @@ export type IndexCompositeKeyPart = */ export type CompositeKeyPart = readonly [ KeyAttribute, - ...KeyAttribute[] + ...(readonly KeyAttribute[]) ]; /** @@ -76,8 +79,7 @@ export type KeyMap< ? { [k in Sort[number]]: Exclude; } - : // eslint-disable-next-line - {}); + : {}); export type KeyPartialTuple< Attr extends Attributes, @@ -122,10 +124,91 @@ export type CompositeKey< | undefined > = KeyMap | KeyTuple; +/** + * Matches if the key attribute is between the start and end value, inclusive. + * + * start <= value <= end + * + * Note: numeric multi-attribute key parts are treated as strings. + */ +export type QueryKeyCondition = + | BetweenQueryKeyCondition + | LessThanQueryKeyCondition + | LessThanEqualsQueryKeyCondition + | GreaterThanQueryKeyCondition + | GreaterThanEqualsQueryKeyCondition + | BeginsWithQueryKeyCondition; + +/** + * Matches if the key attribute is between the start and end value, inclusive. + * + * start <= value <= end + * + * Note: numeric multi-attribute key parts are treated as strings. + */ +export interface BetweenQueryKeyCondition { + $between: [t.LiteralToPrimitive, t.LiteralToPrimitive]; +} + +/** + * Matches if the key attribute starts with the given value. + * + * Can only be used with string fields. + * + * Note: numeric multi-attribute key parts are treated as strings. + */ +export interface BeginsWithQueryKeyCondition< + Value extends KeyValue = KeyValue +> { + $beginsWith: Extract, string>; +} + +/** + * Matches if the key attribute is less than the given value. + * + * Note: numeric multi-attribute key parts are treated as strings. + */ +export interface LessThanQueryKeyCondition { + $lt: t.LiteralToPrimitive; +} + +/** + * Matches if the key attribute is less than or equal to the given value. + * + * Note: numeric multi-attribute key parts are treated as strings. + */ +export interface LessThanEqualsQueryKeyCondition< + Value extends KeyValue = KeyValue +> { + $lte: t.LiteralToPrimitive; +} + +/** + * Matches if the key attribute is greater than the given value. + * + * Note: numeric multi-attribute key parts are treated as strings. + */ +export interface GreaterThanQueryKeyCondition< + Value extends KeyValue = KeyValue +> { + $gt: t.LiteralToPrimitive; +} + +/** + * Matches if the key attribute is greater than or equal to the given value. + * + * Note: numeric multi-attribute key parts are treated as strings. + */ +export interface GreaterThanEqualsQueryKeyCondition< + Value extends KeyValue = KeyValue +> { + $gte: t.LiteralToPrimitive; +} + export type ProgressiveTupleQueryKey< Attr extends Attributes, Sort extends readonly (keyof Attr)[], - Accum extends [] = [] + Accum extends KeyValue[] = [] > = Sort extends readonly [] ? Accum : Sort extends readonly [ @@ -134,52 +217,105 @@ export type ProgressiveTupleQueryKey< ] ? | Accum - | ProgressiveQueryKey]> + | [...Accum, QueryKeyCondition>] + | ProgressiveTupleQueryKey< + Attr, + ks, + [...Accum, Extract] + > : never; export type ProgressiveQueryKey< Attr extends Attributes, - Sort extends readonly (keyof Attr)[], - Accum extends object = object -> = Sort extends readonly [] - ? Accum - : Sort extends readonly [ - infer k extends keyof Attr, - ...infer ks extends readonly (keyof Attr)[] - ] + Sort extends readonly (keyof Attr)[] +> = Sort extends readonly [ + infer k extends keyof Attr, + ...infer ks extends readonly (keyof Attr)[] +] ? - | Accum - | ProgressiveQueryKey< - Attr, - ks, - Accum & { - [sk in k]: Extract; - } - > - : never; + | { [sk in Sort[number]]?: never } + | { + [sk in k]: QueryKeyCondition>; + } + | BetweenProgressiveKeyCondition + | ({ + [sk in k]: Extract; + } & ProgressiveQueryKey) + : {}; /** - * A partial key that can be used to query an entity. + * Supports betweens condition using multiple sort attribute parts. * - * ```ts - * entity.query({ part1: "val", part2: "val2", sort1: "val" }); - * ``` + * At least one attribute must be present in the left and right side. * - * TODO: support expressions like between and starts with on sort properties + * BETWEEN "a" and "c#b" + * { + * $between: [{sort1: "a"}, {sort1: "c", sort2: "b"}] + * } + * + * BETWEEN "a" and "c" + * { + * sort1: { $between: ["a", "c"] } + * } */ -export type QueryKey< +export type BetweenProgressiveKeyCondition< + Attr extends Attributes, + Sort extends readonly (keyof Attr)[] +> = { + $between: Sort extends readonly [ + infer k extends keyof Attr, + ...infer ks extends readonly (keyof Attr)[] + ] + ? [ + ProgressiveKey & { + [sk in k]: Extract; + }, + ProgressiveKey & { + [sk in k]: Extract; + } + ] + : never; +}; + +export type ProgressiveKey< + Attr extends Attributes, + Sort extends readonly (keyof Attr)[] +> = Sort extends readonly [ + infer k extends keyof Attr, + ...infer ks extends readonly (keyof Attr)[] +] + ? + | { [sk in Sort[number]]?: never } + | ({ + [sk in k]: Extract; + } & ProgressiveKey) + : {}; + +export type QueryKeyMap< Attr extends Attributes = Attributes, Partition extends CompositeKeyPart = CompositeKeyPart, Sort extends CompositeKeyPart | undefined = | CompositeKeyPart | undefined +> = { + [pk in Partition[number]]: Extract; +} & (Sort extends undefined + ? {} + : ProgressiveQueryKey]>); + +/** + * A partial key that can be used to query an entity. + * + * ```ts + * entity.query({ part1: "val", part2: "val2", sort1: "val" }); + * ``` + */ +export type QueryKey< + Attr extends Attributes, + Partition extends CompositeKeyPart, + Sort extends CompositeKeyPart | undefined > = - | ({ - [pk in Partition[number]]: Extract; - } & (Sort extends undefined - ? // eslint-disable-next-line - {} - : ProgressiveQueryKey>)) + | QueryKeyMap | [ ...KeyTuple, ...(Sort extends undefined @@ -196,11 +332,11 @@ export type StreamQueryKey< Sort extends CompositeKeyPart | undefined = | CompositeKeyPart | undefined -> = ProgressiveQueryKey & +> = ProgressiveKey & (Sort extends undefined ? // eslint-disable-next-line {} - : ProgressiveQueryKey>); + : ProgressiveKey>); export type KeyAttributes< Attr extends Attributes = any, diff --git a/packages/@eventual/core/src/internal/entity-hook.ts b/packages/@eventual/core/src/internal/entity-hook.ts index 35f9e5bd0..a515eca96 100644 --- a/packages/@eventual/core/src/internal/entity-hook.ts +++ b/packages/@eventual/core/src/internal/entity-hook.ts @@ -32,7 +32,7 @@ export type EntityHook = { queryIndex( entityName: string, indexName: string, - queryKey: QueryKey, + queryKey: QueryKey, options?: EntityQueryOptions ): Promise; scanIndex( diff --git a/packages/@eventual/core/src/internal/entity.ts b/packages/@eventual/core/src/internal/entity.ts index bcfd9c3e8..5ee2e3d73 100644 --- a/packages/@eventual/core/src/internal/entity.ts +++ b/packages/@eventual/core/src/internal/entity.ts @@ -1,5 +1,16 @@ import { z } from "zod"; -import type { CompositeKeyPart } from "../entity/key.js"; +import type { + BeginsWithQueryKeyCondition, + BetweenProgressiveKeyCondition, + BetweenQueryKeyCondition, + CompositeKeyPart, + GreaterThanEqualsQueryKeyCondition, + GreaterThanQueryKeyCondition, + LessThanEqualsQueryKeyCondition, + LessThanQueryKeyCondition, + QueryKey, + QueryKeyCondition, +} from "../entity/key.js"; export interface KeyDefinitionPart { type: "number" | "string"; @@ -51,3 +62,45 @@ export function computeKeyDefinition( }; } } + +export function keyHasInlineBetween>( + key: Q +): key is Q & BetweenProgressiveKeyCondition { + return "$between" in key; +} + +export function isBetweenQueryKeyCondition( + condition: QueryKeyCondition +): condition is BetweenQueryKeyCondition { + return "$between" in condition; +} + +export function isBeginsWithQueryKeyCondition( + condition: QueryKeyCondition +): condition is BeginsWithQueryKeyCondition { + return "$beginsWith" in condition; +} + +export function isLessThanQueryKeyCondition( + condition: QueryKeyCondition +): condition is LessThanQueryKeyCondition { + return "$lt" in condition; +} + +export function isLessThanEqualsQueryKeyCondition( + condition: QueryKeyCondition +): condition is LessThanEqualsQueryKeyCondition { + return "$lte" in condition; +} + +export function isGreaterThanQueryKeyCondition( + condition: QueryKeyCondition +): condition is GreaterThanQueryKeyCondition { + return "$gt" in condition; +} + +export function isGreaterThanEqualsQueryKeyCondition( + condition: QueryKeyCondition +): condition is GreaterThanEqualsQueryKeyCondition { + return "$gte" in condition; +}