From 44ba6f8f84ef3e33aa1c07b0808a42bcf871b8c6 Mon Sep 17 00:00:00 2001 From: danieljbruce Date: Fri, 1 Sep 2023 16:07:10 -0400 Subject: [PATCH] feat: Sum and average aggregation queries (#1097) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial sum aggregation This commit introduces the sum aggregation with a simple test to ensure it works * Modify encoding This change modifies the encoding so that the right data reaches the grpc layer * PropertyAggregateField with tests Adds helper functions and a super class so that average and sum can share the same properties. * Improve transaction tests Add sum and average to the transaction tests here to improve test coverage for transactions. * Change the description in the describe block * Change return type to average The return type for this function is wrong. The function returns an Average so we should use Average. * Make alias optional Make alias optional since the query still works without providing an alias. * Fix the transaction tests to fail on rollback The transaction tests should not pass if the transaction is rolled back like they do currently. We must not catch errors and instead let the test fail. * Add additional assertions to existing tests Ensure that the addAggregation function works correctly with an additional assertion check. * Revert "Add additional assertions to existing tests" This reverts commit 970c0f412d3c31653c389e9d765817cbb27acb56. * Add describe block for comparing equivalent query This test ensures that all the aggregate queries are actually the same no matter how you build them * Average, sum and count toProto tests Write tests to effectively document the toProto output of the various aggregate fields * Add tests for the sum aggregation Equivalent tests to count are written for sum in system tests and some more tests are written too to meet requirements outlined by team. * Add a test for sum and snapshot reads The test for sum and snapshot reads should look at the database before the data is created and run the tests based on the database in that state * Add two test blocks for special cases Add a test block for a dataset with overflow values and a dataset with NaN values. * Export aggregate field from the client Aggregate field should be exported from the client so that it can be used easily by users. * PR follow-up changes Some idiomatic changes to improve the state of the code in the PR. * Adjust the values so that tests pass Values for sum and average should be different from those of count and these tests provide the right values now. # Conflicts: # system-test/datastore.ts * Add average aggregations Average aggregations regarding appearances have been added in tests and correct values have been assigned * Add snapshot reads for run query and aggregate q The future refactor must implement the TODOs so that there is less repeated code in the codebase. Also, this commit implements snapshot reads for queries and adds a test for the snapshot reads. * Remove Google error and entity filter Remove some unused imports as they do not apply to the code anymore * Should use null for an aggregation query read time Snapshot reads read at a time that there is no data so sums and averages should reflect that accordingly. * Remove tests from a bad cherry pick Tests for sum that have values corresponding to count are still there in the test cases. They should be removed. * Linting fix We don’t care about a loss of precision since the literal value indicated is contained in a test and the loss of precision won’t affect the code. * Do the test on rating instead of appearances At this point the datastore is populated with data about ratings so computations should be done on that instead. * The assertion says the request should have failed An assertion error should be thrown so that the test doesn’t pass if the request is successful. * Add a comment about using limits in test The query with the limit will include all data points with the lowest appearance values. This is likely desired, but also important to document. * Add rollbacks to transaction tests The rollbacks for the transaction tests ensure that if a test fails then the data gets reset to where it was before. * refactor getSharedOptionsOnly Introducing getSharedOptionsOnly allows us to use that function in two different places to avoid a repeated block of code. * Remove test related to snapshot reads This test belongs inside another PR because it is not directly related to sum/avg. * Add a test for multiple types of aggregates A test should be included that looks at multiple aggregations in a single query. * Correct descriptions of two tests on overflow The tests themselves should include the word overflow so that it is clear they are working with an overflow dataset. * Add a comment for setting the alias The comment for setting the alias should not make any mention of count since it is agnostic to the aggregation type. * Add tests to compare various ways to encode alias No matter how alias is encoded, the data structures should store the aggregations the same way inside an aggregate field so as not to create any confusion. * Added tests for when an empty alias is provided Tests for when an empty alias is provided should check that each aggregate query still works. * Add a comment clarifying the use of snapshot reads The sleep function should enable us to test snapshot reads for aggregate queries * Add two tests to explore mixed aggregations alias Two tests should be explored that evaluate what happens when multiple aggregations are used and when too many aggregations are used. * Better names for some internal private functions Shared options functions could be given a better name so that they make more sense to the code that is using them. * Add a comment explaining why the sleep is needed The code must explain why the sleep function is needed in the test because its purpose should be clear. * Add getReadTime function and use for sum/avg A sum/average test uses snapshot reads for an aggregate query. The key here is write code that will guarantee the read time occurs well before all the data is saved in order to ensure the test isn’t flakey. * Rename variable to emptyData emptyData is a more accurate variable name for the datastore save data --- src/aggregate.ts | 104 +++++++- src/index.ts | 4 +- src/request.ts | 53 ++-- system-test/datastore.ts | 523 +++++++++++++++++++++++++++++++++++++-- test/query.ts | 158 ++++++++++-- 5 files changed, 772 insertions(+), 70 deletions(-) diff --git a/src/aggregate.ts b/src/aggregate.ts index fbef6e3ab..1eececfe5 100644 --- a/src/aggregate.ts +++ b/src/aggregate.ts @@ -46,11 +46,35 @@ class AggregateQuery { * @param {string} alias * @returns {AggregateQuery} */ - count(alias: string): AggregateQuery { + count(alias?: string): AggregateQuery { this.aggregations.push(AggregateField.count().alias(alias)); return this; } + /** + * Add a `sum` aggregate query to the list of aggregations. + * + * @param {string} property + * @param {string} alias + * @returns {AggregateQuery} + */ + sum(property: string, alias?: string): AggregateQuery { + this.aggregations.push(AggregateField.sum(property).alias(alias)); + return this; + } + + /** + * Add a `average` aggregate query to the list of aggregations. + * + * @param {string} property + * @param {string} alias + * @returns {AggregateQuery} + */ + average(property: string, alias?: string): AggregateQuery { + this.aggregations.push(AggregateField.average(property).alias(alias)); + return this; + } + /** * Add a custom aggregation to the list of aggregations. * @@ -99,7 +123,6 @@ class AggregateQuery { * Get the proto for the list of aggregations. * */ - // eslint-disable-next-line toProto(): any { return this.aggregations.map(aggregation => aggregation.toProto()); } @@ -122,14 +145,34 @@ abstract class AggregateField { } /** - * Gets a copy of the Count aggregate field. + * Gets a copy of the Sum aggregate field. + * + * @returns {Sum} + */ + static sum(property: string): Sum { + return new Sum(property); + } + + /** + * Gets a copy of the Average aggregate field. + * + * @returns {Average} + */ + static average(property: string): Average { + return new Average(property); + } + + /** + * Sets the alias on the aggregate field that should be used. * * @param {string} alias The label used in the results to describe this * aggregate field when a query is run. * @returns {AggregateField} */ - alias(alias: string): AggregateField { - this.alias_ = alias; + alias(alias?: string): AggregateField { + if (alias) { + this.alias_ = alias; + } return this; } @@ -137,7 +180,6 @@ abstract class AggregateField { * Gets the proto for the aggregate field. * */ - // eslint-disable-next-line abstract toProto(): any; } @@ -146,7 +188,6 @@ abstract class AggregateField { * */ class Count extends AggregateField { - // eslint-disable-next-line /** * Gets the proto for the count aggregate field. * @@ -157,4 +198,53 @@ class Count extends AggregateField { } } +/** + * A PropertyAggregateField is a class that contains data that defines any + * aggregation that is performed on a property. + * + */ +abstract class PropertyAggregateField extends AggregateField { + abstract operator: string; + + /** + * Build a PropertyAggregateField object. + * + * @param {string} property + */ + constructor(public property_: string) { + super(); + } + + /** + * Gets the proto for the property aggregate field. + * + */ + toProto(): any { + const aggregation = this.property_ + ? {property: {name: this.property_}} + : {}; + return Object.assign( + {operator: this.operator}, + this.alias_ ? {alias: this.alias_} : null, + {[this.operator]: aggregation} + ); + } +} + +/** + * A Sum is a class that contains data that defines a Sum aggregation. + * + */ +class Sum extends PropertyAggregateField { + operator = 'sum'; +} + +/** + * An Average is a class that contains data that defines an Average aggregation. + * + */ +class Average extends PropertyAggregateField { + operator = 'avg'; +} + export {AggregateField, AggregateQuery, AGGREGATE_QUERY}; diff --git a/src/index.ts b/src/index.ts index 16de85651..9daf984c8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -40,8 +40,9 @@ import * as is from 'is'; import {Transform, pipeline} from 'stream'; import {entity, Entities, Entity, EntityProto, ValueProto} from './entity'; +import {AggregateField} from './aggregate'; import Key = entity.Key; -export {Entity, Key}; +export {Entity, Key, AggregateField}; import {PropertyFilter, and, or} from './filter'; export {PropertyFilter, and, or}; import { @@ -1818,7 +1819,6 @@ promisifyAll(Datastore, { 'isDouble', 'geoPoint', 'getProjectId', - 'getSharedQueryOptions', 'isGeoPoint', 'index', 'int', diff --git a/src/request.ts b/src/request.ts index cb6273ae7..c832d7d28 100644 --- a/src/request.ts +++ b/src/request.ts @@ -276,28 +276,8 @@ class DatastoreRequest { } const makeRequest = (keys: entity.Key[] | KeyProto[]) => { - const reqOpts: RequestOptions = { - keys, - }; - - if (options.consistency) { - const code = CONSISTENCY_PROTO_CODE[options.consistency.toLowerCase()]; - - reqOpts.readOptions = { - readConsistency: code, - }; - } - if (options.readTime) { - if (reqOpts.readOptions === undefined) { - reqOpts.readOptions = {}; - } - const readTime = options.readTime; - const seconds = readTime / 1000; - reqOpts.readOptions.readTime = { - seconds: Math.floor(seconds), - }; - } - + const reqOpts = this.getRequestOptions(options); + Object.assign(reqOpts, {keys}); this.request_( { client: 'DatastoreClient', @@ -596,7 +576,7 @@ class DatastoreRequest { setImmediate(callback, e as Error); return; } - const sharedQueryOpts = this.getSharedQueryOptions(query.query, options); + const sharedQueryOpts = this.getQueryOptions(query.query, options); const aggregationQueryOptions: AggregationQueryOptions = { nestedQuery: queryProto, aggregations: query.toProto(), @@ -811,7 +791,7 @@ class DatastoreRequest { setImmediate(onResultSet, e as Error); return; } - const sharedQueryOpts = this.getSharedQueryOptions(query, options); + const sharedQueryOpts = this.getQueryOptions(query, options); const reqOpts: RequestOptions = sharedQueryOpts; reqOpts.query = queryProto; @@ -887,9 +867,8 @@ class DatastoreRequest { return stream; } - private getSharedQueryOptions( - query: Query, - options: RunQueryStreamOptions = {} + private getRequestOptions( + options: RunQueryStreamOptions ): SharedQueryOptions { const sharedQueryOpts = {} as SharedQueryOptions; if (options.consistency) { @@ -898,6 +877,24 @@ class DatastoreRequest { readConsistency: code, }; } + if (options.readTime) { + if (sharedQueryOpts.readOptions === undefined) { + sharedQueryOpts.readOptions = {}; + } + const readTime = options.readTime; + const seconds = readTime / 1000; + sharedQueryOpts.readOptions.readTime = { + seconds: Math.floor(seconds), + }; + } + return sharedQueryOpts; + } + + private getQueryOptions( + query: Query, + options: RunQueryStreamOptions = {} + ): SharedQueryOptions { + const sharedQueryOpts = this.getRequestOptions(options); if (query.namespace) { sharedQueryOpts.partitionId = { namespaceId: query.namespace, @@ -1191,7 +1188,7 @@ export type DeleteResponse = CommitResponse; * that a callback is omitted. */ promisifyAll(DatastoreRequest, { - exclude: ['getSharedQueryOptions'], + exclude: ['getQueryOptions', 'getRequestOptions'], }); /** diff --git a/system-test/datastore.ts b/system-test/datastore.ts index 1fceb0199..ad62dd4ac 100644 --- a/system-test/datastore.ts +++ b/system-test/datastore.ts @@ -21,11 +21,12 @@ import {Datastore, Index} from '../src'; import {google} from '../protos/protos'; import {Storage} from '@google-cloud/storage'; import {AggregateField} from '../src/aggregate'; -import {PropertyFilter, EntityFilter, and, or} from '../src/filter'; +import {PropertyFilter, and, or} from '../src/filter'; import {entity} from '../src/entity'; import KEY_SYMBOL = entity.KEY_SYMBOL; describe('Datastore', () => { + let timeBeforeDataCreation: number; const testKinds: string[] = []; const datastore = new Datastore({ namespace: `${Date.now()}`, @@ -47,6 +48,26 @@ describe('Datastore', () => { // TODO/DX ensure indexes before testing, and maybe? cleanup indexes after // possible implications with kokoro project + // Gets the read time of the latest save so that the test isn't flakey due to race condition. + async function getReadTime(path: [{kind: string; name: string}]) { + const projectId = await datastore.getProjectId(); + const request = { + keys: [ + { + path, + partitionId: {namespaceId: datastore.namespace}, + }, + ], + projectId, + }; + const dataClient = datastore.clients_.get('DatastoreClient'); + let results: any; + if (dataClient) { + results = await dataClient['lookup'](request); + } + return parseInt(results[0].readTime.seconds) * 1000; + } + after(async () => { async function deleteEntities(kind: string) { const query = datastore.createQuery(kind).select('__key__'); @@ -676,12 +697,28 @@ describe('Datastore', () => { ]; before(async () => { + // This 'sleep' function is used to ensure that when data is saved to datastore, + // the time on the server is far enough ahead to be sure to be later than timeBeforeDataCreation + // so that when we read at timeBeforeDataCreation we get a snapshot of data before the save. + function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); + } const keysToSave = keys.map((key, index) => { return { key, data: characters[index], }; }); + // Save for a key so that a read time can be accessed for snapshot reads. + const emptyData = Object.assign(Object.assign({}, keysToSave[0]), { + data: {}, + }); + await datastore.save(emptyData); + timeBeforeDataCreation = await getReadTime([ + {kind: 'Character', name: 'Rickard'}, + ]); + // Sleep for 3 seconds so that any future reads will be later than timeBeforeDataCreation. + await sleep(3000); await datastore.save(keysToSave); }); @@ -950,6 +987,251 @@ describe('Datastore', () => { }); }); }); + describe('with a sum filter', () => { + it('should run a sum aggregation', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.sum('appearances')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 187}]); + }); + it('should run a sum aggregation with a list of aggregates', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.sum('appearances'), + AggregateField.sum('appearances'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 187, property_2: 187}]); + }); + it('should run a sum aggregation having other filters', async () => { + const q = datastore + .createQuery('Character') + .filter('family', 'Stark') + .filter('appearances', '>=', 20); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.sum('appearances').alias('sum1')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{sum1: 169}]); + }); + it('should run a sum aggregate filter with an alias', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.sum('appearances').alias('sum1')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{sum1: 187}]); + }); + it('should do multiple sum aggregations with aliases', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.sum('appearances').alias('sum1'), + AggregateField.sum('appearances').alias('sum2'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{sum1: 187, sum2: 187}]); + }); + it('should run a sum aggregation filter with a limit', async () => { + // When using a limit the test appears to use data points with the lowest appearance values. + const q = datastore.createQuery('Character').limit(5); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.sum('appearances')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 91}]); + }); + it('should run a sum aggregate filter with a limit and an alias', async () => { + const q = datastore.createQuery('Character').limit(7); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.sum('appearances').alias('sum1')]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{sum1: 154}]); + }); + it('should run a sum aggregate filter against a non-numeric property value', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.sum('family').alias('sum1')]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{sum1: 0}]); + }); + it('should run a sum aggregate filter against __key__ property value', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.sum('__key__').alias('sum1')]); + try { + await datastore.runAggregationQuery(aggregate); + assert.fail('The request should have failed.'); + } catch (err: any) { + assert.strictEqual( + err.message, + '3 INVALID_ARGUMENT: Aggregations are not supported for the property: __key__' + ); + } + }); + it('should run a sum aggregate filter against a query that returns no results', async () => { + const q = datastore + .createQuery('Character') + .filter('family', 'NoMatch'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.sum('appearances').alias('sum1')]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{sum1: 0}]); + }); + it('should run a sum aggregate filter against a query from before the data creation', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.sum('appearances').alias('sum1')]); + const [results] = await datastore.runAggregationQuery(aggregate, { + readTime: timeBeforeDataCreation, + }); + assert.deepStrictEqual(results, [{sum1: 0}]); + }); + it('should run a sum aggregate filter using the alias function, but with no alias', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.sum('appearances').alias()]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 187}]); + }); + }); + describe('with an average filter', () => { + it('should run an average aggregation', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.average('appearances')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 23.375}]); + }); + it('should run an average aggregation with a list of aggregates', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.average('appearances'), + AggregateField.average('appearances'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [ + {property_1: 23.375, property_2: 23.375}, + ]); + }); + it('should run an average aggregation having other filters', async () => { + const q = datastore + .createQuery('Character') + .filter('family', 'Stark') + .filter('appearances', '>=', 20); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.average('appearances').alias('avg1')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{avg1: 28.166666666666668}]); + }); + it('should run an average aggregate filter with an alias', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.average('appearances').alias('avg1')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{avg1: 23.375}]); + }); + it('should do multiple average aggregations with aliases', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.average('appearances').alias('avg1'), + AggregateField.average('appearances').alias('avg2'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{avg1: 23.375, avg2: 23.375}]); + }); + it('should run an average aggregation filter with a limit', async () => { + const q = datastore.createQuery('Character').limit(5); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.average('appearances')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 18.2}]); + }); + it('should run an average aggregate filter with a limit and an alias', async () => { + const q = datastore.createQuery('Character').limit(7); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.average('appearances').alias('avg1'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{avg1: 22}]); + }); + it('should run an average aggregate filter against a non-numeric property value', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.average('family').alias('avg1')]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{avg1: null}]); + }); + it('should run an average aggregate filter against __key__ property value', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.average('__key__').alias('avg1')]); + try { + await datastore.runAggregationQuery(aggregate); + assert.fail('The request should have failed.'); + } catch (err: any) { + assert.strictEqual( + err.message, + '3 INVALID_ARGUMENT: Aggregations are not supported for the property: __key__' + ); + } + }); + it('should run an average aggregate filter against a query that returns no results', async () => { + const q = datastore + .createQuery('Character') + .filter('family', 'NoMatch'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.average('appearances').alias('avg1'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{avg1: null}]); + }); + it('should run an average aggregate filter against a query from before the data creation', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.average('appearances').alias('avg1'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate, { + readTime: timeBeforeDataCreation, + }); + assert.deepStrictEqual(results, [{avg1: null}]); + }); + it('should run an average aggregate filter using the alias function, but with no alias', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.average('appearances').alias()]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 23.375}]); + }); + }); describe('with a count filter', () => { it('should run a count aggregation', async () => { const q = datastore.createQuery('Character'); @@ -1013,6 +1295,73 @@ describe('Datastore', () => { const [results] = await datastore.runAggregationQuery(aggregate); assert.deepStrictEqual(results, [{total: 7}]); }); + it('should run a count aggregate filter using the alias function, but with no alias', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([AggregateField.count().alias()]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 8}]); + }); + }); + describe('with multiple types of filters', () => { + it('should run multiple types of aggregations with a list of aggregates', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.count(), + AggregateField.sum('appearances'), + AggregateField.average('appearances'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [ + {property_1: 8, property_2: 187, property_3: 23.375}, + ]); + }); + it('should run multiple types of aggregations with and without aliases', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.count(), + AggregateField.average('appearances'), + AggregateField.count().alias('alias_count'), + AggregateField.sum('appearances').alias('alias_sum'), + AggregateField.average('appearances').alias('alias_average'), + ]); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [ + { + property_1: 8, + property_2: 23.375, + alias_count: 8, + alias_sum: 187, + alias_average: 23.375, + }, + ]); + }); + it('should throw an error when too many aggregations are run', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregations([ + AggregateField.count(), + AggregateField.sum('appearances'), + AggregateField.average('appearances'), + AggregateField.count().alias('alias_count'), + AggregateField.sum('appearances').alias('alias_sum'), + AggregateField.average('appearances').alias('alias_average'), + ]); + try { + await datastore.runAggregationQuery(aggregate); + } catch (err: any) { + assert.strictEqual( + err.message, + '3 INVALID_ARGUMENT: The maximum number of aggregations allowed in an aggregation query is 5. Received: 6' + ); + } + }); }); it('should filter by ancestor', async () => { const q = datastore.createQuery('Character').hasAncestor(ancestor); @@ -1130,6 +1479,110 @@ describe('Datastore', () => { }); }); + describe('querying the datastore with an overflow data set', () => { + const keys = [ + // Paths: + ['Rickard'], + ['Rickard', 'Character', 'Eddard'], + ].map(path => { + return datastore.key(['Book', 'GoT', 'Character'].concat(path)); + }); + const characters = [ + { + name: 'Rickard', + family: 'Stark', + // eslint-disable-next-line @typescript-eslint/no-loss-of-precision + appearances: 9223372036854775807, + alive: false, + }, + { + name: 'Eddard', + family: 'Stark', + // eslint-disable-next-line @typescript-eslint/no-loss-of-precision + appearances: 9223372036854775807, + alive: false, + }, + ]; + before(async () => { + const keysToSave = keys.map((key, index) => { + return { + key, + data: characters[index], + }; + }); + await datastore.save(keysToSave); + }); + after(async () => { + await datastore.delete(keys); + }); + it('should run a sum aggregation with an overflow dataset', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.sum('appearances')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: -18446744073709552000}]); + }); + it('should run an average aggregation with an overflow dataset', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.average('appearances')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: -9223372036854776000}]); + }); + }); + describe('querying the datastore with an NaN in the data set', () => { + const keys = [ + // Paths: + ['Rickard'], + ['Rickard', 'Character', 'Eddard'], + ].map(path => { + return datastore.key(['Book', 'GoT', 'Character'].concat(path)); + }); + const characters = [ + { + name: 'Rickard', + family: 'Stark', + appearances: 4, + alive: false, + }, + { + name: 'Eddard', + family: 'Stark', + appearances: null, + alive: false, + }, + ]; + before(async () => { + const keysToSave = keys.map((key, index) => { + return { + key, + data: characters[index], + }; + }); + await datastore.save(keysToSave); + }); + after(async () => { + await datastore.delete(keys); + }); + it('should run a sum aggregation', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.sum('appearances')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 4}]); + }); + it('should run an average aggregation', async () => { + const q = datastore.createQuery('Character'); + const aggregate = datastore + .createAggregationQuery(q) + .addAggregation(AggregateField.average('appearances')); + const [results] = await datastore.runAggregationQuery(aggregate); + assert.deepStrictEqual(results, [{property_1: 4}]); + }); + }); describe('transactions', () => { it('should run in a transaction', async () => { const key = datastore.key(['Company', 'Google']); @@ -1231,22 +1684,58 @@ describe('Datastore', () => { await transaction.commit(); }); - it('should aggregate query within a transaction', async () => { - const transaction = datastore.transaction(); - await transaction.run(); - const query = transaction.createQuery('Company'); - const aggregateQuery = transaction - .createAggregationQuery(query) - .count('total'); - let result; - try { - [result] = await aggregateQuery.run(); - } catch (e) { - await transaction.rollback(); - return; - } - assert.deepStrictEqual(result, [{total: 2}]); - await transaction.commit(); + describe('aggregate query within a transaction', async () => { + it('should aggregate query within a count transaction', async () => { + const transaction = datastore.transaction(); + await transaction.run(); + const query = transaction.createQuery('Company'); + const aggregateQuery = transaction + .createAggregationQuery(query) + .count('total'); + let result; + try { + [result] = await aggregateQuery.run(); + } catch (e) { + await transaction.rollback(); + assert.fail('The aggregation query run should have been successful'); + } + assert.deepStrictEqual(result, [{total: 2}]); + await transaction.commit(); + }); + it('should aggregate query within a sum transaction', async () => { + const transaction = datastore.transaction(); + await transaction.run(); + const query = transaction.createQuery('Company'); + const aggregateQuery = transaction + .createAggregationQuery(query) + .sum('rating', 'total rating'); + let result; + try { + [result] = await aggregateQuery.run(); + } catch (e) { + await transaction.rollback(); + assert.fail('The aggregation query run should have been successful'); + } + assert.deepStrictEqual(result, [{'total rating': 200}]); + await transaction.commit(); + }); + it('should aggregate query within a average transaction', async () => { + const transaction = datastore.transaction(); + await transaction.run(); + const query = transaction.createQuery('Company'); + const aggregateQuery = transaction + .createAggregationQuery(query) + .average('rating', 'average rating'); + let result; + try { + [result] = await aggregateQuery.run(); + } catch (e) { + await transaction.rollback(); + assert.fail('The aggregation query run should have been successful'); + } + assert.deepStrictEqual(result, [{'average rating': 100}]); + await transaction.commit(); + }); }); it('should read in a readOnly transaction', async () => { diff --git a/test/query.ts b/test/query.ts index aae51442d..36816c6b2 100644 --- a/test/query.ts +++ b/test/query.ts @@ -58,22 +58,148 @@ describe('Query', () => { }); }); - it('should create a query with a count aggregation', () => { - const query = new Query(['kind1']); - const firstAggregation = AggregateField.count().alias('total'); - const secondAggregation = AggregateField.count().alias('total2'); - const aggregate = new AggregateQuery(query).addAggregations([ - firstAggregation, - secondAggregation, - ]); - const aggregate2 = new AggregateQuery(query) - .count('total') - .count('total2'); - assert.deepStrictEqual(aggregate.aggregations, aggregate2.aggregations); - assert.deepStrictEqual(aggregate.aggregations, [ - firstAggregation, - secondAggregation, - ]); + describe('Aggregation queries', () => { + it('should create a query with a count aggregation', () => { + const query = new Query(['kind1']); + const firstAggregation = AggregateField.count().alias('total'); + const secondAggregation = AggregateField.count().alias('total2'); + const aggregate = new AggregateQuery(query).addAggregations([ + firstAggregation, + secondAggregation, + ]); + const aggregate2 = new AggregateQuery(query) + .count('total') + .count('total2'); + assert.deepStrictEqual(aggregate.aggregations, aggregate2.aggregations); + assert.deepStrictEqual(aggregate.aggregations, [ + firstAggregation, + secondAggregation, + ]); + }); + + describe('AggregateField toProto', () => { + it('should produce the right proto with a count aggregation', () => { + assert.deepStrictEqual( + AggregateField.count().alias('alias1').toProto(), + { + alias: 'alias1', + count: {}, + } + ); + }); + it('should produce the right proto with a sum aggregation', () => { + assert.deepStrictEqual( + AggregateField.sum('property1').alias('alias1').toProto(), + { + alias: 'alias1', + operator: 'sum', + sum: { + property: { + name: 'property1', + }, + }, + } + ); + }); + it('should produce the right proto with an average aggregation', () => { + assert.deepStrictEqual( + AggregateField.average('property1').alias('alias1').toProto(), + { + alias: 'alias1', + avg: { + property: { + name: 'property1', + }, + }, + operator: 'avg', + } + ); + }); + }); + + describe('comparing equivalent aggregation queries', async () => { + function generateAggregateQuery() { + return new AggregateQuery(new Query(['kind1'])); + } + + function compareAggregations( + aggregateQuery: AggregateQuery, + aggregateFields: AggregateField[] + ) { + const addAggregationsAggregate = generateAggregateQuery(); + addAggregationsAggregate.addAggregations(aggregateFields); + const addAggregationAggregate = generateAggregateQuery(); + aggregateFields.forEach(aggregateField => + addAggregationAggregate.addAggregation(aggregateField) + ); + assert.deepStrictEqual( + aggregateQuery.aggregations, + addAggregationsAggregate.aggregations + ); + assert.deepStrictEqual( + aggregateQuery.aggregations, + addAggregationAggregate.aggregations + ); + assert.deepStrictEqual(aggregateQuery.aggregations, aggregateFields); + } + describe('comparing aggregations with an alias', async () => { + it('should compare equivalent count aggregation queries', () => { + compareAggregations( + generateAggregateQuery().count('total1').count('total2'), + ['total1', 'total2'].map(alias => + AggregateField.count().alias(alias) + ) + ); + }); + it('should compare equivalent sum aggregation queries', () => { + compareAggregations( + generateAggregateQuery() + .sum('property1', 'alias1') + .sum('property2', 'alias2'), + [ + AggregateField.sum('property1').alias('alias1'), + AggregateField.sum('property2').alias('alias2'), + ] + ); + }); + it('should compare equivalent average aggregation queries', () => { + compareAggregations( + generateAggregateQuery() + .average('property1', 'alias1') + .average('property2', 'alias2'), + [ + AggregateField.average('property1').alias('alias1'), + AggregateField.average('property2').alias('alias2'), + ] + ); + }); + }); + describe('comparing aggregations without an alias', async () => { + it('should compare equivalent count aggregation queries', () => { + compareAggregations( + generateAggregateQuery().count().count(), + ['total1', 'total2'].map(() => AggregateField.count()) + ); + }); + it('should compare equivalent sum aggregation queries', () => { + compareAggregations( + generateAggregateQuery().sum('property1').sum('property2'), + [AggregateField.sum('property1'), AggregateField.sum('property2')] + ); + }); + it('should compare equivalent average aggregation queries', () => { + compareAggregations( + generateAggregateQuery() + .average('property1') + .average('property2'), + [ + AggregateField.average('property1'), + AggregateField.average('property2'), + ] + ); + }); + }); + }); }); });