From 2316037168efbf8433684f7f020c225083c39b5b Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Fri, 18 Oct 2019 17:32:17 -0500 Subject: [PATCH 1/8] Added basic plumbing to push attributes from the entity projection down to the QueryEngine --- .../yahoo/elide/core/EntityDictionary.java | 11 +++++++++ .../yahoo/elide/core/PersistentResource.java | 23 +++++++++++++++++++ .../queryengines/sql/schema/SQLSchema.java | 8 ++++++- .../graphql/containers/NodeContainer.java | 8 ++++++- .../parser/GraphQLEntityProjectionMaker.java | 4 +++- .../graphql/parser/GraphQLProjectionInfo.java | 3 +++ 6 files changed, 54 insertions(+), 3 deletions(-) diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java index 7d9b956967..fa0e716f9d 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java @@ -7,6 +7,7 @@ import static com.yahoo.elide.core.EntityBinding.EMPTY_BINDING; +import com.google.common.collect.Sets; import com.yahoo.elide.Injector; import com.yahoo.elide.annotation.ComputedAttribute; import com.yahoo.elide.annotation.ComputedRelationship; @@ -1422,6 +1423,16 @@ public void addArgumentsToAttributes(Class cls, String attributeName, Set cls, String attributeName, ArgumentType argument) { + this.addArgumentsToAttributes(cls, attributeName, Sets.newHashSet(argument)); + } + /** * Returns the Collection of all attributes of an argument. * @param cls The entity diff --git a/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java b/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java index 9deca8a35b..457651b473 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/PersistentResource.java @@ -1104,10 +1104,20 @@ public RelationshipType getRelationshipType(String relation) { * @param attr Attribute name * @return Object value for attribute */ + @Deprecated public Object getAttribute(String attr) { return this.getValueChecked(attr); } + /** + * Get the value for a particular attribute (i.e. non-relational field) + * @param attr the Attribute + * @return Object value for attribute + */ + public Object getAttribute(Attribute attr) { + return this.getValueChecked(attr); + } + /** * Wrapped Entity bean. * @@ -1363,6 +1373,7 @@ protected void nullValue(String fieldName, PersistentResource oldValue) { * @param fieldName the field name * @return value value */ + @Deprecated protected Object getValueChecked(String fieldName) { requestScope.publishLifecycleEvent(this, CRUDEvent.CRUDAction.READ); requestScope.publishLifecycleEvent(this, fieldName, CRUDEvent.CRUDAction.READ, Optional.empty()); @@ -1370,6 +1381,18 @@ protected Object getValueChecked(String fieldName) { return getValue(getObject(), fieldName, requestScope); } + /** + * Gets a value from an entity and checks read permissions. + * @param attribute the attribute to fetch. + * @return value value + */ + protected Object getValueChecked(Attribute attribute) { + requestScope.publishLifecycleEvent(this, CRUDEvent.CRUDAction.READ); + requestScope.publishLifecycleEvent(this, attribute.getName(), CRUDEvent.CRUDAction.READ, Optional.empty()); + checkFieldAwareDeferPermissions(ReadPermission.class, attribute.getName(), (Object) null, (Object) null); + return transaction.getAttribute(getObject(), attribute, requestScope); + } + /** * Retrieve an object without checking read permissions (i.e. value is used internally and not sent to others) * diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java index 1d3acbfc05..0013101276 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java @@ -6,6 +6,7 @@ package com.yahoo.elide.datastores.aggregation.queryengines.sql.schema; +import com.yahoo.elide.core.ArgumentType; import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.Path; import com.yahoo.elide.core.filter.FilterPredicate; @@ -18,9 +19,9 @@ import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn; import com.yahoo.elide.datastores.aggregation.schema.metric.Aggregation; import com.yahoo.elide.datastores.aggregation.schema.metric.Metric; +import com.yahoo.elide.datastores.aggregation.time.TimeGrain; import org.hibernate.annotations.Subselect; - import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; @@ -76,6 +77,11 @@ protected DimensionColumn constructDimension(String dimensionField, String columnName = getColumnName(entityClass, dimensionField); if (dim instanceof TimeDimensionColumn) { + + //Add grain to the GraphQL schema. + entityDictionary.addArgumentToAttributes(this.entityClass, dimensionField, + new ArgumentType("grain", TimeGrain.class)); + return new SQLTimeDimensionColumn((TimeDimensionColumn) dim, columnName, getAlias()); } return new SQLDimensionColumn(dim, columnName, getAlias()); diff --git a/elide-graphql/src/main/java/com/yahoo/elide/graphql/containers/NodeContainer.java b/elide-graphql/src/main/java/com/yahoo/elide/graphql/containers/NodeContainer.java index 50a1bd7f14..382f3ace16 100644 --- a/elide-graphql/src/main/java/com/yahoo/elide/graphql/containers/NodeContainer.java +++ b/elide-graphql/src/main/java/com/yahoo/elide/graphql/containers/NodeContainer.java @@ -10,8 +10,12 @@ import com.yahoo.elide.graphql.DeferredId; import com.yahoo.elide.graphql.Environment; import com.yahoo.elide.graphql.PersistentResourceFetcher; +import com.yahoo.elide.request.Argument; +import com.yahoo.elide.request.Attribute; +import com.yahoo.elide.request.EntityProjection; import com.yahoo.elide.request.Relationship; +import graphql.language.Field; import lombok.AllArgsConstructor; import lombok.Getter; @@ -36,7 +40,9 @@ public Object processFetch(Environment context, PersistentResourceFetcher fetche String idFieldName = dictionary.getIdFieldName(parentClass); if (dictionary.isAttribute(parentClass, fieldName)) { /* fetch attribute properties */ - Object attribute = context.parentResource.getAttribute(fieldName); + Attribute requested = context.requestScope.getProjectionInfo() + .getAttributeMap().getOrDefault(context.field.getSourceLocation(), null); + Object attribute = context.parentResource.getAttribute(requested); if (attribute instanceof Map) { return ((Map) attribute).entrySet().stream() .map(MapEntryContainer::new) diff --git a/elide-graphql/src/main/java/com/yahoo/elide/graphql/parser/GraphQLEntityProjectionMaker.java b/elide-graphql/src/main/java/com/yahoo/elide/graphql/parser/GraphQLEntityProjectionMaker.java index e5a1fc9f82..e3bc2ea05c 100644 --- a/elide-graphql/src/main/java/com/yahoo/elide/graphql/parser/GraphQLEntityProjectionMaker.java +++ b/elide-graphql/src/main/java/com/yahoo/elide/graphql/parser/GraphQLEntityProjectionMaker.java @@ -69,6 +69,7 @@ public class GraphQLEntityProjectionMaker { private final Map relationshipMap = new HashMap<>(); private final Map rootProjections = new HashMap<>(); + private final Map attributeMap = new HashMap<>(); /** * Constructor. @@ -134,7 +135,7 @@ public GraphQLProjectionInfo make(String query) { } }); - return new GraphQLProjectionInfo(rootProjections, relationshipMap); + return new GraphQLProjectionInfo(rootProjections, relationshipMap, attributeMap); } /** @@ -351,6 +352,7 @@ private void addAttributeField(Field attributeField, EntityProjectionBuilder pro .build(); projectionBuilder.attribute(attribute); + attributeMap.put(attributeField.getSourceLocation(), attribute); } else { throw new InvalidEntityBodyException(String.format( "Unknown attribute field {%s.%s}.", diff --git a/elide-graphql/src/main/java/com/yahoo/elide/graphql/parser/GraphQLProjectionInfo.java b/elide-graphql/src/main/java/com/yahoo/elide/graphql/parser/GraphQLProjectionInfo.java index af99e2671e..b067687068 100644 --- a/elide-graphql/src/main/java/com/yahoo/elide/graphql/parser/GraphQLProjectionInfo.java +++ b/elide-graphql/src/main/java/com/yahoo/elide/graphql/parser/GraphQLProjectionInfo.java @@ -6,6 +6,7 @@ package com.yahoo.elide.graphql.parser; +import com.yahoo.elide.request.Attribute; import com.yahoo.elide.request.EntityProjection; import com.yahoo.elide.request.Relationship; import graphql.language.SourceLocation; @@ -23,4 +24,6 @@ public class GraphQLProjectionInfo { @Getter private final Map projections; @Getter private final Map relationshipMap; + + @Getter private final Map attributeMap; } From 71dcaa468cebe769c4fdc8a1119887580c8df264 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Mon, 21 Oct 2019 18:24:18 -0500 Subject: [PATCH 2/8] Added logic to expand SQL time expression in SQLQueryEngine --- .../yahoo/elide/core/EntityDictionary.java | 2 +- .../queryengines/sql/SQLQueryEngine.java | 14 +++++++- .../sql/schema/SQLTimeDimensionColumn.java | 32 +++++++++++++++---- .../aggregation/example/PlayerStats.java | 2 ++ .../graphql/containers/NodeContainer.java | 3 -- 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java index fa0e716f9d..2a7e2361d8 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java @@ -7,7 +7,6 @@ import static com.yahoo.elide.core.EntityBinding.EMPTY_BINDING; -import com.google.common.collect.Sets; import com.yahoo.elide.Injector; import com.yahoo.elide.annotation.ComputedAttribute; import com.yahoo.elide.annotation.ComputedRelationship; @@ -30,6 +29,7 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.antlr.v4.runtime.tree.ParseTree; import org.apache.commons.lang3.StringUtils; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index feb071514d..d9d1c81797 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -23,8 +23,10 @@ import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.JoinTo; import com.yahoo.elide.datastores.aggregation.queryengines.sql.schema.SQLDimensionColumn; import com.yahoo.elide.datastores.aggregation.queryengines.sql.schema.SQLSchema; +import com.yahoo.elide.datastores.aggregation.queryengines.sql.schema.SQLTimeDimensionColumn; import com.yahoo.elide.datastores.aggregation.schema.Schema; import com.yahoo.elide.datastores.aggregation.schema.dimension.DimensionColumn; +import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn; import com.yahoo.elide.datastores.aggregation.schema.metric.Aggregation; import com.yahoo.elide.datastores.aggregation.schema.metric.Metric; import com.yahoo.elide.utils.coerce.CoerceUtil; @@ -470,11 +472,21 @@ private String extractGroupBy(Query query) { * @return */ private List extractDimensionProjections(Query query) { - return query.getDimensions().stream() + List dimensionStrings = query.getGroupDimensions().stream() .map(requestedDim -> query.getSchema().getDimension(requestedDim.getName())) .map((SQLDimensionColumn.class::cast)) .map(SQLDimensionColumn::getColumnReference) .collect(Collectors.toList()); + + dimensionStrings.addAll(query.getTimeDimensions().stream() + .map(requestedDim -> { + SQLTimeDimensionColumn timeDim = (SQLTimeDimensionColumn) + query.getSchema().getTimeDimension(requestedDim.getName()); + + return timeDim.getColumnReference(requestedDim.getTimeGrain().grain()); + }).collect(Collectors.toList())); + + return dimensionStrings; } /** diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java index 92c53ba80f..81cec15c15 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java @@ -6,9 +6,11 @@ package com.yahoo.elide.datastores.aggregation.queryengines.sql.schema; +import com.google.common.base.Preconditions; import com.yahoo.elide.core.Path; import com.yahoo.elide.datastores.aggregation.annotation.TimeGrainDefinition; import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn; +import com.yahoo.elide.datastores.aggregation.time.TimeGrain; import java.util.Set; import java.util.TimeZone; @@ -20,9 +22,10 @@ public class SQLTimeDimensionColumn extends SQLDimensionColumn implements TimeDimensionColumn { /** * Constructor. - * @param dimension a wrapped dimension. + * + * @param dimension a wrapped dimension. * @param columnAlias The column alias in SQL to refer to this dimension. - * @param tableAlias The table alias in SQL where this dimension lives. + * @param tableAlias The table alias in SQL where this dimension lives. */ public SQLTimeDimensionColumn(TimeDimensionColumn dimension, String columnAlias, String tableAlias) { super(dimension, columnAlias, tableAlias); @@ -30,11 +33,12 @@ public SQLTimeDimensionColumn(TimeDimensionColumn dimension, String columnAlias, /** * Constructor. - * @param dimension a wrapped dimension. + * + * @param dimension a wrapped dimension. * @param columnAlias The column alias in SQL to refer to this dimension. - * @param tableAlias The table alias in SQL where this dimension lives. - * @param joinPath A '.' separated path through the entity relationship graph that describes - * how to join the time dimension into the current AnalyticView. + * @param tableAlias The table alias in SQL where this dimension lives. + * @param joinPath A '.' separated path through the entity relationship graph that describes + * how to join the time dimension into the current AnalyticView. */ public SQLTimeDimensionColumn(TimeDimensionColumn dimension, String columnAlias, String tableAlias, Path joinPath) { super(dimension, columnAlias, tableAlias, joinPath); @@ -50,4 +54,20 @@ public TimeZone getTimeZone() { public Set getSupportedGrains() { return ((TimeDimensionColumn) wrapped).getSupportedGrains(); } + + /** + * Returns a String that identifies this dimension in a SQL query. + * @param requestedGrain The requested time grain. + * + * @return Something like "table_alias.column_name" + */ + public String getColumnReference(TimeGrain requestedGrain) { + Preconditions.checkArgument(getSupportedGrains().stream() + .anyMatch((grainDef -> grainDef.grain().equals(requestedGrain)))); + + TimeGrainDefinition definition = getSupportedGrains().stream() + .filter(grainDef -> grainDef.grain().equals(requestedGrain)).findFirst().get(); + + return String.format(definition.expression(), getColumnReference()); + } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java index 189a0111a5..a796657eff 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java @@ -41,6 +41,8 @@ @FromTable(name = "playerStats") public class PlayerStats { + public static final String DAY_FORMAT = + /** * PK. */ diff --git a/elide-graphql/src/main/java/com/yahoo/elide/graphql/containers/NodeContainer.java b/elide-graphql/src/main/java/com/yahoo/elide/graphql/containers/NodeContainer.java index 382f3ace16..f246851e54 100644 --- a/elide-graphql/src/main/java/com/yahoo/elide/graphql/containers/NodeContainer.java +++ b/elide-graphql/src/main/java/com/yahoo/elide/graphql/containers/NodeContainer.java @@ -10,12 +10,9 @@ import com.yahoo.elide.graphql.DeferredId; import com.yahoo.elide.graphql.Environment; import com.yahoo.elide.graphql.PersistentResourceFetcher; -import com.yahoo.elide.request.Argument; import com.yahoo.elide.request.Attribute; -import com.yahoo.elide.request.EntityProjection; import com.yahoo.elide.request.Relationship; -import graphql.language.Field; import lombok.AllArgsConstructor; import lombok.Getter; From f5d62bc706331d8e5cc17380617ab8bb7ff029cc Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 22 Oct 2019 11:59:42 -0500 Subject: [PATCH 3/8] Added SQLQueryEngine tests --- .../elide/security/FilterExpressionCheck.java | 1 - .../queryengines/sql/SQLQueryEngine.java | 19 +++++- .../aggregation/example/PlayerStats.java | 8 ++- .../queryengines/sql/SQLQueryEngineTest.java | 60 +++++++++++++++++++ 4 files changed, 83 insertions(+), 5 deletions(-) diff --git a/elide-core/src/main/java/com/yahoo/elide/security/FilterExpressionCheck.java b/elide-core/src/main/java/com/yahoo/elide/security/FilterExpressionCheck.java index c64486f797..36dde42a10 100644 --- a/elide-core/src/main/java/com/yahoo/elide/security/FilterExpressionCheck.java +++ b/elide-core/src/main/java/com/yahoo/elide/security/FilterExpressionCheck.java @@ -43,7 +43,6 @@ public final boolean ok(User user) { throw new UnsupportedOperationException(); } - /** * The filter expression is evaluated in memory if it cannot be pushed to the data store by elide for any reason. * diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index d9d1c81797..09afd6a10b 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -462,7 +462,21 @@ private String extractProjection(Query query) { * @return The SQL GROUP BY clause */ private String extractGroupBy(Query query) { - return "GROUP BY " + extractDimensionProjections(query).stream() + List dimensionStrings = query.getGroupDimensions().stream() + .map(requestedDim -> query.getSchema().getDimension(requestedDim.getName())) + .map((SQLDimensionColumn.class::cast)) + .map(SQLDimensionColumn::getColumnReference) + .collect(Collectors.toList()); + + dimensionStrings.addAll(query.getTimeDimensions().stream() + .map(requestedDim -> { + SQLTimeDimensionColumn timeDim = (SQLTimeDimensionColumn) + query.getSchema().getTimeDimension(requestedDim.getName()); + + return timeDim.getColumnReference(requestedDim.getTimeGrain().grain()); + }).collect(Collectors.toList())); + + return "GROUP BY " + dimensionStrings.stream() .collect(Collectors.joining(",")); } @@ -483,7 +497,8 @@ private List extractDimensionProjections(Query query) { SQLTimeDimensionColumn timeDim = (SQLTimeDimensionColumn) query.getSchema().getTimeDimension(requestedDim.getName()); - return timeDim.getColumnReference(requestedDim.getTimeGrain().grain()); + return timeDim.getColumnReference(requestedDim.getTimeGrain().grain()) + + " AS " + timeDim.getColumnName(); }).collect(Collectors.toList())); return dimensionStrings; diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java index a796657eff..27b9624033 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java @@ -41,7 +41,8 @@ @FromTable(name = "playerStats") public class PlayerStats { - public static final String DAY_FORMAT = + public static final String DAY_FORMAT = "PARSEDATETIME(FORMATDATETIME(%s, 'yyyy-MM-dd'), 'yyyy-MM-dd')"; + public static final String MONTH_FORMAT = "PARSEDATETIME(FORMATDATETIME(%s, 'yyyy-MM-01'), 'yyyy-MM-dd')"; /** * PK. @@ -163,7 +164,10 @@ public void setPlayer(final Player player) { * {@link EntityDimensionTest#testCardinalityScan()}. * @return the date of the player session. */ - @Temporal(grains = { @TimeGrainDefinition(grain = TimeGrain.DAY, expression = "") }, timeZone = "UTC") + @Temporal(grains = { + @TimeGrainDefinition(grain = TimeGrain.DAY, expression = DAY_FORMAT), + @TimeGrainDefinition(grain = TimeGrain.MONTH, expression = MONTH_FORMAT) + }, timeZone = "UTC") public Date getRecordedDate() { return recordedDate; } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java index b8dad3dbc6..c9166202d6 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java @@ -9,7 +9,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import com.google.common.collect.Lists; import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.core.Path; +import com.yahoo.elide.core.filter.FilterPredicate; +import com.yahoo.elide.core.filter.Operator; import com.yahoo.elide.core.filter.dialect.RSQLFilterDialect; import com.yahoo.elide.core.pagination.Pagination; import com.yahoo.elide.core.sort.Sorting; @@ -647,6 +651,62 @@ public void testJoinToSort() throws Exception { assertEquals(stats3, results.get(2)); } + /** + * Test month grain query. + */ + @Test + public void testTotalScoreByMonth() { + QueryEngine engine = new SQLQueryEngine(emf, dictionary); + + Query query = Query.builder() + .schema(playerStatsSchema) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) + .timeDimension(toTimeDimension(playerStatsSchema, TimeGrain.MONTH, "recordedDate")) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStats stats0 = new PlayerStats(); + stats0.setId("0"); + stats0.setHighScore(4646); + stats0.setRecordedDate(Timestamp.valueOf("2019-07-01 00:00:00")); + + assertEquals(1, results.size()); + assertEquals(stats0, results.get(0)); + } + + /** + * Test filter by time dimension + */ + @Test + public void testFilterByTemporalDimension() throws Exception { + QueryEngine engine = new SQLQueryEngine(emf, dictionary); + + FilterPredicate predicate = new FilterPredicate( + new Path(PlayerStats.class, dictionary, "recordedDate"), + Operator.IN, + Lists.newArrayList(Timestamp.valueOf("2019-07-11 00:00:00"))); + + Query query = Query.builder() + .schema(playerStatsSchema) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) + .timeDimension(toTimeDimension(playerStatsSchema, TimeGrain.DAY, "recordedDate")) + .whereFilter(predicate) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStats stats0 = new PlayerStats(); + stats0.setId("0"); + stats0.setHighScore(2412); + stats0.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); + + assertEquals(1, results.size()); + assertEquals(stats0, results.get(0)); + } + //TODO - Add Invalid Request Tests /** From af10caba6bcacad2a71863e0f3e4fba876d85f40 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 22 Oct 2019 14:12:09 -0500 Subject: [PATCH 4/8] Added IT tests --- .../pom.xml.releaseBackup | 198 ++++++++++++++++++ .../AggregationDataStoreHelper.java | 70 ++++--- .../queryengines/sql/SQLQueryEngine.java | 1 - .../queryengines/sql/schema/SQLSchema.java | 3 +- .../sql/schema/SQLTimeDimensionColumn.java | 3 +- .../AggregationDataStoreIntegrationTest.java | 30 +++ .../queryengines/sql/SQLQueryEngineTest.java | 2 +- .../initialization/StandardTestBinder.java | 2 + 8 files changed, 270 insertions(+), 39 deletions(-) create mode 100644 elide-datastore/elide-datastore-aggregation/pom.xml.releaseBackup diff --git a/elide-datastore/elide-datastore-aggregation/pom.xml.releaseBackup b/elide-datastore/elide-datastore-aggregation/pom.xml.releaseBackup new file mode 100644 index 0000000000..ccf7c1fbd9 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/pom.xml.releaseBackup @@ -0,0 +1,198 @@ + + + + 4.0.0 + elide-datastore-aggregation + jar + Elide Data Store: Aggregation Data Store + Elide Data Store for Aggregation + https://github.com/yahoo/elide/tree/master/elide-datastore/elide-datastore-aggregation + + com.yahoo.elide + elide-datastore-parent-pom + 5.0.0-pr2-SNAPSHOT + + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + Yahoo Inc. + https://github.com/yahoo + + + + + scm:git:ssh://git@github.com/yahoo/elide.git + https://github.com/yahoo/elide.git + HEAD + + + + 5.4.1 + + + + + com.yahoo.elide + elide-core + 5.0.0-pr2-SNAPSHOT + + + + com.yahoo.elide + elide-datastore-hibernate + 5.0.0-pr2-SNAPSHOT + + + + com.yahoo.elide + elide-graphql + 5.0.0-pr2-SNAPSHOT + + + + org.projectlombok + lombok + + + + + org.hibernate.javax.persistence + hibernate-jpa-2.1-api + 1.0.0.Final + + + + + org.hibernate + hibernate-entitymanager + ${hibernate5.version} + + + + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-params + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + + + com.h2database + h2 + 1.4.197 + test + + + + + org.mockito + mockito-core + test + + + + + com.yahoo.elide + elide-integration-tests + 5.0.0-pr2-SNAPSHOT + test + test-jar + + + + + org.eclipse.jetty + jetty-servlet + test + + + + org.eclipse.jetty + jetty-webapp + test + + + + + org.glassfish.jersey.containers + jersey-container-servlet-core + 2.29.1 + test + + + + org.glassfish.jersey.inject + jersey-hk2 + test + + + + org.glassfish.jersey.containers + jersey-container-servlet + test + + + + io.rest-assured + rest-assured + 4.0.0 + test + + + + org.glassfish.hk2 + hk2-api + 2.5.0 + test + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java index 19d03d76df..ba4172c604 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java @@ -31,6 +31,7 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -109,46 +110,47 @@ private void splitFilters() { * @throws InvalidOperationException Thrown if a requested time grain is not supported. */ private Set resolveTimeDimensions() { - return entityProjection.getAttributes().stream() - .filter(attribute -> schema.getTimeDimension(attribute.getName()) != null) - .map(attribute -> { - TimeDimensionColumn timeDim = schema.getTimeDimension(attribute.getName()); - Argument timeGrainArgument = attribute.getArguments().stream() - .filter(attr -> attr.getName().equals("grain")) - .findAny() - .orElse(null); + Set timeDims = new LinkedHashSet<>(); + //Only attributes can be time dimensions + entityProjection.getAttributes().stream().forEach((attribute -> { + TimeDimensionColumn timeDim = schema.getTimeDimension(attribute.getName()); + if (timeDim == null) { + return; + } + + Argument timeGrainArgument = attribute.getArguments().stream() + .filter(attr -> attr.getName().equals("grain")) + .findAny() + .orElse(null); + + TimeGrainDefinition requestedGrainDefinition; + if (timeGrainArgument == null) { - TimeGrainDefinition requestedGrainDefinition; - if (timeGrainArgument == null) { + //The first grain is the default. + requestedGrainDefinition = timeDim.getSupportedGrains().iterator().next(); + } else { + String requestedGrainName = timeGrainArgument.getValue().toString().toUpperCase(Locale.ENGLISH); - //The first grain is the default. - requestedGrainDefinition = timeDim.getSupportedGrains().stream() - .findFirst() - .orElseThrow(() -> new IllegalStateException( - String.format("Requested default grain, no grain defined on %s", - attribute.getName()))); - } else { - String requestedGrainName = timeGrainArgument.getValue().toString(); + TimeGrain requestedGrain; + try { + requestedGrain = TimeGrain.valueOf(requestedGrainName); + } catch (IllegalArgumentException e) { + throw new InvalidOperationException(String.format("Invalid grain %s", requestedGrainName)); + } - TimeGrain requestedGrain; - try { - requestedGrain = TimeGrain.valueOf(requestedGrainName); - } catch (IllegalArgumentException e) { - throw new InvalidOperationException(String.format("Invalid grain %s", requestedGrainName)); - } + requestedGrainDefinition = timeDim.getSupportedGrains().stream() + .filter(supportedGrainDef -> supportedGrainDef.grain().equals(requestedGrain)) + .findAny() + .orElseThrow(() -> new InvalidOperationException( + String.format("Requested grain %s, not supported on %s", + requestedGrainName, attribute.getName()))); + } - requestedGrainDefinition = timeDim.getSupportedGrains().stream() - .filter(supportedGrainDef -> supportedGrainDef.grain().equals(requestedGrain)) - .findAny() - .orElseThrow(() -> new InvalidOperationException( - String.format("Requested grain %s, not supported on %s", - requestedGrainName, attribute.getName()))); - } + timeDims.add(timeDim.toProjectedDimension(requestedGrainDefinition)); + })); - return timeDim.toProjectedDimension(requestedGrainDefinition); - }) - .collect(Collectors.toCollection(LinkedHashSet::new)); + return timeDims; } /** diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index 09afd6a10b..0b913e3fd5 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -26,7 +26,6 @@ import com.yahoo.elide.datastores.aggregation.queryengines.sql.schema.SQLTimeDimensionColumn; import com.yahoo.elide.datastores.aggregation.schema.Schema; import com.yahoo.elide.datastores.aggregation.schema.dimension.DimensionColumn; -import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn; import com.yahoo.elide.datastores.aggregation.schema.metric.Aggregation; import com.yahoo.elide.datastores.aggregation.schema.metric.Metric; import com.yahoo.elide.utils.coerce.CoerceUtil; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java index 0013101276..266340cbe7 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java @@ -19,7 +19,6 @@ import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn; import com.yahoo.elide.datastores.aggregation.schema.metric.Aggregation; import com.yahoo.elide.datastores.aggregation.schema.metric.Metric; -import com.yahoo.elide.datastores.aggregation.time.TimeGrain; import org.hibernate.annotations.Subselect; import lombok.EqualsAndHashCode; @@ -80,7 +79,7 @@ protected DimensionColumn constructDimension(String dimensionField, //Add grain to the GraphQL schema. entityDictionary.addArgumentToAttributes(this.entityClass, dimensionField, - new ArgumentType("grain", TimeGrain.class)); + new ArgumentType("grain", String.class)); return new SQLTimeDimensionColumn((TimeDimensionColumn) dim, columnName, getAlias()); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java index 81cec15c15..5368075776 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java @@ -6,12 +6,13 @@ package com.yahoo.elide.datastores.aggregation.queryengines.sql.schema; -import com.google.common.base.Preconditions; import com.yahoo.elide.core.Path; import com.yahoo.elide.datastores.aggregation.annotation.TimeGrainDefinition; import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn; import com.yahoo.elide.datastores.aggregation.time.TimeGrain; +import com.google.common.base.Preconditions; + import java.util.Set; import java.util.TimeZone; diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreIntegrationTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreIntegrationTest.java index ef938fe2ae..3afefb931a 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreIntegrationTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreIntegrationTest.java @@ -477,6 +477,36 @@ public void aggregationComputedMetricTest() throws Exception { runQueryWithExpectedResult(graphQLRequest, expected); } + @Test + public void timeGrainAggregationTest() throws Exception { + String graphQLRequest = document( + selection( + field( + "playerStats", + selections( + field("highScore"), + field("recordedDate", arguments( + argument("grain", "\"month\"") + )) + ) + ) + ) + ).toQuery(); + + String expected = document( + selections( + field( + "playerStats", + selections( + field("highScore", 2412), + field("recordedDate", "2019-07-01T00:00Z") + ) + ) + )).toResponse(); + + runQueryWithExpectedResult(graphQLRequest, expected); + } + private void create(String query, Map variables) throws IOException { runQuery(toJsonQuery(query, variables)); } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java index c9166202d6..84489832d4 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java @@ -9,7 +9,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import com.google.common.collect.Lists; import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.Path; import com.yahoo.elide.core.filter.FilterPredicate; @@ -31,6 +30,7 @@ import com.yahoo.elide.datastores.aggregation.schema.metric.Sum; import com.yahoo.elide.datastores.aggregation.time.TimeGrain; +import com.google.common.collect.Lists; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/elide-integration-tests/src/test/java/com/yahoo/elide/initialization/StandardTestBinder.java b/elide-integration-tests/src/test/java/com/yahoo/elide/initialization/StandardTestBinder.java index 993f3a2f95..b1747e4366 100644 --- a/elide-integration-tests/src/test/java/com/yahoo/elide/initialization/StandardTestBinder.java +++ b/elide-integration-tests/src/test/java/com/yahoo/elide/initialization/StandardTestBinder.java @@ -24,6 +24,7 @@ import org.glassfish.hk2.utilities.binding.AbstractBinder; import java.util.Arrays; +import java.util.Calendar; /** * Typical-use test binder for integration test resource configs. @@ -59,6 +60,7 @@ public Elide provide() { .withJoinFilterDialect(multipleFilterStrategy) .withSubqueryFilterDialect(multipleFilterStrategy) .withEntityDictionary(dictionary) + .withISO8601Dates("yyyy-MM-dd'T'HH:mm'Z'", Calendar.getInstance().getTimeZone()) .build()); } From 87d5e6d24aabfb1d9a02136b951d72112b5e8af3 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 22 Oct 2019 14:58:19 -0500 Subject: [PATCH 5/8] The AggregationStore now adds graphql parameters for parameterized columns --- .../aggregation/AggregationDataStore.java | 11 +++++++++++ .../elide/datastores/aggregation/QueryEngine.java | 15 ++++++++++++++- .../queryengines/sql/SQLQueryEngine.java | 5 +++++ .../queryengines/sql/schema/SQLSchema.java | 8 +------- .../sql/schema/SQLTimeDimensionColumn.java | 1 + .../datastores/aggregation/schema/Schema.java | 13 ++++++++++++- .../queryengines/sql/SQLQueryEngineTest.java | 2 +- 7 files changed, 45 insertions(+), 10 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java index 99a18561ed..163b448beb 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java @@ -5,9 +5,12 @@ */ package com.yahoo.elide.datastores.aggregation; +import com.yahoo.elide.core.ArgumentType; import com.yahoo.elide.core.DataStore; import com.yahoo.elide.core.DataStoreTransaction; import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.datastores.aggregation.schema.Schema; +import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn; /** * DataStore that supports Aggregation. Uses {@link QueryEngine} to return results. @@ -28,6 +31,14 @@ public AggregationDataStore(QueryEngineFactory queryEngineFactory) { @Override public void populateEntityDictionary(EntityDictionary dictionary) { queryEngine = queryEngineFactory.buildQueryEngine(dictionary); + + /* Add 'grain' argument to each TimeDimensionColumn */ + for (Schema schema: queryEngine.getSchemas()) { + for (TimeDimensionColumn timeDim : schema.getTimeDimensions()) { + dictionary.addArgumentToAttributes(schema.getEntityClass(), timeDim.getName(), + new ArgumentType("grain", String.class)); + } + } } @Override diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java index dfabd93838..308c5280cc 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java @@ -10,6 +10,8 @@ import com.yahoo.elide.datastores.aggregation.query.Query; import com.yahoo.elide.datastores.aggregation.schema.Schema; +import java.util.List; + /** * A {@link QueryEngine} is an abstraction that an AggregationDataStore leverages to run analytic queries (OLAP style) * against an underlying persistence layer. @@ -68,5 +70,16 @@ public interface QueryEngine { * @param entityClass The class to map to a schema. * @return The schema that represents the provided entity. */ - Schema getSchema(Class entityClass); + default Schema getSchema(Class entityClass) { + return getSchemas().stream() + .filter(schema -> schema.getEntityClass().equals(entityClass)) + .findFirst() + .orElse(null); + } + + /** + * Returns all schemas managed by the engine. + * @return The schemas of the managed entities. + */ + List getSchemas(); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index 0b913e3fd5..25ade26785 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -83,6 +83,11 @@ public Schema getSchema(Class entityClass) { return schemas.get(entityClass); } + @Override + public List getSchemas() { + return schemas.values().stream().collect(Collectors.toList()); + } + @Override public Iterable executeQuery(Query query) { EntityManager entityManager = null; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java index 266340cbe7..d413c655b3 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java @@ -6,7 +6,6 @@ package com.yahoo.elide.datastores.aggregation.queryengines.sql.schema; -import com.yahoo.elide.core.ArgumentType; import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.Path; import com.yahoo.elide.core.filter.FilterPredicate; @@ -76,12 +75,7 @@ protected DimensionColumn constructDimension(String dimensionField, String columnName = getColumnName(entityClass, dimensionField); if (dim instanceof TimeDimensionColumn) { - - //Add grain to the GraphQL schema. - entityDictionary.addArgumentToAttributes(this.entityClass, dimensionField, - new ArgumentType("grain", String.class)); - - return new SQLTimeDimensionColumn((TimeDimensionColumn) dim, columnName, getAlias()); + return new SQLTimeDimensionColumn((TimeDimensionColumn) dim, columnName, getAlias()); } return new SQLDimensionColumn(dim, columnName, getAlias()); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java index 5368075776..bff2db3a0a 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java @@ -69,6 +69,7 @@ public String getColumnReference(TimeGrain requestedGrain) { TimeGrainDefinition definition = getSupportedGrains().stream() .filter(grainDef -> grainDef.grain().equals(requestedGrain)).findFirst().get(); + //TODO - We will likely migrate to a templating language when we support parameterized metrics. return String.format(definition.expression(), getColumnReference()); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java index 4965221582..3717c77985 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java @@ -53,7 +53,7 @@ public class Schema { @Getter - protected final Class entityClass; + public final Class entityClass; @Getter protected final Set metrics; protected final Map dimensions; @@ -124,6 +124,17 @@ public TimeDimensionColumn getTimeDimension(String dimensionName) { return null; } + /** + * Returns the complete list of time dimensions. + * @return the complete list of time dimensions. + */ + public List getTimeDimensions() { + return dimensions.values().stream() + .filter(dim -> dim instanceof TimeDimensionColumn) + .map(TimeDimensionColumn.class::cast) + .collect(Collectors.toList()); + } + /** * Finds the {@link Metric} by name. * diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java index 84489832d4..ea42244337 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngineTest.java @@ -677,7 +677,7 @@ public void testTotalScoreByMonth() { } /** - * Test filter by time dimension + * Test filter by time dimension. */ @Test public void testFilterByTemporalDimension() throws Exception { From fd63d4ab793dabab86ec9834b8c3bcb76950a26c Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 22 Oct 2019 15:03:59 -0500 Subject: [PATCH 6/8] Minor refactor --- .../queryengines/sql/SQLQueryEngine.java | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java index 25ade26785..aeffba11d1 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/SQLQueryEngine.java @@ -466,22 +466,7 @@ private String extractProjection(Query query) { * @return The SQL GROUP BY clause */ private String extractGroupBy(Query query) { - List dimensionStrings = query.getGroupDimensions().stream() - .map(requestedDim -> query.getSchema().getDimension(requestedDim.getName())) - .map((SQLDimensionColumn.class::cast)) - .map(SQLDimensionColumn::getColumnReference) - .collect(Collectors.toList()); - - dimensionStrings.addAll(query.getTimeDimensions().stream() - .map(requestedDim -> { - SQLTimeDimensionColumn timeDim = (SQLTimeDimensionColumn) - query.getSchema().getTimeDimension(requestedDim.getName()); - - return timeDim.getColumnReference(requestedDim.getTimeGrain().grain()); - }).collect(Collectors.toList())); - - return "GROUP BY " + dimensionStrings.stream() - .collect(Collectors.joining(",")); + return "GROUP BY " + extractDimensionProjections(query).stream().collect(Collectors.joining(",")); } /** @@ -501,8 +486,7 @@ private List extractDimensionProjections(Query query) { SQLTimeDimensionColumn timeDim = (SQLTimeDimensionColumn) query.getSchema().getTimeDimension(requestedDim.getName()); - return timeDim.getColumnReference(requestedDim.getTimeGrain().grain()) - + " AS " + timeDim.getColumnName(); + return timeDim.getColumnReference(requestedDim.getTimeGrain().grain()); }).collect(Collectors.toList())); return dimensionStrings; From a594c401843cc4271c4e93a058935b01fe18833a Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 22 Oct 2019 17:14:39 -0500 Subject: [PATCH 7/8] Inspection rework --- .../yahoo/elide/core/EntityDictionary.java | 2 +- .../pom.xml.releaseBackup | 198 ------------------ .../aggregation/AggregationDataStore.java | 2 +- .../AggregationDataStoreHelper.java | 70 +++---- .../queryengines/sql/schema/SQLSchema.java | 2 +- .../sql/schema/SQLTimeDimensionColumn.java | 8 +- .../datastores/aggregation/schema/Schema.java | 2 +- .../aggregation/example/SubCountry.java | 2 +- 8 files changed, 41 insertions(+), 245 deletions(-) delete mode 100644 elide-datastore/elide-datastore-aggregation/pom.xml.releaseBackup diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java index 2a7e2361d8..1e1785c94d 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java @@ -1429,7 +1429,7 @@ public void addArgumentsToAttributes(Class cls, String attributeName, Set cls, String attributeName, ArgumentType argument) { + public void addArgumentToAttribute(Class cls, String attributeName, ArgumentType argument) { this.addArgumentsToAttributes(cls, attributeName, Sets.newHashSet(argument)); } diff --git a/elide-datastore/elide-datastore-aggregation/pom.xml.releaseBackup b/elide-datastore/elide-datastore-aggregation/pom.xml.releaseBackup deleted file mode 100644 index ccf7c1fbd9..0000000000 --- a/elide-datastore/elide-datastore-aggregation/pom.xml.releaseBackup +++ /dev/null @@ -1,198 +0,0 @@ - - - - 4.0.0 - elide-datastore-aggregation - jar - Elide Data Store: Aggregation Data Store - Elide Data Store for Aggregation - https://github.com/yahoo/elide/tree/master/elide-datastore/elide-datastore-aggregation - - com.yahoo.elide - elide-datastore-parent-pom - 5.0.0-pr2-SNAPSHOT - - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - - Yahoo Inc. - https://github.com/yahoo - - - - - scm:git:ssh://git@github.com/yahoo/elide.git - https://github.com/yahoo/elide.git - HEAD - - - - 5.4.1 - - - - - com.yahoo.elide - elide-core - 5.0.0-pr2-SNAPSHOT - - - - com.yahoo.elide - elide-datastore-hibernate - 5.0.0-pr2-SNAPSHOT - - - - com.yahoo.elide - elide-graphql - 5.0.0-pr2-SNAPSHOT - - - - org.projectlombok - lombok - - - - - org.hibernate.javax.persistence - hibernate-jpa-2.1-api - 1.0.0.Final - - - - - org.hibernate - hibernate-entitymanager - ${hibernate5.version} - - - - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - - org.junit.jupiter - junit-jupiter-params - ${junit.version} - test - - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - - - com.h2database - h2 - 1.4.197 - test - - - - - org.mockito - mockito-core - test - - - - - com.yahoo.elide - elide-integration-tests - 5.0.0-pr2-SNAPSHOT - test - test-jar - - - - - org.eclipse.jetty - jetty-servlet - test - - - - org.eclipse.jetty - jetty-webapp - test - - - - - org.glassfish.jersey.containers - jersey-container-servlet-core - 2.29.1 - test - - - - org.glassfish.jersey.inject - jersey-hk2 - test - - - - org.glassfish.jersey.containers - jersey-container-servlet - test - - - - io.rest-assured - rest-assured - 4.0.0 - test - - - - org.glassfish.hk2 - hk2-api - 2.5.0 - test - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - org.apache.maven.plugins - maven-dependency-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.apache.maven.plugins - maven-checkstyle-plugin - - - - diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java index 163b448beb..18f15155d6 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java @@ -35,7 +35,7 @@ public void populateEntityDictionary(EntityDictionary dictionary) { /* Add 'grain' argument to each TimeDimensionColumn */ for (Schema schema: queryEngine.getSchemas()) { for (TimeDimensionColumn timeDim : schema.getTimeDimensions()) { - dictionary.addArgumentToAttributes(schema.getEntityClass(), timeDim.getName(), + dictionary.addArgumentToAttribute(schema.getEntityClass(), timeDim.getName(), new ArgumentType("grain", String.class)); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java index ba4172c604..d9bab6f9f8 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreHelper.java @@ -102,7 +102,6 @@ private void splitFilters() { } } - //TODO - Add tests in the next PR. /** * Gets time dimensions based on relationships and attributes from {@link EntityProjection}. * @@ -110,47 +109,46 @@ private void splitFilters() { * @throws InvalidOperationException Thrown if a requested time grain is not supported. */ private Set resolveTimeDimensions() { + return entityProjection.getAttributes().stream() + .filter(attribute -> schema.getTimeDimension(attribute.getName()) != null) + .map(attribute -> { + TimeDimensionColumn timeDim = schema.getTimeDimension(attribute.getName()); - Set timeDims = new LinkedHashSet<>(); - //Only attributes can be time dimensions - entityProjection.getAttributes().stream().forEach((attribute -> { - TimeDimensionColumn timeDim = schema.getTimeDimension(attribute.getName()); - if (timeDim == null) { - return; - } - - Argument timeGrainArgument = attribute.getArguments().stream() - .filter(attr -> attr.getName().equals("grain")) - .findAny() - .orElse(null); - - TimeGrainDefinition requestedGrainDefinition; - if (timeGrainArgument == null) { + Argument timeGrainArgument = attribute.getArguments().stream() + .filter(attr -> attr.getName().equals("grain")) + .findAny() + .orElse(null); - //The first grain is the default. - requestedGrainDefinition = timeDim.getSupportedGrains().iterator().next(); - } else { - String requestedGrainName = timeGrainArgument.getValue().toString().toUpperCase(Locale.ENGLISH); + TimeGrainDefinition requestedGrainDefinition; + if (timeGrainArgument == null) { - TimeGrain requestedGrain; - try { - requestedGrain = TimeGrain.valueOf(requestedGrainName); - } catch (IllegalArgumentException e) { - throw new InvalidOperationException(String.format("Invalid grain %s", requestedGrainName)); - } + //The first grain is the default. + requestedGrainDefinition = timeDim.getSupportedGrains().stream() + .findFirst() + .orElseThrow(() -> new IllegalStateException( + String.format("Requested default grain, no grain defined on %s", + attribute.getName()))); + } else { + String requestedGrainName = timeGrainArgument.getValue().toString().toUpperCase(Locale.ENGLISH); - requestedGrainDefinition = timeDim.getSupportedGrains().stream() - .filter(supportedGrainDef -> supportedGrainDef.grain().equals(requestedGrain)) - .findAny() - .orElseThrow(() -> new InvalidOperationException( - String.format("Requested grain %s, not supported on %s", - requestedGrainName, attribute.getName()))); - } + TimeGrain requestedGrain; + try { + requestedGrain = TimeGrain.valueOf(requestedGrainName); + } catch (IllegalArgumentException e) { + throw new InvalidOperationException(String.format("Invalid grain %s", requestedGrainName)); + } - timeDims.add(timeDim.toProjectedDimension(requestedGrainDefinition)); - })); + requestedGrainDefinition = timeDim.getSupportedGrains().stream() + .filter(supportedGrainDef -> supportedGrainDef.grain().equals(requestedGrain)) + .findAny() + .orElseThrow(() -> new InvalidOperationException( + String.format("Requested grain %s, not supported on %s", + requestedGrainName, attribute.getName()))); + } - return timeDims; + return timeDim.toProjectedDimension(requestedGrainDefinition); + }) + .collect(Collectors.toCollection(LinkedHashSet::new)); } /** diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java index d413c655b3..b1aa5270ad 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLSchema.java @@ -75,7 +75,7 @@ protected DimensionColumn constructDimension(String dimensionField, String columnName = getColumnName(entityClass, dimensionField); if (dim instanceof TimeDimensionColumn) { - return new SQLTimeDimensionColumn((TimeDimensionColumn) dim, columnName, getAlias()); + return new SQLTimeDimensionColumn((TimeDimensionColumn) dim, columnName, getAlias()); } return new SQLDimensionColumn(dim, columnName, getAlias()); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java index bff2db3a0a..438ff9a7ad 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java @@ -11,8 +11,6 @@ import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn; import com.yahoo.elide.datastores.aggregation.time.TimeGrain; -import com.google.common.base.Preconditions; - import java.util.Set; import java.util.TimeZone; @@ -63,11 +61,9 @@ public Set getSupportedGrains() { * @return Something like "table_alias.column_name" */ public String getColumnReference(TimeGrain requestedGrain) { - Preconditions.checkArgument(getSupportedGrains().stream() - .anyMatch((grainDef -> grainDef.grain().equals(requestedGrain)))); - TimeGrainDefinition definition = getSupportedGrains().stream() - .filter(grainDef -> grainDef.grain().equals(requestedGrain)).findFirst().get(); + .filter(grainDef -> grainDef.grain().equals(requestedGrain)).findFirst() + .orElseThrow(() -> new IllegalStateException("Requested time grain not supported.")); //TODO - We will likely migrate to a templating language when we support parameterized metrics. return String.format(definition.expression(), getColumnReference()); diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java index 3717c77985..d1641bffa0 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java @@ -53,7 +53,7 @@ public class Schema { @Getter - public final Class entityClass; + protected final Class entityClass; @Getter protected final Set metrics; protected final Map dimensions; diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/SubCountry.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/SubCountry.java index a009fd3911..f6c14c611e 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/SubCountry.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/SubCountry.java @@ -18,7 +18,7 @@ import javax.persistence.Id; /** - * A root level entity for testing AggregationDataStore with @Subselect annotation + * A root level entity for testing AggregationDataStore with @Subselect annotation. */ @Data @Entity From 740ba3833a66e78e4ae22203c2e89fe65e2b113a Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Wed, 30 Oct 2019 18:01:56 -0500 Subject: [PATCH 8/8] Minor fix --- .../queryengines/sql/schema/SQLTimeDimensionColumn.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java index 438ff9a7ad..375cf0874c 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/queryengines/sql/schema/SQLTimeDimensionColumn.java @@ -62,7 +62,8 @@ public Set getSupportedGrains() { */ public String getColumnReference(TimeGrain requestedGrain) { TimeGrainDefinition definition = getSupportedGrains().stream() - .filter(grainDef -> grainDef.grain().equals(requestedGrain)).findFirst() + .filter(grainDef -> grainDef.grain().equals(requestedGrain)) + .findFirst() .orElseThrow(() -> new IllegalStateException("Requested time grain not supported.")); //TODO - We will likely migrate to a templating language when we support parameterized metrics.