Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add time grain to GraphQL schema #1042

Merged
merged 8 commits into from
Oct 31, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;

import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -1422,6 +1423,16 @@ public void addArgumentsToAttributes(Class<?> cls, String attributeName, Set<Arg
getEntityBinding(cls).addArgumentsToAttribute(attributeName, arguments);
}

/**
* Add a single argument to the attribute
* @param cls The entity
* @param attributeName attribute name to which argument has to be added
* @param argument A single argument
*/
public void addArgumentToAttribute(Class<?> cls, String attributeName, ArgumentType argument) {
this.addArgumentsToAttributes(cls, attributeName, Sets.newHashSet(argument));
}

/**
* Returns the Collection of all attributes of an argument.
* @param cls The entity
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1104,10 +1104,20 @@ public RelationshipType getRelationshipType(String relation) {
* @param attr Attribute name
* @return Object value for attribute
*/
@Deprecated
public Object getAttribute(String attr) {
return this.getValueChecked(attr);
}

/**
* Get the value for a particular attribute (i.e. non-relational field)
* @param attr the Attribute
* @return Object value for attribute
*/
public Object getAttribute(Attribute attr) {
return this.getValueChecked(attr);
}

/**
* Wrapped Entity bean.
*
Expand Down Expand Up @@ -1363,13 +1373,26 @@ protected void nullValue(String fieldName, PersistentResource oldValue) {
* @param fieldName the field name
* @return value value
*/
@Deprecated
protected Object getValueChecked(String fieldName) {
requestScope.publishLifecycleEvent(this, CRUDEvent.CRUDAction.READ);
requestScope.publishLifecycleEvent(this, fieldName, CRUDEvent.CRUDAction.READ, Optional.empty());
checkFieldAwareDeferPermissions(ReadPermission.class, fieldName, (Object) null, (Object) null);
return getValue(getObject(), fieldName, requestScope);
}

/**
* Gets a value from an entity and checks read permissions.
* @param attribute the attribute to fetch.
* @return value value
*/
protected Object getValueChecked(Attribute attribute) {
requestScope.publishLifecycleEvent(this, CRUDEvent.CRUDAction.READ);
requestScope.publishLifecycleEvent(this, attribute.getName(), CRUDEvent.CRUDAction.READ, Optional.empty());
checkFieldAwareDeferPermissions(ReadPermission.class, attribute.getName(), (Object) null, (Object) null);
return transaction.getAttribute(getObject(), attribute, requestScope);
}

/**
* Retrieve an object without checking read permissions (i.e. value is used internally and not sent to others)
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ public final boolean ok(User user) {
throw new UnsupportedOperationException();
}


/**
* The filter expression is evaluated in memory if it cannot be pushed to the data store by elide for any reason.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
*/
package com.yahoo.elide.datastores.aggregation;

import com.yahoo.elide.core.ArgumentType;
import com.yahoo.elide.core.DataStore;
import com.yahoo.elide.core.DataStoreTransaction;
import com.yahoo.elide.core.EntityDictionary;
import com.yahoo.elide.datastores.aggregation.schema.Schema;
import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn;

/**
* DataStore that supports Aggregation. Uses {@link QueryEngine} to return results.
Expand All @@ -28,6 +31,14 @@ public AggregationDataStore(QueryEngineFactory queryEngineFactory) {
@Override
public void populateEntityDictionary(EntityDictionary dictionary) {
queryEngine = queryEngineFactory.buildQueryEngine(dictionary);

/* Add 'grain' argument to each TimeDimensionColumn */
for (Schema schema: queryEngine.getSchemas()) {
for (TimeDimensionColumn timeDim : schema.getTimeDimensions()) {
dictionary.addArgumentToAttribute(schema.getEntityClass(), timeDim.getName(),
new ArgumentType("grain", String.class));
}
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
Expand Down Expand Up @@ -101,7 +102,6 @@ private void splitFilters() {
}
}

//TODO - Add tests in the next PR.
/**
* Gets time dimensions based on relationships and attributes from {@link EntityProjection}.
*
Expand Down Expand Up @@ -129,7 +129,7 @@ private Set<TimeDimensionProjection> resolveTimeDimensions() {
String.format("Requested default grain, no grain defined on %s",
attribute.getName())));
} else {
String requestedGrainName = timeGrainArgument.getValue().toString();
String requestedGrainName = timeGrainArgument.getValue().toString().toUpperCase(Locale.ENGLISH);

TimeGrain requestedGrain;
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import com.yahoo.elide.datastores.aggregation.query.Query;
import com.yahoo.elide.datastores.aggregation.schema.Schema;

import java.util.List;

/**
* A {@link QueryEngine} is an abstraction that an AggregationDataStore leverages to run analytic queries (OLAP style)
* against an underlying persistence layer.
Expand Down Expand Up @@ -68,5 +70,16 @@ public interface QueryEngine {
* @param entityClass The class to map to a schema.
* @return The schema that represents the provided entity.
*/
Schema getSchema(Class<?> entityClass);
default Schema getSchema(Class<?> entityClass) {
return getSchemas().stream()
.filter(schema -> schema.getEntityClass().equals(entityClass))
.findFirst()
.orElse(null);
}

/**
* Returns all schemas managed by the engine.
* @return The schemas of the managed entities.
*/
List<Schema> getSchemas();
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.JoinTo;
import com.yahoo.elide.datastores.aggregation.queryengines.sql.schema.SQLDimensionColumn;
import com.yahoo.elide.datastores.aggregation.queryengines.sql.schema.SQLSchema;
import com.yahoo.elide.datastores.aggregation.queryengines.sql.schema.SQLTimeDimensionColumn;
import com.yahoo.elide.datastores.aggregation.schema.Schema;
import com.yahoo.elide.datastores.aggregation.schema.dimension.DimensionColumn;
import com.yahoo.elide.datastores.aggregation.schema.metric.Aggregation;
Expand Down Expand Up @@ -82,6 +83,11 @@ public Schema getSchema(Class<?> entityClass) {
return schemas.get(entityClass);
}

@Override
public List<Schema> getSchemas() {
return schemas.values().stream().collect(Collectors.toList());
}

@Override
public Iterable<Object> executeQuery(Query query) {
EntityManager entityManager = null;
Expand Down Expand Up @@ -460,8 +466,7 @@ private String extractProjection(Query query) {
* @return The SQL GROUP BY clause
*/
private String extractGroupBy(Query query) {
return "GROUP BY " + extractDimensionProjections(query).stream()
.collect(Collectors.joining(","));
return "GROUP BY " + extractDimensionProjections(query).stream().collect(Collectors.joining(","));
}

/**
Expand All @@ -470,11 +475,21 @@ private String extractGroupBy(Query query) {
* @return
*/
private List<String> extractDimensionProjections(Query query) {
return query.getDimensions().stream()
List<String> dimensionStrings = query.getGroupDimensions().stream()
.map(requestedDim -> query.getSchema().getDimension(requestedDim.getName()))
.map((SQLDimensionColumn.class::cast))
.map(SQLDimensionColumn::getColumnReference)
.collect(Collectors.toList());

dimensionStrings.addAll(query.getTimeDimensions().stream()
.map(requestedDim -> {
SQLTimeDimensionColumn timeDim = (SQLTimeDimensionColumn)
query.getSchema().getTimeDimension(requestedDim.getName());

return timeDim.getColumnReference(requestedDim.getTimeGrain().grain());
}).collect(Collectors.toList()));

return dimensionStrings;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import com.yahoo.elide.datastores.aggregation.schema.metric.Metric;

import org.hibernate.annotations.Subselect;

import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.ToString;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.yahoo.elide.core.Path;
import com.yahoo.elide.datastores.aggregation.annotation.TimeGrainDefinition;
import com.yahoo.elide.datastores.aggregation.schema.dimension.TimeDimensionColumn;
import com.yahoo.elide.datastores.aggregation.time.TimeGrain;

import java.util.Set;
import java.util.TimeZone;
Expand All @@ -20,21 +21,23 @@
public class SQLTimeDimensionColumn extends SQLDimensionColumn implements TimeDimensionColumn {
/**
* Constructor.
* @param dimension a wrapped dimension.
*
* @param dimension a wrapped dimension.
* @param columnAlias The column alias in SQL to refer to this dimension.
* @param tableAlias The table alias in SQL where this dimension lives.
* @param tableAlias The table alias in SQL where this dimension lives.
*/
public SQLTimeDimensionColumn(TimeDimensionColumn dimension, String columnAlias, String tableAlias) {
super(dimension, columnAlias, tableAlias);
}

/**
* Constructor.
* @param dimension a wrapped dimension.
*
* @param dimension a wrapped dimension.
* @param columnAlias The column alias in SQL to refer to this dimension.
* @param tableAlias The table alias in SQL where this dimension lives.
* @param joinPath A '.' separated path through the entity relationship graph that describes
* how to join the time dimension into the current AnalyticView.
* @param tableAlias The table alias in SQL where this dimension lives.
* @param joinPath A '.' separated path through the entity relationship graph that describes
* how to join the time dimension into the current AnalyticView.
*/
public SQLTimeDimensionColumn(TimeDimensionColumn dimension, String columnAlias, String tableAlias, Path joinPath) {
super(dimension, columnAlias, tableAlias, joinPath);
Expand All @@ -50,4 +53,20 @@ public TimeZone getTimeZone() {
public Set<TimeGrainDefinition> getSupportedGrains() {
return ((TimeDimensionColumn) wrapped).getSupportedGrains();
}

/**
* Returns a String that identifies this dimension in a SQL query.
* @param requestedGrain The requested time grain.
*
* @return Something like "table_alias.column_name"
*/
public String getColumnReference(TimeGrain requestedGrain) {
TimeGrainDefinition definition = getSupportedGrains().stream()
.filter(grainDef -> grainDef.grain().equals(requestedGrain))
.findFirst()
.orElseThrow(() -> new IllegalStateException("Requested time grain not supported."));

//TODO - We will likely migrate to a templating language when we support parameterized metrics.
return String.format(definition.expression(), getColumnReference());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,17 @@ public TimeDimensionColumn getTimeDimension(String dimensionName) {
return null;
}

/**
* Returns the complete list of time dimensions.
* @return the complete list of time dimensions.
*/
public List<TimeDimensionColumn> getTimeDimensions() {
return dimensions.values().stream()
.filter(dim -> dim instanceof TimeDimensionColumn)
.map(TimeDimensionColumn.class::cast)
.collect(Collectors.toList());
}

/**
* Finds the {@link Metric} by name.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,36 @@ public void aggregationComputedMetricTest() throws Exception {
runQueryWithExpectedResult(graphQLRequest, expected);
}

@Test
public void timeGrainAggregationTest() throws Exception {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we can add more records into the test data and do both daily and monthly aggregation.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The point of the IT tests is more to test the integration of the API with the backend. If we need more tests - they belong in the SQLQueryEngineTest. I did both daily and monthly tests there. Do you feel like we should add more?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I personally prefer to put more tests in IntegrationTest to make sure we are generating the expected Query object.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What tests do you suggest adding to more fully test the Query object?

String graphQLRequest = document(
selection(
field(
"playerStats",
selections(
field("highScore"),
field("recordedDate", arguments(
argument("grain", "\"month\"")
))
)
)
)
).toQuery();

String expected = document(
selections(
field(
"playerStats",
selections(
field("highScore", 2412),
field("recordedDate", "2019-07-01T00:00Z")
)
)
)).toResponse();

runQueryWithExpectedResult(graphQLRequest, expected);
}

private void create(String query, Map<String, Object> variables) throws IOException {
runQuery(toJsonQuery(query, variables));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@
@FromTable(name = "playerStats")
public class PlayerStats {

public static final String DAY_FORMAT = "PARSEDATETIME(FORMATDATETIME(%s, 'yyyy-MM-dd'), 'yyyy-MM-dd')";
public static final String MONTH_FORMAT = "PARSEDATETIME(FORMATDATETIME(%s, 'yyyy-MM-01'), 'yyyy-MM-dd')";

/**
* PK.
*/
Expand Down Expand Up @@ -161,7 +164,10 @@ public void setPlayer(final Player player) {
* {@link EntityDimensionTest#testCardinalityScan()}.
* @return the date of the player session.
*/
@Temporal(grains = { @TimeGrainDefinition(grain = TimeGrain.DAY, expression = "") }, timeZone = "UTC")
@Temporal(grains = {
@TimeGrainDefinition(grain = TimeGrain.DAY, expression = DAY_FORMAT),
@TimeGrainDefinition(grain = TimeGrain.MONTH, expression = MONTH_FORMAT)
}, timeZone = "UTC")
public Date getRecordedDate() {
return recordedDate;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import javax.persistence.Id;

/**
* A root level entity for testing AggregationDataStore with @Subselect annotation
* A root level entity for testing AggregationDataStore with @Subselect annotation.
*/
@Data
@Entity
Expand Down
Loading