diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java index 33a91241b9..73b4e9262d 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java @@ -8,9 +8,6 @@ import com.yahoo.elide.core.DataStore; import com.yahoo.elide.core.DataStoreTransaction; import com.yahoo.elide.core.EntityDictionary; -import com.yahoo.elide.core.RequestScope; -import com.yahoo.elide.datastores.aggregation.schema.Schema; -import com.yahoo.elide.request.EntityProjection; /** * DataStore that supports Aggregation. Uses {@link QueryEngine} to return results. @@ -30,10 +27,4 @@ public AggregationDataStore(QueryEngine queryEngine) { public DataStoreTransaction beginTransaction() { return new AggregationDataStoreTransaction(queryEngine); } - - public static Query buildQuery(EntityProjection entityProjection, RequestScope scope) { - Schema schema = new Schema(entityProjection.getType(), scope.getDictionary()); - AggregationDataStoreHelper agHelper = new AggregationDataStoreHelper(schema, entityProjection); - return agHelper.getQuery(); - } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java index c27d42421d..7c31d041b0 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreTransaction.java @@ -7,8 +7,11 @@ import com.yahoo.elide.core.DataStoreTransaction; import com.yahoo.elide.core.RequestScope; +import com.yahoo.elide.datastores.aggregation.schema.Schema; import com.yahoo.elide.request.EntityProjection; +import com.google.common.annotations.VisibleForTesting; + import java.io.IOException; /** @@ -48,7 +51,7 @@ public void createObject(Object entity, RequestScope scope) { @Override public Iterable loadObjects(EntityProjection entityProjection, RequestScope scope) { - Query query = AggregationDataStore.buildQuery(entityProjection, scope); + Query query = buildQuery(entityProjection, scope); return queryEngine.executeQuery(query); } @@ -56,4 +59,12 @@ public Iterable loadObjects(EntityProjection entityProjection, RequestSc public void close() throws IOException { } + + @VisibleForTesting + Query buildQuery(EntityProjection entityProjection, RequestScope scope) { + Schema schema = queryEngine.getSchema(entityProjection.getType()); + + AggregationDataStoreHelper agHelper = new AggregationDataStoreHelper(schema, entityProjection); + return agHelper.getQuery(); + } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java index 4e85c64bce..cdcdac4db5 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/QueryEngine.java @@ -7,6 +7,7 @@ import com.yahoo.elide.core.DataStore; import com.yahoo.elide.core.DataStoreTransaction; +import com.yahoo.elide.datastores.aggregation.schema.Schema; /** * A {@link QueryEngine} is an abstraction that an AggregationDataStore leverages to run analytic queries (OLAP style) @@ -49,7 +50,6 @@ *

* This is a {@link java.util.function functional interface} whose functional method is {@link #executeQuery(Query)}. */ -@FunctionalInterface public interface QueryEngine { /** @@ -61,4 +61,11 @@ public interface QueryEngine { * @return query results */ Iterable executeQuery(Query query); + + /** + * Returns the schema for a given entity class. + * @param entityClass The class to map to a schema. + * @return The schema that represents the provided entity. + */ + Schema getSchema(Class entityClass); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java index 4262ad93d1..18117c0d38 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java @@ -71,8 +71,12 @@ protected Map getRelationshipValues( .getResultList(); return loaded.stream() - .map(obj -> new AbstractMap.SimpleImmutableEntry<>(CoerceUtil.coerce((Object) getEntityDictionary() - .getId(obj), getEntityDictionary().getIdType(relationshipType)), obj)) + .map(obj -> new AbstractMap.SimpleImmutableEntry<>( + CoerceUtil.coerce( + (Object) getEntityDictionary().getId(obj), + getEntityDictionary().getIdType(relationshipType) + ), + obj)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 7055f48b1b..18d3465918 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -21,6 +21,8 @@ import com.yahoo.elide.datastores.aggregation.dimension.Dimension; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromSubquery; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromTable; +import com.yahoo.elide.datastores.aggregation.engine.annotation.JoinTo; +import com.yahoo.elide.datastores.aggregation.engine.schema.SQLDimension; import com.yahoo.elide.datastores.aggregation.engine.schema.SQLSchema; import com.yahoo.elide.datastores.aggregation.metric.Aggregation; import com.yahoo.elide.datastores.aggregation.metric.Metric; @@ -41,9 +43,8 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import javax.persistence.Column; import javax.persistence.EntityManager; -import javax.persistence.JoinColumn; +import javax.persistence.EntityManagerFactory; import javax.persistence.Table; /** @@ -52,14 +53,14 @@ @Slf4j public class SQLQueryEngine implements QueryEngine { - private EntityManager entityManager; + private EntityManagerFactory emf; private EntityDictionary dictionary; @Getter private Map, SQLSchema> schemas; - public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { - this.entityManager = entityManager; + public SQLQueryEngine(EntityManagerFactory emf, EntityDictionary dictionary) { + this.emf = emf; this.dictionary = dictionary; // Construct the list of schemas that will be managed by this query engine. @@ -75,49 +76,63 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) )); } + @Override + public Schema getSchema(Class entityClass) { + return schemas.get(entityClass); + } + @Override public Iterable executeQuery(Query query) { - SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); + EntityManager entityManager = null; + try { + entityManager = emf.createEntityManager(); + SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); - //Make sure we actually manage this schema. - Preconditions.checkNotNull(schema); + //Make sure we actually manage this schema. + Preconditions.checkNotNull(schema); - //Translate the query into SQL. - SQLQuery sql = toSQL(query, schema); + //Translate the query into SQL. + SQLQuery sql = toSQL(query, schema); + log.debug("SQL Query: " + sql); - javax.persistence.Query jpaQuery = entityManager.createNativeQuery(sql.toString()); + javax.persistence.Query jpaQuery = entityManager.createNativeQuery(sql.toString()); - Pagination pagination = query.getPagination(); - if (pagination != null) { - jpaQuery.setFirstResult(pagination.getOffset()); - jpaQuery.setMaxResults(pagination.getLimit()); + Pagination pagination = query.getPagination(); + if (pagination != null) { + jpaQuery.setFirstResult(pagination.getOffset()); + jpaQuery.setMaxResults(pagination.getLimit()); - if (pagination.isGenerateTotals()) { + if (pagination.isGenerateTotals()) { - SQLQuery paginationSQL = toPageTotalSQL(sql); - javax.persistence.Query pageTotalQuery = - entityManager.createNativeQuery(paginationSQL.toString()); + SQLQuery paginationSQL = toPageTotalSQL(sql); + javax.persistence.Query pageTotalQuery = + entityManager.createNativeQuery(paginationSQL.toString()); - //Supply the query parameters to the query - supplyFilterQueryParameters(query, pageTotalQuery); + //Supply the query parameters to the query + supplyFilterQueryParameters(query, pageTotalQuery); - //Run the Pagination query and log the time spent. - long total = new TimedFunction<>( - () -> CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class), - "Running Query: " + paginationSQL - ).get(); + //Run the Pagination query and log the time spent. + long total = new TimedFunction<>( + () -> CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class), + "Running Query: " + paginationSQL + ).get(); - pagination.setPageTotals(total); + pagination.setPageTotals(total); + } } - } - //Supply the query parameters to the query - supplyFilterQueryParameters(query, jpaQuery); + //Supply the query parameters to the query + supplyFilterQueryParameters(query, jpaQuery); - //Run the primary query and log the time spent. - List results = new TimedFunction<>(() -> jpaQuery.getResultList(), "Running Query: " + sql).get(); + //Run the primary query and log the time spent. + List results = new TimedFunction<>(() -> jpaQuery.getResultList(), "Running Query: " + sql).get(); - return new SQLEntityHydrator(results, query, dictionary, entityManager).hydrate(); + return new SQLEntityHydrator(results, query, dictionary, entityManager).hydrate(); + } finally { + if (entityManager != null) { + entityManager.close(); + } + } } /** @@ -151,6 +166,8 @@ protected SQLQuery toSQL(Query query, SQLSchema schema) { if (! query.getDimensions().isEmpty()) { builder.groupByClause(extractGroupBy(query)); + + joinPredicates.addAll(extractPathElements(query.getDimensions())); } if (query.getSorting() != null) { @@ -185,6 +202,21 @@ private String translateFilterExpression(FilterExpression expression, return filterVisitor.apply(expression, columnGenerator); } + /** + * Given the set of group by dimensions, extract any entity relationship traversals that require joins. + * @param groupByDims The list of dimensions we are grouping on. + * @return A set of path elements that capture a relationship traversal. + */ + private Set extractPathElements(Set groupByDims) { + return groupByDims.stream() + .map((SQLDimension.class::cast)) + .filter((dim) -> dim.getJoinPath() != null) + .map(SQLDimension::getJoinPath) + .map((path) -> extractPathElements(path)) + .flatMap((elements) -> elements.stream()) + .collect(Collectors.toSet()); + } + /** * Given a filter expression, extracts any entity relationship traversals that require joins. * @param expression The filter expression @@ -194,9 +226,42 @@ private Set extractPathElements(FilterExpression expression) { Collection predicates = expression.accept(new PredicateExtractionVisitor()); return predicates.stream() - .filter(predicate -> predicate.getPath().getPathElements().size() > 1) .map(FilterPredicate::getPath) - .flatMap((path) -> path.getPathElements().stream()) + .map(this::expandJoinToPath) + .filter(path -> path.getPathElements().size() > 1) + .map((path) -> extractPathElements(path)) + .flatMap((elements) -> elements.stream()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + /** + * Expands a predicate path (from a sort or filter predicate) to the path contained in + * the JoinTo annotation. If no JoinTo annotation is present, the original path is returned. + * @param path The path to expand. + * @return The expanded path. + */ + private Path expandJoinToPath(Path path) { + List pathElements = path.getPathElements(); + Path.PathElement pathElement = pathElements.get(0); + + Class type = pathElement.getType(); + String fieldName = pathElement.getFieldName(); + JoinTo joinTo = dictionary.getAttributeOrRelationAnnotation(type, JoinTo.class, fieldName); + + if (joinTo == null) { + return path; + } + + return new Path(pathElement.getType(), dictionary, joinTo.path()); + } + + /** + * Given a path , extracts any entity relationship traversals that require joins. + * @param path The path + * @return A set of path elements that capture a relationship traversal. + */ + private Set extractPathElements(Path path) { + return path.getPathElements().stream() .filter((p) -> dictionary.isRelation(p.getType(), p.getFieldName())) .collect(Collectors.toCollection(LinkedHashSet::new)); } @@ -208,6 +273,10 @@ private Set extractPathElements(FilterExpression expression) { * @return A SQL expression */ private String extractJoin(Path.PathElement pathElement) { + //TODO - support composite join keys. + //TODO - support joins where either side owns the relationship. + //TODO - Support INNER and RIGHT joins. + //TODO - Support toMany joins. String relationshipName = pathElement.getFieldName(); Class relationshipClass = pathElement.getFieldType(); String relationshipAlias = FilterPredicate.getTypeAlias(relationshipClass); @@ -243,8 +312,9 @@ private Set extractPathElements(Map s return sortClauses.entrySet().stream() .map(Map.Entry::getKey) - .flatMap((path) -> path.getPathElements().stream()) - .filter((element) -> dictionary.isRelation(element.getType(), element.getFieldName())) + .map(this::expandJoinToPath) + .map((path) -> extractPathElements(path)) + .flatMap((elements) -> elements.stream()) .collect(Collectors.toCollection(LinkedHashSet::new)); } @@ -259,9 +329,12 @@ private String extractOrderBy(Class entityClass, Map return ""; } + //TODO - Ensure that order by columns are also present in the group by. + return " ORDER BY " + sortClauses.entrySet().stream() .map((entry) -> { Path path = entry.getKey(); + path = expandJoinToPath(path); Sorting.SortOrder order = entry.getValue(); Path.PathElement last = path.lastElement().get(); @@ -308,20 +381,7 @@ private void supplyFilterQueryParameters(Query query, * @return */ private String getColumnName(Class entityClass, String fieldName) { - Column[] column = dictionary.getAttributeOrRelationAnnotations(entityClass, Column.class, fieldName); - - JoinColumn[] joinColumn = dictionary.getAttributeOrRelationAnnotations(entityClass, - JoinColumn.class, fieldName); - - if (column == null || column.length == 0) { - if (joinColumn == null || joinColumn.length == 0) { - return fieldName; - } else { - return joinColumn[0].name(); - } - } else { - return column[0].name(); - } + return SQLSchema.getColumnName(dictionary, entityClass, fieldName); } /** @@ -334,8 +394,8 @@ private SQLQuery toPageTotalSQL(SQLQuery sql) { Query clientQuery = sql.getClientQuery(); String groupByDimensions = clientQuery.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name)) + .map((SQLDimension.class::cast)) + .map(SQLDimension::getColumnName) .collect(Collectors.joining(",")); String projectionClause = String.format("SELECT COUNT(DISTINCT(%s))", groupByDimensions); @@ -364,8 +424,8 @@ private String extractProjection(Query query) { .collect(Collectors.toList()); List dimensionProjections = query.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(query.getSchema().getEntityClass(), name)) + .map((SQLDimension.class::cast)) + .map(SQLDimension::getColumnReference) .collect(Collectors.toList()); String projectionClause = metricProjections.stream() @@ -373,7 +433,6 @@ private String extractProjection(Query query) { if (!dimensionProjections.isEmpty()) { projectionClause = projectionClause + "," + dimensionProjections.stream() - .map((name) -> query.getSchema().getAlias() + "." + name) .collect(Collectors.joining(",")); } @@ -387,12 +446,11 @@ private String extractProjection(Query query) { */ private String extractGroupBy(Query query) { List dimensionProjections = query.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(query.getSchema().getEntityClass(), name)) + .map((SQLDimension.class::cast)) + .map(SQLDimension::getColumnReference) .collect(Collectors.toList()); return "GROUP BY " + dimensionProjections.stream() - .map((name) -> query.getSchema().getAlias() + "." + name) .collect(Collectors.joining(",")); } @@ -402,10 +460,26 @@ private String extractGroupBy(Query query) { * @return A SQL fragment that references a database column */ private String generateWhereClauseColumnReference(FilterPredicate predicate) { - Path.PathElement last = predicate.getPath().lastElement().get(); + return generateWhereClauseColumnReference(predicate.getPath()); + } + + /** + * Converts a filter predicate path into a SQL WHERE clause column reference. + * @param path The predicate path to convert + * @return A SQL fragment that references a database column + */ + private String generateWhereClauseColumnReference(Path path) { + Path.PathElement last = path.lastElement().get(); Class lastClass = last.getType(); + String fieldName = last.getFieldName(); + + JoinTo joinTo = dictionary.getAttributeOrRelationAnnotation(lastClass, JoinTo.class, fieldName); - return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); + if (joinTo == null) { + return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); + } else { + return generateWhereClauseColumnReference(new Path(lastClass, dictionary, joinTo.path())); + } } /** diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/annotation/JoinTo.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/annotation/JoinTo.java new file mode 100644 index 0000000000..02b9b22be1 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/annotation/JoinTo.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation.engine.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that the annotated entity field is derived from a join to another table. + */ +@Documented +@Target({ElementType.FIELD, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface JoinTo { + + /** + * Dot separated path through the entity relationship graph to an attribute. + * If the current entity is author, then a path would be "book.publisher.name". + * @return + */ + String path(); +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLDimension.java new file mode 100644 index 0000000000..84acaf271a --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLDimension.java @@ -0,0 +1,107 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.datastores.aggregation.engine.schema; + +import com.yahoo.elide.core.Path; +import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; +import com.yahoo.elide.datastores.aggregation.dimension.Dimension; +import com.yahoo.elide.datastores.aggregation.dimension.DimensionType; + +/** + * A dimension but supporting extra metadata needed to generate SQL. + */ +public class SQLDimension implements Dimension { + + private final Dimension wrapped; + private final String columnAlias; + private final String tableAlias; + private final Path joinPath; + + /** + * Constructor + * @param dimension a wrapped dimension. + * @param columnAlias The column alias in SQL to refer to this dimension. + * @param tableAlias The table alias in SQL where this dimension lives. + */ + public SQLDimension(Dimension dimension, String columnAlias, String tableAlias) { + this(dimension, columnAlias, tableAlias, null); + } + + /** + * Constructor + * @param dimension a wrapped dimension. + * @param columnAlias The column alias in SQL to refer to this dimension. + * @param tableAlias The table alias in SQL where this dimension lives. + * @param joinPath A '.' separated path through the entity relationship graph that describes + * how to join the time dimension into the current AnalyticView. + */ + public SQLDimension(Dimension dimension, String columnAlias, String tableAlias, Path joinPath) { + this.wrapped = dimension; + this.columnAlias = columnAlias; + this.tableAlias = tableAlias; + this.joinPath = joinPath; + } + + @Override + public String getName() { + return wrapped.getName(); + } + + @Override + public String getLongName() { + return wrapped.getLongName(); + } + + @Override + public String getDescription() { + return wrapped.getDescription(); + } + + @Override + public DimensionType getDimensionType() { + return wrapped.getDimensionType(); + } + + @Override + public Class getDataType() { + return wrapped.getDataType(); + } + + @Override + public CardinalitySize getCardinality() { + return wrapped.getCardinality(); + } + + @Override + public String getFriendlyName() { + return wrapped.getFriendlyName(); + } + + public String getColumnName() { + return columnAlias; + } + + public String getTableAlias() { + return tableAlias; + } + + /** + * Returns the join path of this dimension. + * @return Something like "author.book.publisher.name" + */ + public Path getJoinPath() { + return joinPath; + } + + /** + * Returns a String that identifies this dimension in a SQL query. + * @return Something like "table_alias.column_name" + */ + public String getColumnReference() { + return getTableAlias() + "." + getColumnName(); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java index 8b2395f5c3..b02bdeb82b 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java @@ -7,13 +7,21 @@ package com.yahoo.elide.datastores.aggregation.engine.schema; import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.core.Path; +import com.yahoo.elide.core.filter.FilterPredicate; +import com.yahoo.elide.datastores.aggregation.dimension.Dimension; +import com.yahoo.elide.datastores.aggregation.dimension.TimeDimension; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromSubquery; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromTable; +import com.yahoo.elide.datastores.aggregation.engine.annotation.JoinTo; import com.yahoo.elide.datastores.aggregation.schema.Schema; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import javax.persistence.Column; +import javax.persistence.JoinColumn; + /** * A subclass of Schema that supports additional metadata to construct the FROM clause of a SQL query. */ @@ -48,4 +56,75 @@ public SQLSchema(Class entityClass, EntityDictionary dictionary) { } } } + + @Override + protected Dimension constructDimension(String dimensionField, Class cls, EntityDictionary entityDictionary) { + Dimension dim = super.constructDimension(dimensionField, cls, entityDictionary); + + JoinTo joinTo = entityDictionary.getAttributeOrRelationAnnotation(cls, JoinTo.class, dimensionField); + + if (joinTo == null) { + String columnName = getColumnName(entityClass, dimensionField); + + if (dim instanceof TimeDimension) { + return new SQLTimeDimension(dim, columnName, getAlias()); + } + return new SQLDimension(dim, columnName, getAlias()); + } + + Path path = new Path(entityClass, entityDictionary, joinTo.path()); + + if (dim instanceof TimeDimension) { + return new SQLTimeDimension(dim, getJoinColumn(path), getJoinTableAlias(path), path); + } + return new SQLDimension(dim, getJoinColumn(path), getJoinTableAlias(path), path); + } + + /** + * Maps a logical entity attribute into a physical SQL column name. + * @param entityDictionary The dictionary for this elide instance. + * @param clazz The entity class. + * @param fieldName The entity attribute. + * @return The physical SQL column name. + */ + public static String getColumnName(EntityDictionary entityDictionary, Class clazz, String fieldName) { + Column[] column = entityDictionary.getAttributeOrRelationAnnotations(clazz, Column.class, fieldName); + + JoinColumn[] joinColumn = entityDictionary.getAttributeOrRelationAnnotations(clazz, + JoinColumn.class, fieldName); + + if (column == null || column.length == 0) { + if (joinColumn == null || joinColumn.length == 0) { + return fieldName; + } else { + return joinColumn[0].name(); + } + } else { + return column[0].name(); + } + } + + /** + * Returns the physical database column name of an entity field. + * @param clazz The entity which owns the field. + * @param fieldName The field name to lookup + * @return + */ + private String getColumnName(Class clazz, String fieldName) { + return getColumnName(entityDictionary, clazz, fieldName); + } + + private String getJoinColumn(Path path) { + Path.PathElement last = path.lastElement().get(); + Class lastClass = last.getType(); + + return getColumnName(lastClass, last.getFieldName()); + } + + private String getJoinTableAlias(Path path) { + Path.PathElement last = path.lastElement().get(); + Class lastClass = last.getType(); + + return FilterPredicate.getTypeAlias(lastClass); + } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLTimeDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLTimeDimension.java new file mode 100644 index 0000000000..5ed864b903 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLTimeDimension.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ + +package com.yahoo.elide.datastores.aggregation.engine.schema; + +import com.yahoo.elide.core.Path; +import com.yahoo.elide.datastores.aggregation.dimension.Dimension; +import com.yahoo.elide.datastores.aggregation.dimension.TimeDimension; +import com.yahoo.elide.datastores.aggregation.time.TimeGrain; + +import java.util.TimeZone; + +/** + * A time dimension that supports special sauce needed to generate SQL. + * This dimension will be created by the SQLQueryEngine in place of a plain TimeDimension. + */ +public class SQLTimeDimension extends SQLDimension implements TimeDimension { + + /** + * Constructor + * @param dimension a wrapped dimension. + * @param columnAlias The column alias in SQL to refer to this dimension. + * @param tableAlias The table alias in SQL where this dimension lives. + */ + public SQLTimeDimension(Dimension dimension, String columnAlias, String tableAlias) { + super(dimension, columnAlias, tableAlias); + } + + /** + * Constructor + * @param dimension a wrapped dimension. + * @param columnAlias The column alias in SQL to refer to this dimension. + * @param tableAlias The table alias in SQL where this dimension lives. + * @param joinPath A '.' separated path through the entity relationship graph that describes + * how to join the time dimension into the current AnalyticView. + */ + public SQLTimeDimension(Dimension dimension, String columnAlias, String tableAlias, Path joinPath) { + super(dimension, columnAlias, tableAlias, joinPath); + } + + @Override + public TimeZone getTimeZone() { + //TODO + return null; + + } + + @Override + public TimeGrain getTimeGrain() { + //TODO + return null; + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java index 7de82e344c..554e23d3dc 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/schema/Schema.java @@ -51,13 +51,13 @@ public class Schema { @Getter - private final Class entityClass; + protected final Class entityClass; @Getter - private final Set metrics; + protected final Set metrics; @Getter - private final Set dimensions; - @Getter(value = AccessLevel.PRIVATE) - private final EntityDictionary entityDictionary; + protected final Set dimensions; + @Getter(value = AccessLevel.PROTECTED) + protected final EntityDictionary entityDictionary; /** * Constructor diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreIntegrationTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreIntegrationTest.java index d16a0f4d3f..4b468461b1 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreIntegrationTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/AggregationDataStoreIntegrationTest.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.Persistence; import javax.ws.rs.core.MediaType; @@ -65,8 +64,7 @@ protected DataStoreTestHarness createHarness() { entityDictionary.bindEntity(VideoGame.class); EntityManagerFactory emf = Persistence.createEntityManagerFactory("aggregationStore"); - EntityManager em = emf.createEntityManager(); - qE = new SQLQueryEngine(em, entityDictionary); + qE = new SQLQueryEngine(emf, entityDictionary); return new AggregationDataStoreTestHarness(qE); } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 241ce8903f..5ff9b8c719 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -15,7 +15,7 @@ import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; -import com.yahoo.elide.datastores.aggregation.dimension.impl.TimeDimension; +import com.yahoo.elide.datastores.aggregation.dimension.TimeDimension; import com.yahoo.elide.datastores.aggregation.engine.schema.SQLSchema; import com.yahoo.elide.datastores.aggregation.example.Country; import com.yahoo.elide.datastores.aggregation.example.Player; @@ -33,7 +33,6 @@ import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.StreamSupport; -import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.Persistence; @@ -44,6 +43,9 @@ public class SQLQueryEngineTest { private static EntityDictionary dictionary; private static RSQLFilterDialect filterParser; + private static final Country HONG_KONG = new Country(); + private static final Country USA = new Country(); + @BeforeAll public static void init() { emf = Persistence.createEntityManagerFactory("aggregationStore"); @@ -56,6 +58,14 @@ public static void init() { playerStatsSchema = new SQLSchema(PlayerStats.class, dictionary); playerStatsViewSchema = new SQLSchema(PlayerStatsView.class, dictionary); + + HONG_KONG.setIsoCode("HKG"); + HONG_KONG.setName("Hong Kong"); + HONG_KONG.setId("344"); + + USA.setIsoCode("USA"); + USA.setName("United States"); + USA.setId("840"); } /** @@ -63,8 +73,7 @@ public static void init() { */ @Test public void testFullTableLoad() { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Query query = Query.builder() .schema(playerStatsSchema) @@ -107,8 +116,7 @@ public void testFullTableLoad() { */ @Test public void testDegenerateDimensionFilter() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Query query = Query.builder() .schema(playerStatsSchema) @@ -141,8 +149,7 @@ public void testDegenerateDimensionFilter() throws Exception { */ @Test public void testFilterJoin() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Query query = Query.builder() .schema(playerStatsSchema) @@ -158,18 +165,12 @@ public void testFilterJoin() throws Exception { List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); - Country expectedCountry = new Country(); - expectedCountry.setId("840"); - expectedCountry.setIsoCode("USA"); - expectedCountry.setName("United States"); - - PlayerStats usa0 = new PlayerStats(); usa0.setId("0"); usa0.setLowScore(241); usa0.setHighScore(2412); usa0.setOverallRating("Great"); - usa0.setCountry(expectedCountry); + usa0.setCountry(USA); usa0.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); PlayerStats usa1 = new PlayerStats(); @@ -177,7 +178,7 @@ public void testFilterJoin() throws Exception { usa1.setLowScore(35); usa1.setHighScore(1234); usa1.setOverallRating("Good"); - usa1.setCountry(expectedCountry); + usa1.setCountry(USA); usa1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); assertEquals(2, results.size()); @@ -196,8 +197,7 @@ public void testFilterJoin() throws Exception { */ @Test public void testSubqueryFilterJoin() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Query query = Query.builder() .schema(playerStatsViewSchema) @@ -224,8 +224,7 @@ public void testSubqueryFilterJoin() throws Exception { */ @Test public void testSubqueryLoad() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Query query = Query.builder() .schema(playerStatsViewSchema) @@ -248,8 +247,7 @@ public void testSubqueryLoad() throws Exception { */ @Test public void testSortJoin() { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Map sortMap = new TreeMap<>(); sortMap.put("player.name", Sorting.SortOrder.asc); @@ -294,8 +292,7 @@ public void testSortJoin() { */ @Test public void testPagination() { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Pagination pagination = Pagination.fromOffsetAndLimit(1, 0, true); @@ -331,8 +328,7 @@ public void testPagination() { */ @Test public void testHavingClause() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Query query = Query.builder() .schema(playerStatsSchema) @@ -362,8 +358,7 @@ public void testHavingClause() throws Exception { */ @Test public void testTheEverythingQuery() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Map sortMap = new TreeMap<>(); sortMap.put("player.name", Sorting.SortOrder.asc); @@ -396,8 +391,7 @@ public void testTheEverythingQuery() throws Exception { */ @Test public void testSortByMultipleColumns() { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Map sortMap = new TreeMap<>(); sortMap.put("lowScore", Sorting.SortOrder.desc); @@ -443,8 +437,7 @@ public void testSortByMultipleColumns() { */ @Test public void testRelationshipHydration() { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); + QueryEngine engine = new SQLQueryEngine(emf, dictionary); Map sortMap = new TreeMap<>(); sortMap.put("country.name", Sorting.SortOrder.desc); @@ -462,22 +455,12 @@ public void testRelationshipHydration() { List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); - Country usa = new Country(); - usa.setId("840"); - usa.setIsoCode("USA"); - usa.setName("United States"); - - Country hk = new Country(); - hk.setId("344"); - hk.setIsoCode("HKG"); - hk.setName("Hong Kong"); - PlayerStats usa0 = new PlayerStats(); usa0.setId("0"); usa0.setLowScore(241); usa0.setHighScore(2412); usa0.setOverallRating("Great"); - usa0.setCountry(usa); + usa0.setCountry(USA); usa0.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); PlayerStats usa1 = new PlayerStats(); @@ -485,7 +468,7 @@ public void testRelationshipHydration() { usa1.setLowScore(35); usa1.setHighScore(1234); usa1.setOverallRating("Good"); - usa1.setCountry(usa); + usa1.setCountry(USA); usa1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); PlayerStats hk2 = new PlayerStats(); @@ -493,7 +476,7 @@ public void testRelationshipHydration() { hk2.setLowScore(72); hk2.setHighScore(1000); hk2.setOverallRating("Good"); - hk2.setCountry(hk); + hk2.setCountry(HONG_KONG); hk2.setRecordedDate(Timestamp.valueOf("2019-07-13 00:00:00")); assertEquals(3, results.size()); @@ -505,4 +488,121 @@ public void testRelationshipHydration() { PlayerStats actualStats1 = (PlayerStats) results.get(0); assertNotNull(actualStats1.getCountry()); } + + /** + * Test grouping by a dimension with a JoinTo annotation. + * + * @throws Exception exception + */ + @Test + public void testJoinToGroupBy() throws Exception { + QueryEngine engine = new SQLQueryEngine(emf, dictionary); + + Query query = Query.builder() + .schema(playerStatsSchema) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) + .groupDimension(playerStatsSchema.getDimension("countryIsoCode")) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); + stats1.setHighScore(3646); + stats1.setCountryIsoCode("USA"); + + PlayerStats stats2 = new PlayerStats(); + stats2.setId("1"); + stats2.setHighScore(1000); + stats2.setCountryIsoCode("HKG"); + + assertEquals(2, results.size()); + assertEquals(stats1, results.get(0)); + assertEquals(stats2, results.get(1)); + } + + /** + * Test grouping by a dimension with a JoinTo annotation. + * + * @throws Exception exception + */ + @Test + public void testJoinToFilter() throws Exception { + QueryEngine engine = new SQLQueryEngine(emf, dictionary); + + Query query = Query.builder() + .schema(playerStatsSchema) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) + .groupDimension(playerStatsSchema.getDimension("overallRating")) + .whereFilter(filterParser.parseFilterExpression("countryIsoCode==USA", + PlayerStats.class, false)) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); + stats1.setOverallRating("Good"); + stats1.setHighScore(1234); + + PlayerStats stats2 = new PlayerStats(); + stats2.setId("1"); + stats2.setOverallRating("Great"); + stats2.setHighScore(2412); + + assertEquals(2, results.size()); + assertEquals(stats1, results.get(0)); + assertEquals(stats2, results.get(1)); + } + + /** + * Test grouping by a dimension with a JoinTo annotation. + * + * @throws Exception exception + */ + @Test + public void testJoinToSort() throws Exception { + QueryEngine engine = new SQLQueryEngine(emf, dictionary); + + Map sortMap = new TreeMap<>(); + sortMap.put("countryIsoCode", Sorting.SortOrder.asc); + + Query query = Query.builder() + .schema(playerStatsSchema) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) + .groupDimension(playerStatsSchema.getDimension("overallRating")) + .groupDimension(playerStatsSchema.getDimension("country")) + .sorting(new Sorting(sortMap)) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); + stats1.setOverallRating("Good"); + stats1.setCountry(HONG_KONG); + stats1.setHighScore(1000); + + PlayerStats stats2 = new PlayerStats(); + stats2.setId("1"); + stats2.setOverallRating("Good"); + stats2.setCountry(USA); + stats2.setHighScore(1234); + + PlayerStats stats3 = new PlayerStats(); + stats3.setId("2"); + stats3.setOverallRating("Great"); + stats3.setCountry(USA); + stats3.setHighScore(2412); + + assertEquals(3, results.size()); + assertEquals(stats1, results.get(0)); + assertEquals(stats2, results.get(1)); + assertEquals(stats3, results.get(2)); + } + + //TODO - Add Invalid Request Tests } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java index 7495748026..ff95548840 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/PlayerStats.java @@ -14,6 +14,7 @@ import com.yahoo.elide.datastores.aggregation.annotation.Temporal; import com.yahoo.elide.datastores.aggregation.dimension.EntityDimensionTest; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromTable; +import com.yahoo.elide.datastores.aggregation.engine.annotation.JoinTo; import com.yahoo.elide.datastores.aggregation.metric.Max; import com.yahoo.elide.datastores.aggregation.metric.Min; import com.yahoo.elide.datastores.aggregation.time.TimeGrain; @@ -63,6 +64,11 @@ public class PlayerStats { */ private Country country; + /** + * A dimension field joined to this table. + */ + private String countryIsoCode; + /** * A table dimension. */ @@ -141,4 +147,13 @@ public Date getRecordedDate() { public void setRecordedDate(final Date recordedDate) { this.recordedDate = recordedDate; } + + @JoinTo(path = "country.isoCode") + public String getCountryIsoCode() { + return countryIsoCode; + } + + public void setCountryIsoCode(String isoCode) { + this.countryIsoCode = isoCode; + } }