From 84638ee895afc223620f6b6e64c956228be7e6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jack=20=28=EC=A0=95=ED=99=98=29?= Date: Fri, 12 Jul 2019 17:29:49 -0700 Subject: [PATCH 01/47] AggregationDataStore: Schema (#846) * AggregationDataStore: Static Attribute Aggregation * Address comments * Implement TimeDimension and all its supporting components * refactor * Address comments from @aklish * Address comments from @aklish && Tests & Javadoc * Address comments from @aklish * Address comments from @aklish and @hellohanchen * Address comments from Aaron * ToMany is not supported * Address comments from Aaron --- .../elide/core/EntityDictionaryTest.java | 12 + .../elide-datastore-aggregation/pom.xml | 2 - .../elide/datastores/aggregation/Schema.java | 291 ++++++++++++++++++ 3 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java diff --git a/elide-core/src/test/java/com/yahoo/elide/core/EntityDictionaryTest.java b/elide-core/src/test/java/com/yahoo/elide/core/EntityDictionaryTest.java index 34c3657a0e..0d3554a220 100644 --- a/elide-core/src/test/java/com/yahoo/elide/core/EntityDictionaryTest.java +++ b/elide-core/src/test/java/com/yahoo/elide/core/EntityDictionaryTest.java @@ -511,4 +511,16 @@ public void testIsValidField() { assertTrue(isValidField(Job.class, "title")); assertFalse(isValidField(Job.class, "foo")); } + + @Test + public void testAttributeOrRelationAnnotationExists() { + Assert.assertTrue(attributeOrRelationAnnotationExists(Job.class, "jobId", Id.class)); + Assert.assertFalse(attributeOrRelationAnnotationExists(Job.class, "title", OneToOne.class)); + } + + @Test + public void testIsValidField() { + Assert.assertTrue(isValidField(Job.class, "title")); + Assert.assertFalse(isValidField(Job.class, "foo")); + } } diff --git a/elide-datastore/elide-datastore-aggregation/pom.xml b/elide-datastore/elide-datastore-aggregation/pom.xml index 3104bf2097..3c7bd71274 100644 --- a/elide-datastore/elide-datastore-aggregation/pom.xml +++ b/elide-datastore/elide-datastore-aggregation/pom.xml @@ -91,8 +91,6 @@ mockito-core test - - diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java new file mode 100644 index 0000000000..e190a86554 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java @@ -0,0 +1,291 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.aggregation; + +import com.yahoo.elide.core.DataStore; +import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; +import com.yahoo.elide.datastores.aggregation.annotation.Meta; +import com.yahoo.elide.datastores.aggregation.annotation.MetricAggregation; +import com.yahoo.elide.datastores.aggregation.annotation.MetricComputation; +import com.yahoo.elide.datastores.aggregation.annotation.Temporal; +import com.yahoo.elide.datastores.aggregation.dimension.DegenerateDimension; +import com.yahoo.elide.datastores.aggregation.dimension.Dimension; +import com.yahoo.elide.datastores.aggregation.dimension.EntityDimension; +import com.yahoo.elide.datastores.aggregation.dimension.TimeDimension; +import com.yahoo.elide.datastores.aggregation.metric.AggregatedMetric; +import com.yahoo.elide.datastores.aggregation.metric.Aggregation; +import com.yahoo.elide.datastores.aggregation.metric.Metric; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TimeZone; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * {@link Schema} keeps track of table, metrics, and dimensions of an entity for AggregationDataStore. + *

+ * On calling {@link DataStore#populateEntityDictionary(EntityDictionary)}, one {@link Schema} will be created for each + * entity. + *

+ * By overriding {@link #constructDimension(String, Class, EntityDictionary)} and + * {@link #constructMetric(String, Class, EntityDictionary)}, people can have new schema backed by their own defined + * {@link Dimension}s and {@link Metric}s. + *

+ * {@link Schema} is thread-safe and can be accessed by multiple-threads. + */ +@Slf4j +public class Schema { + + @Getter + private final Class entityClass; + @Getter + private final Set metrics; + @Getter + private final Set dimensions; + @Getter(value = AccessLevel.PRIVATE) + private final EntityDictionary entityDictionary; + + /** + * Constructor + *

+ * This constructor calls {@link #constructDimension(String, Class, EntityDictionary)} and + * {@link #constructMetric(String, Class, EntityDictionary)} ()} to construct all {@link Dimension}s and + * {@link Metric}s associated with the entity class passed in. + * + * @param cls The type of the entity, whose {@link Schema} is to be constructed + * @param entityDictionary The meta info object that helps to construct {@link Schema} + * + * @throws NullPointerException if anyone of the arguments is {@code null} + */ + public Schema(Class cls, EntityDictionary entityDictionary) { + this.entityClass = Objects.requireNonNull(cls, "cls"); + this.entityDictionary = Objects.requireNonNull(entityDictionary, "entityDictionary"); + + this.metrics = getAllMetrics(); + this.dimensions = getAllDimensions(); + } + + /** + * Returns an immutable view of all {@link Dimension}s and {@link Metric}s described by this {@link Schema}. + * + * @return union of all {@link Dimension}s and {@link Metric}s under this {@link Schema} + */ + public Set getAllColumns() { + return Stream.concat(getMetrics().stream(), getDimensions().stream()) + .map(item -> (Column) item) + .collect( + Collectors.collectingAndThen( + Collectors.toSet(), + Collections::unmodifiableSet + ) + ); + } + + /** + * Finds the {@link Dimension} by name. + * + * @param dimensionName The entity field name associated with the searched {@link Dimension} + * + * @return {@link Dimension} found or {@code null} if not found + */ + public Dimension getDimension(String dimensionName) { + return getDimensions().stream() + .filter(dimension -> dimension.getName().equals(dimensionName)) + .findAny() + .orElse(null); + } + + /** + * Finds the {@link Metric} by name. + * + * @param metricName The entity field name associated with the searched {@link Metric} + * + * @return {@link Metric} found or {@code null} if not found + */ + public Metric getMetric(String metricName) { + return getMetrics().stream() + .filter(metric -> metric.getName().equals(metricName)) + .findAny() + .orElse(null); + } + + /** + * Returns whether or not an entity field is a metric field. + *

+ * A field is a metric field iff that field is annotated by at least one of + *

    + *
  1. {@link MetricAggregation} + *
  2. {@link MetricComputation} + *
+ * + * @param fieldName The entity field + * + * @return {@code true} if the field is a metric field + */ + public boolean isMetricField(String fieldName) { + return getEntityDictionary().attributeOrRelationAnnotationExists( + getEntityClass(), fieldName, MetricAggregation.class + ) + || getEntityDictionary().attributeOrRelationAnnotationExists( + getEntityClass(), fieldName, MetricComputation.class + ); + } + + /** + * Constructs a new {@link Metric} instance. + * + * @param metricField The entity field of the metric being constructed + * @param cls The entity that contains the metric being constructed + * @param entityDictionary The auxiliary object that offers binding info used to construct this + * {@link Metric} + * + * @return a {@link Metric} + */ + protected Metric constructMetric(String metricField, Class cls, EntityDictionary entityDictionary) { + Meta metaData = entityDictionary.getAttributeOrRelationAnnotation(cls, Meta.class, metricField); + Class fieldType = entityDictionary.getType(cls, metricField); + + // get all metric aggregations + List> aggregations = Arrays.stream( + entityDictionary.getAttributeOrRelationAnnotation( + cls, + MetricAggregation.class, + metricField + ).aggregations() + ) + .collect( + Collectors.collectingAndThen( + Collectors.toList(), + Collections::unmodifiableList + ) + ); + + return new AggregatedMetric( + metricField, + metaData, + fieldType, + aggregations + ); + } + + /** + * Constructs and returns a new instance of {@link Dimension}. + * + * @param dimensionField The entity field of the dimension being constructed + * @param cls The entity that contains the dimension being constructed + * @param entityDictionary The auxiliary object that offers binding info used to construct this + * {@link Dimension} + * + * @return a {@link Dimension} + */ + protected Dimension constructDimension(String dimensionField, Class cls, EntityDictionary entityDictionary) { + // field with ToMany relationship is not supported + if (getEntityDictionary().getRelationshipType(cls, dimensionField).isToMany()) { + String message = String.format("ToMany relationship is not supported in '%s'", cls.getCanonicalName()); + log.error(message); + throw new IllegalStateException(message); + } + + Meta metaData = entityDictionary.getAttributeOrRelationAnnotation(cls, Meta.class, dimensionField); + Class fieldType = entityDictionary.getType(cls, dimensionField); + + String friendlyName = EntityDimension.getFriendlyNameField(cls, entityDictionary); + CardinalitySize cardinality = EntityDimension.getEstimatedCardinality(dimensionField, cls, entityDictionary); + + if (entityDictionary.isRelation(cls, dimensionField)) { + // relationship field + return new EntityDimension( + dimensionField, + metaData, + fieldType, + cardinality, + friendlyName + ); + } else if (!getEntityDictionary().attributeOrRelationAnnotationExists(cls, dimensionField, Temporal.class)) { + // regular field + return new DegenerateDimension( + dimensionField, + metaData, + fieldType, + cardinality, + friendlyName, + DegenerateDimension.parseColumnType(dimensionField, cls, entityDictionary) + ); + } else { + // temporal field + Temporal temporal = getEntityDictionary().getAttributeOrRelationAnnotation( + cls, + Temporal.class, + dimensionField + ); + + return new TimeDimension( + dimensionField, + metaData, + fieldType, + cardinality, + friendlyName, + TimeZone.getTimeZone(temporal.timeZone()), + temporal.timeGrain() + ); + + } + } + + /** + * Constructs all metrics found in an entity. + *

+ * This method calls {@link #constructMetric(String, Class, EntityDictionary)} to create each dimension inside the + * entity + * + * @return all metric fields as {@link Metric} objects + */ + private Set getAllMetrics() { + return getEntityDictionary().getAllFields(getEntityClass()).stream() + .filter(this::isMetricField) + // TODO: remove the filter() below when computedMetric is supported + .filter(field -> + getEntityDictionary() + .attributeOrRelationAnnotationExists(getEntityClass(), field, MetricAggregation.class) + ) + .map(field -> constructMetric(field, getEntityClass(), getEntityDictionary())) + .collect( + Collectors.collectingAndThen( + Collectors.toSet(), + Collections::unmodifiableSet + ) + ); + } + + /** + * Constructs all dimensions found in an entity. + *

+ * This method calls {@link #constructDimension(String, Class, EntityDictionary)} to create each dimension inside + * the entity + * + * @return all non-metric fields as {@link Dimension} objects + */ + private Set getAllDimensions() { + return getEntityDictionary().getAllFields(getEntityClass()).stream() + .filter(field -> !isMetricField(field)) + .map(field -> constructDimension(field, getEntityClass(), getEntityDictionary())) + .collect( + Collectors.collectingAndThen( + Collectors.toSet(), + Collections::unmodifiableSet + ) + ); + } +} From 823cbe54133483afb59c5eec1b8cfd5a25a90762 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 10:54:10 -0500 Subject: [PATCH 02/47] Added basic H2 DB test harness --- .../datastores/aggregation/engine/SQLQueryEngine.java | 10 ++++++++++ .../src/test/resources/player_stats.csv | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 9b01f2af39..c2f0e0b603 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -48,6 +48,9 @@ import javax.persistence.JoinColumn; import javax.persistence.Table; +import java.util.List; +import javax.persistence.EntityManager; + /** * QueryEngine for SQL backed stores. */ @@ -77,6 +80,13 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) )); } + private EntityManager entityManager; + + public SQLQueryEngine(EntityManager entityManager) { + this.entityManager = entityManager; + } + + @Override public Iterable executeQuery(Query query) { SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); diff --git a/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv b/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv index 7a75dd4765..9f493a03dc 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv +++ b/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv @@ -1,3 +1,9 @@ +<<<<<<< HEAD id,highScore,lowScore,overallRating,country_id,player_id,recordedDate Jon Doe,1234,72,Good,840,1,2019-07-12 00:00:00 Jane Doe,2412,241,Great,840,2,2019-07-11 00:00:00 +======= +id,highScore,lowScore,overallRating,country_id,recordedDate +Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 +Jane Doe,2412,241,Great,840,2019-07-11 00:00:00 +>>>>>>> Added basic H2 DB test harness From a1e9d8ea94857b4680dbc57e1d603224a0d02739 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 11:44:00 -0500 Subject: [PATCH 03/47] Started breaking out projections --- .../yahoo/elide/datastores/aggregation/Schema.java | 5 +++++ .../aggregation/engine/SQLQueryEngine.java | 13 ++++--------- .../aggregation/engine/SQLQueryEngineTest.java | 2 ++ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java index e190a86554..6b4f8c3631 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java @@ -22,6 +22,7 @@ import lombok.AccessLevel; import lombok.Getter; +import lombok.Singular; import lombok.extern.slf4j.Slf4j; import java.util.Arrays; @@ -50,8 +51,12 @@ public class Schema { @Getter private final Class entityClass; + + @Singular @Getter private final Set metrics; + + @Singular @Getter private final Set dimensions; @Getter(value = AccessLevel.PRIVATE) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index c2f0e0b603..0f92dae445 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -48,9 +48,6 @@ import javax.persistence.JoinColumn; import javax.persistence.Table; -import java.util.List; -import javax.persistence.EntityManager; - /** * QueryEngine for SQL backed stores. */ @@ -80,12 +77,6 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) )); } - private EntityManager entityManager; - - public SQLQueryEngine(EntityManager entityManager) { - this.entityManager = entityManager; - } - @Override public Iterable executeQuery(Query query) { @@ -501,4 +492,8 @@ private String generateHavingClauseColumnReference(FilterPredicate predicate, Qu return metric.getMetricExpression(Optional.of(agg)); } + + protected Object coerceObjectToEntity(Object result) { + return null; + } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index bc69f7a953..4751b63eb0 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -37,6 +37,7 @@ public class SQLQueryEngineTest { private EntityManagerFactory emf; + private Schema playerStatsSchema; private Schema playerStatsSchema; private Schema playerStatsViewSchema; @@ -186,6 +187,7 @@ public void testSubqueryLoad() throws Exception { EntityManager em = emf.createEntityManager(); QueryEngine engine = new SQLQueryEngine(em, dictionary); + Query query = Query.builder() .schema(playerStatsViewSchema) .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) From 8a4b70ac11b69271709a78cf0357974e9d723c68 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 13:44:11 -0500 Subject: [PATCH 04/47] Moved getValue and setValue from PersistentResource to EntityDictionary --- .../yahoo/elide/core/EntityDictionary.java | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java index c5006c457b..148cd7981b 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java @@ -1402,6 +1402,161 @@ private boolean isValidParameterizedMap(Map values, Class keyType, Clas return true; } + /** + * Invoke the get[fieldName] method on the target object OR get the field with the corresponding name. + * @param target the object to get + * @param fieldName the field name to get or invoke equivalent get method + * @return the value + */ + public Object getValue(Object target, String fieldName, RequestScope scope) { + AccessibleObject accessor = getAccessibleObject(target, fieldName); + try { + if (accessor instanceof Method) { + // Pass RequestScope into @Computed fields if requested + if (isMethodRequestScopeable(target, (Method) accessor)) { + return ((Method) accessor).invoke(target, scope); + } + return ((Method) accessor).invoke(target); + } + if (accessor instanceof Field) { + return ((Field) accessor).get(target); + } + } catch (IllegalAccessException e) { + throw new InvalidAttributeException(fieldName, getJsonAliasFor(target.getClass()), e); + } catch (InvocationTargetException e) { + throw handleInvocationTargetException(e); + } + throw new InvalidAttributeException(fieldName, getJsonAliasFor(target.getClass())); + } + + /** + * Invoke the set[fieldName] method on the target object OR set the field with the corresponding name. + * @param fieldName the field name to set or invoke equivalent set method + * @param value the value to set + */ + public void setValue(Object target, String fieldName, Object value) { + Class targetClass = target.getClass(); + String targetType = getJsonAliasFor(targetClass); + + try { + Class fieldClass = getType(targetClass, fieldName); + String realName = getNameFromAlias(target, fieldName); + fieldName = (realName != null) ? realName : fieldName; + String setMethod = "set" + StringUtils.capitalize(fieldName); + Method method = EntityDictionary.findMethod(targetClass, setMethod, fieldClass); + method.invoke(target, coerce(target, value, fieldName, fieldClass)); + } catch (IllegalAccessException e) { + throw new InvalidAttributeException(fieldName, targetType, e); + } catch (InvocationTargetException e) { + throw handleInvocationTargetException(e); + } catch (IllegalArgumentException | NoSuchMethodException noMethod) { + AccessibleObject accessor = getAccessibleObject(target, fieldName); + if (accessor != null && accessor instanceof Field) { + Field field = (Field) accessor; + try { + field.set(target, coerce(target, value, fieldName, field.getType())); + } catch (IllegalAccessException noField) { + throw new InvalidAttributeException(fieldName, targetType, noField); + } + } else { + throw new InvalidAttributeException(fieldName, targetType); + } + } + } + + /** + * Handle an invocation target exception. + * + * @param e Exception the exception encountered while reflecting on an object's field + * @return Equivalent runtime exception + */ + private static RuntimeException handleInvocationTargetException(InvocationTargetException e) { + Throwable exception = e.getTargetException(); + if (exception instanceof HttpStatusException || exception instanceof WebApplicationException) { + return (RuntimeException) exception; + } + log.error("Caught an unexpected exception (rethrowing as internal server error)", e); + return new InternalServerErrorException("Unexpected exception caught", e); + } + + /** + * Coerce provided value into expected class type. + * + * @param value provided value + * @param fieldName the field name + * @param fieldClass expected class type + * @return coerced value + */ + Object coerce(Object target, Object value, String fieldName, Class fieldClass) { + if (fieldClass != null && Collection.class.isAssignableFrom(fieldClass) && value instanceof Collection) { + return coerceCollection(target, (Collection) value, fieldName, fieldClass); + } + + if (fieldClass != null && Map.class.isAssignableFrom(fieldClass) && value instanceof Map) { + return coerceMap(target, (Map) value, fieldName, fieldClass); + } + + return CoerceUtil.coerce(value, fieldClass); + } + + private Collection coerceCollection(Object target, Collection values, String fieldName, Class fieldClass) { + Class providedType = getParameterizedType(target, fieldName); + + // check if collection is of and contains the correct types + if (fieldClass.isAssignableFrom(values.getClass())) { + boolean valid = true; + for (Object member : values) { + if (member != null && !providedType.isAssignableFrom(member.getClass())) { + valid = false; + break; + } + } + if (valid) { + return values; + } + } + + ArrayList list = new ArrayList<>(values.size()); + for (Object member : values) { + list.add(CoerceUtil.coerce(member, providedType)); + } + + if (Set.class.isAssignableFrom(fieldClass)) { + return new LinkedHashSet<>(list); + } + + return list; + } + + private Map coerceMap(Object target, Map values, String fieldName, Class fieldClass) { + Class keyType = getParameterizedType(target, fieldName, 0); + Class valueType = getParameterizedType(target, fieldName, 1); + + // Verify the existing Map + if (isValidParameterizedMap(values, keyType, valueType)) { + return values; + } + + LinkedHashMap result = new LinkedHashMap<>(values.size()); + for (Map.Entry entry : values.entrySet()) { + result.put(CoerceUtil.coerce(entry.getKey(), keyType), CoerceUtil.coerce(entry.getValue(), valueType)); + } + + return result; + } + + private boolean isValidParameterizedMap(Map values, Class keyType, Class valueType) { + for (Map.Entry entry : values.entrySet()) { + Object key = entry.getKey(); + Object value = entry.getValue(); + if ((key != null && !keyType.isAssignableFrom(key.getClass())) + || (value != null && !valueType.isAssignableFrom(value.getClass()))) { + return false; + } + } + return true; + } + /** * Binds the entity class if not yet bound. * @param entityClass the class to bind. From 542240e64ac24cc498918dc220436e52ff363348 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 14:51:29 -0500 Subject: [PATCH 05/47] Added basic logic to hydrate entities --- .../yahoo/elide/core/EntityDictionary.java | 173 +----------------- .../aggregation/engine/SQLQueryEngine.java | 21 ++- .../engine/SQLQueryEngineTest.java | 1 + 3 files changed, 28 insertions(+), 167 deletions(-) diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java index 148cd7981b..76b57fb0a8 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java @@ -1243,6 +1243,7 @@ public Object getValue(Object target, String fieldName, RequestScope scope) { throw new InvalidAttributeException(fieldName, getJsonAliasFor(target.getClass())); } + /** * Invoke the set[fieldName] method on the target object OR set the field with the corresponding name. * @param fieldName the field name to set or invoke equivalent set method @@ -1252,48 +1253,32 @@ public void setValue(Object target, String fieldName, Object value) { Class targetClass = target.getClass(); String targetType = getJsonAliasFor(targetClass); - String fieldAlias = fieldName; try { Class fieldClass = getType(targetClass, fieldName); String realName = getNameFromAlias(target, fieldName); - fieldAlias = (realName != null) ? realName : fieldName; - String setMethod = "set" + StringUtils.capitalize(fieldAlias); + fieldName = (realName != null) ? realName : fieldName; + String setMethod = "set" + StringUtils.capitalize(fieldName); Method method = EntityDictionary.findMethod(targetClass, setMethod, fieldClass); - method.invoke(target, coerce(target, value, fieldAlias, fieldClass)); + method.invoke(target, coerce(target, value, fieldName, fieldClass)); } catch (IllegalAccessException e) { - throw new InvalidAttributeException(fieldAlias, targetType, e); + throw new InvalidAttributeException(fieldName, targetType, e); } catch (InvocationTargetException e) { throw handleInvocationTargetException(e); } catch (IllegalArgumentException | NoSuchMethodException noMethod) { - AccessibleObject accessor = getAccessibleObject(target, fieldAlias); + AccessibleObject accessor = getAccessibleObject(target, fieldName); if (accessor != null && accessor instanceof Field) { Field field = (Field) accessor; try { - field.set(target, coerce(target, value, fieldAlias, field.getType())); + field.set(target, coerce(target, value, fieldName, field.getType())); } catch (IllegalAccessException noField) { - throw new InvalidAttributeException(fieldAlias, targetType, noField); + throw new InvalidAttributeException(fieldName, targetType, noField); } } else { - throw new InvalidAttributeException(fieldAlias, targetType); + throw new InvalidAttributeException(fieldName, targetType); } } } - /** - * Handle an invocation target exception. - * - * @param e Exception the exception encountered while reflecting on an object's field - * @return Equivalent runtime exception - */ - private static RuntimeException handleInvocationTargetException(InvocationTargetException e) { - Throwable exception = e.getTargetException(); - if (exception instanceof HttpStatusException || exception instanceof WebApplicationException) { - return (RuntimeException) exception; - } - log.error("Caught an unexpected exception (rethrowing as internal server error)", e); - return new InternalServerErrorException("Unexpected exception caught", e); - } - /** * Coerce provided value into expected class type. * @@ -1314,35 +1299,6 @@ public Object coerce(Object target, Object value, String fieldName, Class fie return CoerceUtil.coerce(value, fieldClass); } - private Collection coerceCollection(Object target, Collection values, String fieldName, Class fieldClass) { - Class providedType = getParameterizedType(target, fieldName); - - // check if collection is of and contains the correct types - if (fieldClass.isAssignableFrom(values.getClass())) { - boolean valid = true; - for (Object member : values) { - if (member != null && !providedType.isAssignableFrom(member.getClass())) { - valid = false; - break; - } - } - if (valid) { - return values; - } - } - - ArrayList list = new ArrayList<>(values.size()); - for (Object member : values) { - list.add(CoerceUtil.coerce(member, providedType)); - } - - if (Set.class.isAssignableFrom(fieldClass)) { - return new LinkedHashSet<>(list); - } - - return list; - } - private Map coerceMap(Object target, Map values, String fieldName) { Class keyType = getParameterizedType(target, fieldName, 0); Class valueType = getParameterizedType(target, fieldName, 1); @@ -1390,80 +1346,6 @@ public boolean isValidField(Class cls, String fieldName) { return getAllFields(cls).contains(fieldName); } - private boolean isValidParameterizedMap(Map values, Class keyType, Class valueType) { - for (Map.Entry entry : values.entrySet()) { - Object key = entry.getKey(); - Object value = entry.getValue(); - if ((key != null && !keyType.isAssignableFrom(key.getClass())) - || (value != null && !valueType.isAssignableFrom(value.getClass()))) { - return false; - } - } - return true; - } - - /** - * Invoke the get[fieldName] method on the target object OR get the field with the corresponding name. - * @param target the object to get - * @param fieldName the field name to get or invoke equivalent get method - * @return the value - */ - public Object getValue(Object target, String fieldName, RequestScope scope) { - AccessibleObject accessor = getAccessibleObject(target, fieldName); - try { - if (accessor instanceof Method) { - // Pass RequestScope into @Computed fields if requested - if (isMethodRequestScopeable(target, (Method) accessor)) { - return ((Method) accessor).invoke(target, scope); - } - return ((Method) accessor).invoke(target); - } - if (accessor instanceof Field) { - return ((Field) accessor).get(target); - } - } catch (IllegalAccessException e) { - throw new InvalidAttributeException(fieldName, getJsonAliasFor(target.getClass()), e); - } catch (InvocationTargetException e) { - throw handleInvocationTargetException(e); - } - throw new InvalidAttributeException(fieldName, getJsonAliasFor(target.getClass())); - } - - /** - * Invoke the set[fieldName] method on the target object OR set the field with the corresponding name. - * @param fieldName the field name to set or invoke equivalent set method - * @param value the value to set - */ - public void setValue(Object target, String fieldName, Object value) { - Class targetClass = target.getClass(); - String targetType = getJsonAliasFor(targetClass); - - try { - Class fieldClass = getType(targetClass, fieldName); - String realName = getNameFromAlias(target, fieldName); - fieldName = (realName != null) ? realName : fieldName; - String setMethod = "set" + StringUtils.capitalize(fieldName); - Method method = EntityDictionary.findMethod(targetClass, setMethod, fieldClass); - method.invoke(target, coerce(target, value, fieldName, fieldClass)); - } catch (IllegalAccessException e) { - throw new InvalidAttributeException(fieldName, targetType, e); - } catch (InvocationTargetException e) { - throw handleInvocationTargetException(e); - } catch (IllegalArgumentException | NoSuchMethodException noMethod) { - AccessibleObject accessor = getAccessibleObject(target, fieldName); - if (accessor != null && accessor instanceof Field) { - Field field = (Field) accessor; - try { - field.set(target, coerce(target, value, fieldName, field.getType())); - } catch (IllegalAccessException noField) { - throw new InvalidAttributeException(fieldName, targetType, noField); - } - } else { - throw new InvalidAttributeException(fieldName, targetType); - } - } - } - /** * Handle an invocation target exception. * @@ -1479,26 +1361,6 @@ private static RuntimeException handleInvocationTargetException(InvocationTarget return new InternalServerErrorException("Unexpected exception caught", e); } - /** - * Coerce provided value into expected class type. - * - * @param value provided value - * @param fieldName the field name - * @param fieldClass expected class type - * @return coerced value - */ - Object coerce(Object target, Object value, String fieldName, Class fieldClass) { - if (fieldClass != null && Collection.class.isAssignableFrom(fieldClass) && value instanceof Collection) { - return coerceCollection(target, (Collection) value, fieldName, fieldClass); - } - - if (fieldClass != null && Map.class.isAssignableFrom(fieldClass) && value instanceof Map) { - return coerceMap(target, (Map) value, fieldName, fieldClass); - } - - return CoerceUtil.coerce(value, fieldClass); - } - private Collection coerceCollection(Object target, Collection values, String fieldName, Class fieldClass) { Class providedType = getParameterizedType(target, fieldName); @@ -1528,23 +1390,6 @@ private Collection coerceCollection(Object target, Collection values, String return list; } - private Map coerceMap(Object target, Map values, String fieldName, Class fieldClass) { - Class keyType = getParameterizedType(target, fieldName, 0); - Class valueType = getParameterizedType(target, fieldName, 1); - - // Verify the existing Map - if (isValidParameterizedMap(values, keyType, valueType)) { - return values; - } - - LinkedHashMap result = new LinkedHashMap<>(values.size()); - for (Map.Entry entry : values.entrySet()) { - result.put(CoerceUtil.coerce(entry.getKey(), keyType), CoerceUtil.coerce(entry.getValue(), valueType)); - } - - return result; - } - private boolean isValidParameterizedMap(Map values, Class keyType, Class valueType) { for (Map.Entry entry : values.entrySet()) { Object key = entry.getKey(); diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 0f92dae445..c471e787d2 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -77,7 +77,6 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) )); } - @Override public Iterable executeQuery(Query query) { SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); @@ -493,7 +492,23 @@ private String generateHavingClauseColumnReference(FilterPredicate predicate, Qu return metric.getMetricExpression(Optional.of(agg)); } - protected Object coerceObjectToEntity(Object result) { - return null; + protected Object coerceObjectToEntity(Class entityClass, List projections, Object[] result) { + Preconditions.checkArgument(result.length == projections.size()); + + Object entityInstance; + try { + entityInstance = entityClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new IllegalStateException(e); + } + + for (int idx = 0; idx < result.length; idx++) { + Object value = result[idx]; + String fieldName = projections.get(idx); + + dictionary.setValue(entityInstance, fieldName, value); + } + + return entityInstance; } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 4751b63eb0..3e2b67b8dc 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -38,6 +38,7 @@ public class SQLQueryEngineTest { private EntityManagerFactory emf; private Schema playerStatsSchema; + private EntityDictionary dictionary; private Schema playerStatsSchema; private Schema playerStatsViewSchema; From 3e6a54982bdc57c9b4a17afb019f6e01bc3e194f Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 15:37:24 -0500 Subject: [PATCH 06/47] Added FromTable and FromSubquery annotations. Add explicit exclusion of entity relationship hydration --- .../datastores/aggregation/engine/SQLQueryEngine.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index c471e787d2..3475a02470 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -56,7 +56,6 @@ public class SQLQueryEngine implements QueryEngine { private EntityManager entityManager; private EntityDictionary dictionary; - @Getter private Map, SQLSchema> schemas; @@ -493,6 +492,9 @@ private String generateHavingClauseColumnReference(FilterPredicate predicate, Qu } protected Object coerceObjectToEntity(Class entityClass, List projections, Object[] result) { + SQLSchema schema = schemas.get(entityClass); + + Preconditions.checkNotNull(schema); Preconditions.checkArgument(result.length == projections.size()); Object entityInstance; @@ -506,6 +508,13 @@ protected Object coerceObjectToEntity(Class entityClass, List project Object value = result[idx]; String fieldName = projections.get(idx); + Dimension dim = schema.getDimension(fieldName); + if (dim != null && dim.getDimensionType() == DimensionType.ENTITY) { + + //We don't hydrate relationships here. + continue; + } + dictionary.setValue(entityInstance, fieldName, value); } From 2b41fa284346f0821141a2ef5408de205d64a184 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 16:53:52 -0500 Subject: [PATCH 07/47] Refactored HQLFilterOperation to take an alias generator --- .../elide-datastore-aggregation/pom.xml | 7 + .../aggregation/engine/SQLQueryEngine.java | 6 + .../src/test/resources/player_stats.csv | 6 - .../elide/core/filter/HQLFilterOperation.java | 220 ++++++++++++++++++ 4 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java diff --git a/elide-datastore/elide-datastore-aggregation/pom.xml b/elide-datastore/elide-datastore-aggregation/pom.xml index 3c7bd71274..a42c2c225d 100644 --- a/elide-datastore/elide-datastore-aggregation/pom.xml +++ b/elide-datastore/elide-datastore-aggregation/pom.xml @@ -56,6 +56,12 @@ lombok + + com.yahoo.elide + elide-datastore-hibernate + 4.4.5-SNAPSHOT + + org.hibernate.javax.persistence @@ -91,6 +97,7 @@ mockito-core test + diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 3475a02470..323e428ffb 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -520,4 +520,10 @@ protected Object coerceObjectToEntity(Class entityClass, List project return entityInstance; } + + public String getWhereClause(SQLSchema schema, FilterExpression expression) { + + HQLFilterOperation filterVisitor = new HQLFilterOperation(); + return ""; + } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv b/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv index 9f493a03dc..7a75dd4765 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv +++ b/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv @@ -1,9 +1,3 @@ -<<<<<<< HEAD id,highScore,lowScore,overallRating,country_id,player_id,recordedDate Jon Doe,1234,72,Good,840,1,2019-07-12 00:00:00 Jane Doe,2412,241,Great,840,2,2019-07-11 00:00:00 -======= -id,highScore,lowScore,overallRating,country_id,recordedDate -Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 -Jane Doe,2412,241,Great,840,2019-07-11 00:00:00 ->>>>>>> Added basic H2 DB test harness diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java new file mode 100644 index 0000000000..d24f439fec --- /dev/null +++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java @@ -0,0 +1,220 @@ +/* + * Copyright 2016, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.core.filter; + +import com.yahoo.elide.core.exceptions.InvalidPredicateException; +import com.yahoo.elide.core.exceptions.InvalidValueException; +import com.yahoo.elide.core.filter.FilterPredicate.FilterParameter; +import com.yahoo.elide.core.filter.expression.AndFilterExpression; +import com.yahoo.elide.core.filter.expression.FilterExpression; +import com.yahoo.elide.core.filter.expression.FilterExpressionVisitor; +import com.yahoo.elide.core.filter.expression.NotFilterExpression; +import com.yahoo.elide.core.filter.expression.OrFilterExpression; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * FilterOperation that creates Hibernate query language fragments. + */ +public class HQLFilterOperation implements FilterOperation { + private static final String FILTER_PATH_NOT_NULL = "Filtering field path cannot be empty."; + private static final String FILTER_ALIAS_NOT_NULL = "Filtering alias cannot be empty."; + public static final String PARAM_JOIN = ", "; + public static final Function LOWERED_PARAMETER = p -> + String.format("lower(%s)", p.getPlaceholder()); + + @Override + public String apply(FilterPredicate filterPredicate) { + return apply(filterPredicate, Optional.empty()); + } + + /** + * Transforms a filter predicate into a HQL query fragment. + * @param filterPredicate The predicate to transform. + * @param aliasProvider Function which supplies an alias to append to the predicate fields. + * This is useful for table aliases referenced in HQL for some kinds of joins. + * @return The hql query fragment. + */ + protected String apply(FilterPredicate filterPredicate, Optional> aliasProvider) { + String fieldPath = filterPredicate.getFieldPath(); + + if (aliasProvider.isPresent()) { + fieldPath = aliasProvider.get().apply(filterPredicate) + "." + filterPredicate.getField(); + } + + //HQL doesn't support 'this', but it does support aliases. + fieldPath = fieldPath.replaceAll("\\.this", ""); + + List params = filterPredicate.getParameters(); + String firstParam = params.size() > 0 ? params.get(0).getPlaceholder() : null; + switch (filterPredicate.getOperator()) { + case IN: + Preconditions.checkState(!filterPredicate.getValues().isEmpty()); + return String.format("%s IN (%s)", fieldPath, params.stream() + .map(FilterParameter::getPlaceholder) + .collect(Collectors.joining(PARAM_JOIN))); + + case IN_INSENSITIVE: + Preconditions.checkState(!filterPredicate.getValues().isEmpty()); + return String.format("lower(%s) IN (%s)", fieldPath, params.stream() + .map(LOWERED_PARAMETER) + .collect(Collectors.joining(PARAM_JOIN))); + + case NOT: + Preconditions.checkState(!filterPredicate.getValues().isEmpty()); + return String.format("%s NOT IN (%s)", fieldPath, params.stream() + .map(FilterParameter::getPlaceholder) + .collect(Collectors.joining(PARAM_JOIN))); + + case NOT_INSENSITIVE: + Preconditions.checkState(!filterPredicate.getValues().isEmpty()); + return String.format("lower(%s) NOT IN (%s)", fieldPath, params.stream() + .map(LOWERED_PARAMETER) + .collect(Collectors.joining(PARAM_JOIN))); + + case PREFIX: + return String.format("%s LIKE CONCAT(%s, '%%')", fieldPath, firstParam); + + case PREFIX_CASE_INSENSITIVE: + assertValidValues(fieldPath, firstParam); + return String.format("lower(%s) LIKE CONCAT(lower(%s), '%%')", fieldPath, firstParam); + + case POSTFIX: + return String.format("%s LIKE CONCAT('%%', %s)", fieldPath, firstParam); + + case POSTFIX_CASE_INSENSITIVE: + assertValidValues(fieldPath, firstParam); + return String.format("lower(%s) LIKE CONCAT('%%', lower(%s))", fieldPath, firstParam); + + case INFIX: + return String.format("%s LIKE CONCAT('%%', %s, '%%')", fieldPath, firstParam); + + case INFIX_CASE_INSENSITIVE: + assertValidValues(fieldPath, firstParam); + return String.format("lower(%s) LIKE CONCAT('%%', lower(%s), '%%')", fieldPath, firstParam); + + case LT: + return String.format("%s < %s", fieldPath, params.size() == 1 ? firstParam : leastClause(params)); + + case LE: + return String.format("%s <= %s", fieldPath, params.size() == 1 ? firstParam : leastClause(params)); + + case GT: + return String.format("%s > %s", fieldPath, params.size() == 1 ? firstParam : greatestClause(params)); + + case GE: + return String.format("%s >= %s", fieldPath, params.size() == 1 ? firstParam : greatestClause(params)); + + // Not parametric checks + case ISNULL: + return String.format("%s IS NULL", fieldPath); + + case NOTNULL: + return String.format("%s IS NOT NULL", fieldPath); + + case TRUE: + return "(1 = 1)"; + + case FALSE: + return "(1 = 0)"; + + default: + throw new InvalidPredicateException("Operator not implemented: " + filterPredicate.getOperator()); + } + } + + private String greatestClause(List params) { + return String.format("greatest(%s)", params.stream() + .map(FilterParameter::getPlaceholder) + .collect(Collectors.joining(PARAM_JOIN))); + } + + private String leastClause(List params) { + return String.format("least(%s)", params.stream() + .map(FilterParameter::getPlaceholder) + .collect(Collectors.joining(PARAM_JOIN))); + } + + private void assertValidValues(String fieldPath, String alias) { + if (Strings.isNullOrEmpty(fieldPath)) { + throw new InvalidValueException(FILTER_PATH_NOT_NULL); + } + if (Strings.isNullOrEmpty(alias)) { + throw new IllegalStateException(FILTER_ALIAS_NOT_NULL); + } + } + + /** + * Translates the filterExpression to a JPQL filter fragment. + * @param filterExpression The filterExpression to translate + * @param prefixWithAlias If true, use the default alias provider to append the predicates with an alias. + * Otherwise, don't append aliases. + * @return A JPQL filter fragment. + */ + public String apply(FilterExpression filterExpression, boolean prefixWithAlias) { + Optional> aliasProvider = Optional.empty(); + if (prefixWithAlias) { + aliasProvider = Optional.of((predicate) -> predicate.getAlias()); + } + + return apply(filterExpression, aliasProvider); + } + + /** + * Translates the filterExpression to a JPQL filter fragment. + * @param filterExpression The filterExpression to translate + * @param aliasProvider Optionally appends the predicates clause with an alias. + * @return A JPQL filter fragment. + */ + public String apply(FilterExpression filterExpression, Optional> aliasProvider) { + HQLQueryVisitor visitor = new HQLQueryVisitor(aliasProvider); + return "WHERE " + filterExpression.accept(visitor); + } + + /** + * Filter expression visitor which builds an HQL query. + */ + public class HQLQueryVisitor implements FilterExpressionVisitor { + public static final String TWO_NON_FILTERING_EXPRESSIONS = + "Cannot build a filter from two non-filtering expressions"; + private Optional> aliasProvider; + + public HQLQueryVisitor(Optional> aliasProvider) { + this.aliasProvider = aliasProvider; + } + + @Override + public String visitPredicate(FilterPredicate filterPredicate) { + return apply(filterPredicate, aliasProvider); + } + + @Override + public String visitAndExpression(AndFilterExpression expression) { + FilterExpression left = expression.getLeft(); + FilterExpression right = expression.getRight(); + return "(" + left.accept(this) + " AND " + right.accept(this) + ")"; + } + + @Override + public String visitOrExpression(OrFilterExpression expression) { + FilterExpression left = expression.getLeft(); + FilterExpression right = expression.getRight(); + return "(" + left.accept(this) + " OR " + right.accept(this) + ")"; + } + + @Override + public String visitNotExpression(NotFilterExpression expression) { + String negated = expression.getNegated().accept(this); + return "NOT (" + negated + ")"; + } + } +} From 2e32ee5406ab2693044d0a11fda87887c4952991 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 18:43:48 -0500 Subject: [PATCH 08/47] Added test support for RSQL filter generation. Some cleanup --- .../elide/datastores/aggregation/Query.java | 20 +++++-- .../aggregation/engine/SQLQueryEngine.java | 60 +++++++++++++------ 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java index 70c50379af..f3b325f580 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java @@ -31,10 +31,10 @@ @Data @Builder public class Query { - private final Schema schema; + private final Class entityClass; @Singular - private final Map> metrics; + private final Set metrics; @Singular private final Set groupDimensions; @@ -42,10 +42,18 @@ public class Query { @Singular private final Set timeDimensions; - private final FilterExpression whereFilter; - private final FilterExpression havingFilter; - private final Sorting sorting; - private final Pagination pagination; + @Builder.Default + private final Optional whereFilter = Optional.empty(); + + @Builder.Default + private final Optional havingFilter = Optional.empty(); + + @Builder.Default + private final Optional sorting = Optional.empty(); + + @Builder.Default + private final Optional pagination = Optional.empty(); + private final RequestScope scope; /** diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 323e428ffb..1a398c91db 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -8,14 +8,10 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.Path; -import com.yahoo.elide.core.TimedFunction; -import com.yahoo.elide.core.exceptions.InvalidPredicateException; import com.yahoo.elide.core.filter.FilterPredicate; -import com.yahoo.elide.core.filter.FilterTranslator; +import com.yahoo.elide.core.filter.HQLFilterOperation; import com.yahoo.elide.core.filter.expression.FilterExpression; import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor; -import com.yahoo.elide.core.pagination.Pagination; -import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.dimension.Dimension; @@ -33,14 +29,10 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import javax.persistence.Column; @@ -59,6 +51,15 @@ public class SQLQueryEngine implements QueryEngine { @Getter private Map, SQLSchema> schemas; + //Function to return the alias to apply to a filter predicate expression. + private static final Function ALIAS_PROVIDER = (predicate) -> { + List elements = predicate.getPath().getPathElements(); + + Path.PathElement last = elements.get(elements.size() - 1); + + return FilterPredicate.getTypeAlias(last.getType()); + }; + public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { this.entityManager = entityManager; this.dictionary = dictionary; @@ -433,14 +434,23 @@ private String extractProjection(Query query) { String projectionClause = metricProjections.stream() .collect(Collectors.joining(",")); - if (!dimensionProjections.isEmpty()) { - projectionClause = projectionClause + "," + dimensionProjections.stream() - .map((name) -> query.getSchema().getAlias() + "." + name) - .collect(Collectors.joining(",")); + String sql = String.format("SELECT %s FROM %s AS %s", + projectionClause, tableName, tableAlias); + String nativeSql = translateSqlToNative(sql, dialect); + + if (query.getWhereFilter().isPresent()) { + nativeSql += translateFilterExpression(schema, query.getWhereFilter().get()); } - return projectionClause; - } + log.debug("Running native SQL query: {}", nativeSql); + + javax.persistence.Query jpaQuery = entityManager.createNativeQuery(nativeSql); + + if (query.getWhereFilter().isPresent()) { + supplyFilterQueryParameters(query.getWhereFilter().get(), jpaQuery); + } + + List results = jpaQuery.getResultList(); /** * Extracts a GROUP BY SQL clause. @@ -521,9 +531,23 @@ protected Object coerceObjectToEntity(Class entityClass, List project return entityInstance; } - public String getWhereClause(SQLSchema schema, FilterExpression expression) { - + private String translateFilterExpression(SQLSchema schema, FilterExpression expression) { HQLFilterOperation filterVisitor = new HQLFilterOperation(); - return ""; + + return filterVisitor.apply(expression, Optional.of(ALIAS_PROVIDER)); + } + + private void supplyFilterQueryParameters(FilterExpression expression, + javax.persistence.Query query) { + Collection predicates = expression.accept(new PredicateExtractionVisitor()); + + for (FilterPredicate filterPredicate : predicates) { + if (filterPredicate.getOperator().isParameterized()) { + boolean shouldEscape = filterPredicate.isMatchingOperator(); + filterPredicate.getParameters().forEach(param -> { + query.setParameter(param.getName(), shouldEscape ? param.escapeMatching() : param.getValue()); + }); + } + } } } From fac27e4078fac88fc27bb40ed36378169b419249 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 19:17:43 -0500 Subject: [PATCH 09/47] Added basic support for WHERE clause filtering on the fact table --- .../elide-datastore-aggregation/pom.xml | 5 + .../elide/datastores/aggregation/Query.java | 22 +- .../aggregation/engine/SQLQueryEngine.java | 53 ++-- .../engine/SQLQueryEngineTest.java | 253 +----------------- 4 files changed, 31 insertions(+), 302 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/pom.xml b/elide-datastore/elide-datastore-aggregation/pom.xml index a42c2c225d..8c0e460410 100644 --- a/elide-datastore/elide-datastore-aggregation/pom.xml +++ b/elide-datastore/elide-datastore-aggregation/pom.xml @@ -62,6 +62,11 @@ 4.4.5-SNAPSHOT + + org.projectlombok + lombok + + org.hibernate.javax.persistence diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java index f3b325f580..f27a68738b 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java @@ -31,10 +31,10 @@ @Data @Builder public class Query { - private final Class entityClass; + private final Schema schema; @Singular - private final Set metrics; + private final Map> metrics; @Singular private final Set groupDimensions; @@ -42,18 +42,10 @@ public class Query { @Singular private final Set timeDimensions; - @Builder.Default - private final Optional whereFilter = Optional.empty(); - - @Builder.Default - private final Optional havingFilter = Optional.empty(); - - @Builder.Default - private final Optional sorting = Optional.empty(); - - @Builder.Default - private final Optional pagination = Optional.empty(); - + private final FilterExpression whereFilter; + private final FilterExpression havingFilter; + private final Sorting sorting; + private final Pagination pagination; private final RequestScope scope; /** @@ -66,4 +58,4 @@ public Set getDimensions() { Collectors.toCollection(LinkedHashSet::new) ); } -} +} \ No newline at end of file diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 1a398c91db..ec33c85e4f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -8,10 +8,14 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.Path; +import com.yahoo.elide.core.TimedFunction; +import com.yahoo.elide.core.exceptions.InvalidPredicateException; import com.yahoo.elide.core.filter.FilterPredicate; import com.yahoo.elide.core.filter.HQLFilterOperation; import com.yahoo.elide.core.filter.expression.FilterExpression; import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor; +import com.yahoo.elide.core.pagination.Pagination; +import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.dimension.Dimension; @@ -29,10 +33,14 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import javax.persistence.Column; @@ -158,7 +166,7 @@ protected SQLQuery toSQL(Query query) { (predicate) -> { return generateHavingClauseColumnReference(predicate, query); })); } - if (!query.getDimensions().isEmpty()) { + if (! query.getDimensions().isEmpty()) { builder.groupByClause(extractGroupBy(query)); } @@ -243,7 +251,7 @@ protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt c private String translateFilterExpression(SQLSchema schema, FilterExpression expression, Function columnGenerator) { - FilterTranslator filterVisitor = new FilterTranslator(); + HQLFilterOperation filterVisitor = new HQLFilterOperation(); return filterVisitor.apply(expression, columnGenerator); } @@ -434,23 +442,14 @@ private String extractProjection(Query query) { String projectionClause = metricProjections.stream() .collect(Collectors.joining(",")); - String sql = String.format("SELECT %s FROM %s AS %s", - projectionClause, tableName, tableAlias); - String nativeSql = translateSqlToNative(sql, dialect); - - if (query.getWhereFilter().isPresent()) { - nativeSql += translateFilterExpression(schema, query.getWhereFilter().get()); - } - - log.debug("Running native SQL query: {}", nativeSql); - - javax.persistence.Query jpaQuery = entityManager.createNativeQuery(nativeSql); - - if (query.getWhereFilter().isPresent()) { - supplyFilterQueryParameters(query.getWhereFilter().get(), jpaQuery); + if (!dimensionProjections.isEmpty()) { + projectionClause = projectionClause + "," + dimensionProjections.stream() + .map((name) -> query.getSchema().getAlias() + "." + name) + .collect(Collectors.joining(",")); } - List results = jpaQuery.getResultList(); + return projectionClause; + } /** * Extracts a GROUP BY SQL clause. @@ -530,24 +529,4 @@ protected Object coerceObjectToEntity(Class entityClass, List project return entityInstance; } - - private String translateFilterExpression(SQLSchema schema, FilterExpression expression) { - HQLFilterOperation filterVisitor = new HQLFilterOperation(); - - return filterVisitor.apply(expression, Optional.of(ALIAS_PROVIDER)); - } - - private void supplyFilterQueryParameters(FilterExpression expression, - javax.persistence.Query query) { - Collection predicates = expression.accept(new PredicateExtractionVisitor()); - - for (FilterPredicate filterPredicate : predicates) { - if (filterPredicate.getOperator().isParameterized()) { - boolean shouldEscape = filterPredicate.isMatchingOperator(); - filterPredicate.getParameters().forEach(param -> { - query.setParameter(param.getName(), shouldEscape ? param.escapeMatching() : param.getValue()); - }); - } - } - } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 3e2b67b8dc..f532cfce27 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -8,8 +8,6 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.filter.dialect.RSQLFilterDialect; -import com.yahoo.elide.core.pagination.Pagination; -import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.dimension.TimeDimension; @@ -39,6 +37,7 @@ public class SQLQueryEngineTest { private EntityManagerFactory emf; private Schema playerStatsSchema; private EntityDictionary dictionary; + private RSQLFilterDialect filterParser; private Schema playerStatsSchema; private Schema playerStatsViewSchema; @@ -51,7 +50,6 @@ public SQLQueryEngineTest() { dictionary.bindEntity(PlayerStats.class); dictionary.bindEntity(PlayerStatsView.class); dictionary.bindEntity(Country.class); - dictionary.bindEntity(Player.class); filterParser = new RSQLFilterDialect(dictionary); playerStatsSchema = new SQLSchema(PlayerStats.class, dictionary); @@ -83,7 +81,6 @@ public void testFullTableLoad() throws Exception { stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); PlayerStats stats2 = new PlayerStats(); - stats2.setId("1"); stats2.setLowScore(241); stats2.setHighScore(2412); stats2.setOverallRating("Great"); @@ -100,9 +97,8 @@ public void testDegenerateDimensionFilter() throws Exception { QueryEngine engine = new SQLQueryEngine(em, dictionary); Query query = Query.builder() - .schema(playerStatsSchema) - .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) - .metric(playerStatsSchema.getMetric("highScore"), Sum.class) + .entityClass(PlayerStats.class) + .metrics(playerStatsSchema.getMetrics()) .groupDimension(playerStatsSchema.getDimension("overallRating")) .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) .whereFilter(filterParser.parseFilterExpression("overallRating==Great", @@ -113,7 +109,6 @@ public void testDegenerateDimensionFilter() throws Exception { .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); - stats1.setId("0"); stats1.setLowScore(241); stats1.setHighScore(2412); stats1.setOverallRating("Great"); @@ -121,248 +116,6 @@ public void testDegenerateDimensionFilter() throws Exception { Assert.assertEquals(results.size(), 1); Assert.assertEquals(results.get(0), stats1); - } - - @Test - public void testFilterJoin() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); - - Query query = Query.builder() - .schema(playerStatsSchema) - .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) - .metric(playerStatsSchema.getMetric("highScore"), Sum.class) - .groupDimension(playerStatsSchema.getDimension("overallRating")) - .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) - .whereFilter(filterParser.parseFilterExpression("country.name=='United States'", - PlayerStats.class, false)) - .build(); - - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) - .collect(Collectors.toList()); - - PlayerStats stats1 = new PlayerStats(); - stats1.setId("0"); - stats1.setLowScore(72); - stats1.setHighScore(1234); - stats1.setOverallRating("Good"); - stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); - - PlayerStats stats2 = new PlayerStats(); - stats2.setId("1"); - stats2.setLowScore(241); - stats2.setHighScore(2412); - stats2.setOverallRating("Great"); - stats2.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); - - Assert.assertEquals(results.size(), 2); - Assert.assertEquals(results.get(0), stats1); - Assert.assertEquals(results.get(1), stats2); - } - - @Test - public void testSubqueryFilterJoin() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); - - Query query = Query.builder() - .schema(playerStatsViewSchema) - .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) - .whereFilter(filterParser.parseFilterExpression("player.name=='Jane Doe'", - PlayerStatsView.class, false)) - .build(); - - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) - .collect(Collectors.toList()); - - PlayerStatsView stats2 = new PlayerStatsView(); - stats2.setId("0"); - stats2.setHighScore(2412); - - Assert.assertEquals(results.size(), 1); - Assert.assertEquals(results.get(0), stats2); - } - - @Test - public void testSubqueryLoad() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); - - - Query query = Query.builder() - .schema(playerStatsViewSchema) - .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) - .build(); - - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) - .collect(Collectors.toList()); - - PlayerStatsView stats2 = new PlayerStatsView(); - stats2.setId("0"); - stats2.setHighScore(2412); - - Assert.assertEquals(results.size(), 1); - Assert.assertEquals(results.get(0), stats2); - } - - @Test - public void testSortJoin() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); - - Map sortMap = new TreeMap<>(); - sortMap.put("player.name", Sorting.SortOrder.asc); - - Query query = Query.builder() - .schema(playerStatsSchema) - .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) - .groupDimension(playerStatsSchema.getDimension("overallRating")) - .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) - .sorting(new Sorting(sortMap)) - .build(); - - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) - .collect(Collectors.toList()); - - PlayerStats stats1 = new PlayerStats(); - stats1.setId("0"); - stats1.setLowScore(241); - stats1.setOverallRating("Great"); - stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); - - PlayerStats stats2 = new PlayerStats(); - stats2.setId("1"); - stats2.setLowScore(72); - stats2.setOverallRating("Good"); - stats2.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); - - Assert.assertEquals(results.size(), 2); - Assert.assertEquals(results.get(0), stats1); - Assert.assertEquals(results.get(1), stats2); - } - - @Test - public void testPagination() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); - - Pagination pagination = Pagination.fromOffsetAndLimit(1, 0, true); - - Query query = Query.builder() - .schema(playerStatsSchema) - .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) - .metric(playerStatsSchema.getMetric("highScore"), Sum.class) - .groupDimension(playerStatsSchema.getDimension("overallRating")) - .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) - .pagination(pagination) - .build(); - - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) - .collect(Collectors.toList()); - - //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 - PlayerStats stats1 = new PlayerStats(); - stats1.setId("0"); - stats1.setLowScore(72); - stats1.setHighScore(1234); - stats1.setOverallRating("Good"); - stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); - - Assert.assertEquals(results.size(), 1, "Number of records returned does not match"); - Assert.assertEquals(results.get(0), stats1, "Returned record does not match"); - Assert.assertEquals(pagination.getPageTotals(), 2, "Page totals does not match"); - } - - @Test - public void testHavingClause() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); - - Query query = Query.builder() - .schema(playerStatsSchema) - .metric(playerStatsSchema.getMetric("highScore"), Sum.class) - .havingFilter(filterParser.parseFilterExpression("highScore > 300", - PlayerStats.class, false)) - .build(); - - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) - .collect(Collectors.toList()); - - //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 - PlayerStats stats1 = new PlayerStats(); - stats1.setId("0"); - stats1.setHighScore(3646); - - Assert.assertEquals(results.size(), 1); - Assert.assertEquals(results.get(0), stats1); - } - - @Test - public void testTheEverythingQuery() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); - - Map sortMap = new TreeMap<>(); - sortMap.put("player.name", Sorting.SortOrder.asc); - - Query query = Query.builder() - .schema(playerStatsViewSchema) - .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) - .groupDimension(playerStatsViewSchema.getDimension("countryName")) - .whereFilter(filterParser.parseFilterExpression("player.name=='Jane Doe'", - PlayerStatsView.class, false)) - .havingFilter(filterParser.parseFilterExpression("highScore > 300", - PlayerStatsView.class, false)) - .sorting(new Sorting(sortMap)) - .build(); - - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) - .collect(Collectors.toList()); - - PlayerStatsView stats2 = new PlayerStatsView(); - stats2.setId("0"); - stats2.setHighScore(2412); - stats2.setCountryName("United States"); - - - Assert.assertEquals(results.size(), 1); - Assert.assertEquals(results.get(0), stats2); - } - - @Test - public void testSortByMultipleColumns() throws Exception { - EntityManager em = emf.createEntityManager(); - QueryEngine engine = new SQLQueryEngine(em, dictionary); - - Map sortMap = new TreeMap<>(); - sortMap.put("lowScore", Sorting.SortOrder.desc); - sortMap.put("player.name", Sorting.SortOrder.asc); - - Query query = Query.builder() - .schema(playerStatsSchema) - .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) - .groupDimension(playerStatsSchema.getDimension("overallRating")) - .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) - .sorting(new Sorting(sortMap)) - .build(); - - List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) - .collect(Collectors.toList()); - - PlayerStats stats1 = new PlayerStats(); - stats1.setId("0"); - stats1.setLowScore(241); - stats1.setOverallRating("Great"); - stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); - - PlayerStats stats2 = new PlayerStats(); - stats2.setId("1"); - stats2.setLowScore(72); - stats2.setOverallRating("Good"); - stats2.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); - - Assert.assertEquals(results.size(), 2); - Assert.assertEquals(results.get(0), stats1); Assert.assertEquals(results.get(1), stats2); } } From e5977a301cb2ca722af77db325f5ad6ebf4ea22d Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sat, 13 Jul 2019 19:58:03 -0500 Subject: [PATCH 10/47] Added working test for subquery SQL --- .../engine/SQLQueryEngineTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index f532cfce27..c1c6e60aa6 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -36,6 +36,7 @@ public class SQLQueryEngineTest { private EntityManagerFactory emf; private Schema playerStatsSchema; + private Schema playerStatsViewSchema; private EntityDictionary dictionary; private RSQLFilterDialect filterParser; @@ -118,4 +119,24 @@ public void testDegenerateDimensionFilter() throws Exception { Assert.assertEquals(results.get(0), stats1); Assert.assertEquals(results.get(1), stats2); } + + @Test + public void testSubqueryLoad() throws Exception { + EntityManager em = emf.createEntityManager(); + QueryEngine engine = new SQLQueryEngine(em, dictionary); + + Query query = Query.builder() + .entityClass(PlayerStatsView.class) + .metrics(playerStatsViewSchema.getMetrics()) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStatsView stats2 = new PlayerStatsView(); + stats2.setHighScore(2412); + + Assert.assertEquals(results.size(), 1); + Assert.assertEquals(results.get(0), stats2); + } } From 3057a9aed7c45ea75906ba3842c3f469df3a3e3b Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sun, 14 Jul 2019 10:27:59 -0500 Subject: [PATCH 11/47] Added basic join logic for filters --- .../engine/SQLQueryEngineTest.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index c1c6e60aa6..98e101ab9f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -120,6 +120,40 @@ public void testDegenerateDimensionFilter() throws Exception { Assert.assertEquals(results.get(1), stats2); } + @Test + public void testFilterJoin() throws Exception { + EntityManager em = emf.createEntityManager(); + QueryEngine engine = new SQLQueryEngine(em, dictionary); + + Query query = Query.builder() + .entityClass(PlayerStats.class) + .metrics(playerStatsSchema.getMetrics()) + .groupDimension(playerStatsSchema.getDimension("overallRating")) + .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) + .whereFilter(filterParser.parseFilterExpression("country.name=='United States'", + PlayerStats.class, false)) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStats stats1 = new PlayerStats(); + stats1.setLowScore(72); + stats1.setHighScore(1234); + stats1.setOverallRating("Good"); + stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); + + PlayerStats stats2 = new PlayerStats(); + stats2.setLowScore(241); + stats2.setHighScore(2412); + stats2.setOverallRating("Great"); + stats2.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); + + Assert.assertEquals(results.size(), 2); + Assert.assertEquals(results.get(0), stats1); + Assert.assertEquals(results.get(1), stats2); + } + @Test public void testSubqueryLoad() throws Exception { EntityManager em = emf.createEntityManager(); From ac3c5188ccf9a1cafb48a6fbde7ee4896d91f802 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sun, 14 Jul 2019 13:44:27 -0500 Subject: [PATCH 12/47] Added a test with a subquery and a filter join --- .../aggregation/engine/SQLQueryEngine.java | 2 ++ .../aggregation/engine/schema/SQLSchema.java | 3 +++ .../engine/SQLQueryEngineTest.java | 23 +++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index ec33c85e4f..a7ec05583b 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -59,6 +59,8 @@ public class SQLQueryEngine implements QueryEngine { @Getter private Map, SQLSchema> schemas; + private static final String SUBQUERY = "__SUBQUERY__"; + //Function to return the alias to apply to a filter predicate expression. private static final Function ALIAS_PROVIDER = (predicate) -> { List elements = predicate.getPath().getPathElements(); diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java index 8b2395f5c3..889ad24c35 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java @@ -24,6 +24,9 @@ public class SQLSchema extends Schema { @Getter private boolean isSubquery; + @Getter + private boolean isSubquery; + @Getter private String tableDefinition; diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 98e101ab9f..52e79c7d70 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -51,6 +51,7 @@ public SQLQueryEngineTest() { dictionary.bindEntity(PlayerStats.class); dictionary.bindEntity(PlayerStatsView.class); dictionary.bindEntity(Country.class); + dictionary.bindEntity(Player.class); filterParser = new RSQLFilterDialect(dictionary); playerStatsSchema = new SQLSchema(PlayerStats.class, dictionary); @@ -154,6 +155,28 @@ public void testFilterJoin() throws Exception { Assert.assertEquals(results.get(1), stats2); } + @Test + public void testSubqueryFilterJoin() throws Exception { + EntityManager em = emf.createEntityManager(); + QueryEngine engine = new SQLQueryEngine(em, dictionary); + + Query query = Query.builder() + .entityClass(PlayerStatsView.class) + .metrics(playerStatsViewSchema.getMetrics()) + .whereFilter(filterParser.parseFilterExpression("player.name=='Jane Doe'", + PlayerStatsView.class, false)) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStatsView stats2 = new PlayerStatsView(); + stats2.setHighScore(2412); + + Assert.assertEquals(results.size(), 1); + Assert.assertEquals(results.get(0), stats2); + } + @Test public void testSubqueryLoad() throws Exception { EntityManager em = emf.createEntityManager(); From c4094dff2c9e5ae9506bc7aad745413ec51d1d2b Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sun, 14 Jul 2019 15:39:15 -0500 Subject: [PATCH 13/47] Refactored Schema classes and Query to support metric aggregation SQL expansion --- .../yahoo/elide/datastores/aggregation/Schema.java | 13 +++++++++++++ .../aggregation/dimension/DegenerateDimension.java | 1 + .../aggregation/dimension/EntityDimension.java | 1 + .../aggregation/dimension/TimeDimension.java | 1 + .../aggregation/metric/AggregatedMetric.java | 1 + .../aggregation/engine/SQLQueryEngineTest.java | 10 ++++++---- 6 files changed, 23 insertions(+), 4 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java index 6b4f8c3631..dfdae6fd04 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java @@ -7,6 +7,7 @@ import com.yahoo.elide.core.DataStore; import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.core.filter.FilterPredicate; import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; import com.yahoo.elide.datastores.aggregation.annotation.Meta; import com.yahoo.elide.datastores.aggregation.annotation.MetricAggregation; @@ -148,6 +149,14 @@ public boolean isMetricField(String fieldName) { ); } + /** + * An alias to assign this schema. + * @return an alias that can be used in SQL. + */ + public String getAlias() { + return FilterPredicate.getTypeAlias(entityClass); + } + /** * Constructs a new {@link Metric} instance. * @@ -178,6 +187,7 @@ protected Metric constructMetric(String metricField, Class cls, EntityDiction ); return new AggregatedMetric( + this, metricField, metaData, fieldType, @@ -212,6 +222,7 @@ protected Dimension constructDimension(String dimensionField, Class cls, Enti if (entityDictionary.isRelation(cls, dimensionField)) { // relationship field return new EntityDimension( + this, dimensionField, metaData, fieldType, @@ -221,6 +232,7 @@ protected Dimension constructDimension(String dimensionField, Class cls, Enti } else if (!getEntityDictionary().attributeOrRelationAnnotationExists(cls, dimensionField, Temporal.class)) { // regular field return new DegenerateDimension( + this, dimensionField, metaData, fieldType, @@ -237,6 +249,7 @@ protected Dimension constructDimension(String dimensionField, Class cls, Enti ); return new TimeDimension( + this, dimensionField, metaData, fieldType, diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java index 22f84dc885..84fa2c51c7 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java @@ -6,6 +6,7 @@ package com.yahoo.elide.datastores.aggregation.dimension; import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; import com.yahoo.elide.datastores.aggregation.annotation.Meta; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java index f95726862f..21eeff5569 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java @@ -7,6 +7,7 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.datastores.aggregation.Column; +import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.annotation.Cardinality; import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; import com.yahoo.elide.datastores.aggregation.annotation.FriendlyName; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java index 5e3331ff04..44d01f0f17 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java @@ -5,6 +5,7 @@ */ package com.yahoo.elide.datastores.aggregation.dimension; +import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; import com.yahoo.elide.datastores.aggregation.annotation.Meta; import com.yahoo.elide.datastores.aggregation.schema.Schema; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java index f8149c1208..a3d2c92405 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java @@ -6,6 +6,7 @@ package com.yahoo.elide.datastores.aggregation.metric; import com.yahoo.elide.datastores.aggregation.Column; +import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.annotation.Meta; import com.yahoo.elide.datastores.aggregation.annotation.MetricAggregation; diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 52e79c7d70..f6fb76cf86 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -100,7 +100,8 @@ public void testDegenerateDimensionFilter() throws Exception { Query query = Query.builder() .entityClass(PlayerStats.class) - .metrics(playerStatsSchema.getMetrics()) + .metric(playerStatsSchema.getMetric("lowScore"), new Sum()) + .metric(playerStatsSchema.getMetric("highScore"), new Sum()) .groupDimension(playerStatsSchema.getDimension("overallRating")) .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) .whereFilter(filterParser.parseFilterExpression("overallRating==Great", @@ -128,7 +129,8 @@ public void testFilterJoin() throws Exception { Query query = Query.builder() .entityClass(PlayerStats.class) - .metrics(playerStatsSchema.getMetrics()) + .metric(playerStatsSchema.getMetric("lowScore"), new Sum()) + .metric(playerStatsSchema.getMetric("highScore"), new Sum()) .groupDimension(playerStatsSchema.getDimension("overallRating")) .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) .whereFilter(filterParser.parseFilterExpression("country.name=='United States'", @@ -162,7 +164,7 @@ public void testSubqueryFilterJoin() throws Exception { Query query = Query.builder() .entityClass(PlayerStatsView.class) - .metrics(playerStatsViewSchema.getMetrics()) + .metric(playerStatsSchema.getMetric("highScore"), new Sum()) .whereFilter(filterParser.parseFilterExpression("player.name=='Jane Doe'", PlayerStatsView.class, false)) .build(); @@ -184,7 +186,7 @@ public void testSubqueryLoad() throws Exception { Query query = Query.builder() .entityClass(PlayerStatsView.class) - .metrics(playerStatsViewSchema.getMetrics()) + .metric(playerStatsSchema.getMetric("highScore"), new Sum()) .build(); List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) From fef2e6d932495cfa31695ec901aa4076e6fc86e1 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sun, 14 Jul 2019 16:58:30 -0500 Subject: [PATCH 14/47] Added group by support --- .../aggregation/engine/SQLQueryEngine.java | 100 ++++++++++++------ .../engine/SQLQueryEngineTest.java | 14 +-- 2 files changed, 72 insertions(+), 42 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index a7ec05583b..755b649574 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -29,8 +29,11 @@ import com.yahoo.elide.utils.coerce.CoerceUtil; import com.google.common.base.Preconditions; -import org.apache.commons.lang3.mutable.MutableInt; -import lombok.Getter; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.dialect.H2SqlDialect; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.sql.parser.SqlParser; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; @@ -38,6 +41,7 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -87,6 +91,11 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) )); } + public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { + //this(entityManager, dictionary, CalciteSqlDialect.DEFAULT); + this(entityManager, dictionary, new H2SqlDialect(SqlDialect.EMPTY_CONTEXT)); + } + @Override public Iterable executeQuery(Query query) { SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); @@ -97,18 +106,28 @@ public Iterable executeQuery(Query query) { //Translate the query into SQL. SQLQuery sql = toSQL(query); - javax.persistence.Query jpaQuery = entityManager.createNativeQuery(sql.toString()); + List metricProjections = query.getMetrics().entrySet().stream() + .map((entry) -> entry.getKey()) + .map(Metric::getName) + .map((name) -> "__" + name.toUpperCase(Locale.ENGLISH) + "__") + .collect(Collectors.toList()); - Pagination pagination = query.getPagination(); - if (pagination != null) { - jpaQuery.setFirstResult(pagination.getOffset()); - jpaQuery.setMaxResults(pagination.getLimit()); + List dimensionProjections = query.getGroupDimensions().stream() + .map(Dimension::getName) + .collect(Collectors.toList()); - if (pagination.isGenerateTotals()) { + dimensionProjections.addAll(query.getTimeDimensions().stream() + .map(Dimension::getName) + .collect(Collectors.toList())); - SQLQuery paginationSQL = toPageTotalSQL(sql); - javax.persistence.Query pageTotalQuery = - entityManager.createNativeQuery(paginationSQL.toString()); + String projectionClause = metricProjections.stream() + .collect(Collectors.joining(",")); + + if (!dimensionProjections.isEmpty()) { + projectionClause = projectionClause + "," + dimensionProjections.stream() + .map((name) -> tableAlias + "." + name) + .collect(Collectors.joining(",")); + } //Supply the query parameters to the query supplyFilterQueryParameters(query, pageTotalQuery); @@ -122,20 +141,25 @@ public Iterable executeQuery(Query query) { } } - //Supply the query parameters to the query - supplyFilterQueryParameters(query, jpaQuery); + if (!dimensionProjections.isEmpty()) { + sql += " GROUP BY "; + sql += dimensionProjections.stream() + .map((name) -> tableAlias + "." + name) + .collect(Collectors.joining(",")); + } + + String nativeSql = translateSqlToNative(sql, dialect); + + nativeSql = expandMetricTemplates(nativeSql, query.getMetrics()); - //Run the primary query and log the time spent. - List results = new TimedFunction<>(() -> { - return jpaQuery.getResultList(); - }, "Running Query: " + sql).get(); + log.debug("Running native SQL query: {}", nativeSql); //Coerce the results into entity objects. MutableInt counter = new MutableInt(0); return results.stream() .map((result) -> { return result instanceof Object[] ? (Object []) result : new Object[] { result }; }) - .map((result) -> coerceObjectToEntity(query, result, counter)) + .map((result) -> coerceObjectToEntity(query, result)) .collect(Collectors.toList()); } @@ -191,26 +215,24 @@ protected SQLQuery toSQL(Query query) { return builder.build(); } - /** - * Coerces results from a JPA query into an Object. - * @param query The client query - * @param result A row from the results. - * @param counter Monotonically increasing number to generate IDs. - * @return A hydrated entity object. - */ - protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt counter) { - Class entityClass = query.getSchema().getEntityClass(); + protected Object coerceObjectToEntity(Query query, Object[] result) { - //Get all the projections from the client query. + Class entityClass = query.getEntityClass(); List projections = query.getMetrics().entrySet().stream() .map(Map.Entry::getKey) .map(Metric::getName) .collect(Collectors.toList()); - projections.addAll(query.getDimensions().stream() + projections.addAll(query.getGroupDimensions().stream() .map(Dimension::getName) .collect(Collectors.toList())); + projections.addAll(query.getTimeDimensions().stream() + .map(Dimension::getName) + .collect(Collectors.toList())); + + SQLSchema schema = schemas.get(entityClass); + Preconditions.checkArgument(result.length == projections.size()); SQLSchema schema = (SQLSchema) query.getSchema(); @@ -258,12 +280,20 @@ private String translateFilterExpression(SQLSchema schema, return filterVisitor.apply(expression, columnGenerator); } - /** - * Given a filter expression, extracts any entity relationship traversals that require joins. - * @param expression The filter expression - * @return A set of path elements that capture a relationship traversal. - */ - private Set extractPathElements(FilterExpression expression) { + private String expandMetricTemplates(String sql, Map> metrics) { + String expanded = sql; + for (Map.Entry entry : metrics.entrySet()) { + Metric metric = (Metric) entry.getKey(); + Class agg = (Class) entry.getValue(); + + expanded = expanded.replaceFirst( + "__" + metric.getName().toUpperCase(Locale.ENGLISH) + "__", + metric.getMetricExpression(Optional.of(agg)) + " AS " + metric.getName()); + } + return expanded; + } + + private String extractJoin(FilterExpression expression) { Collection predicates = expression.accept(new PredicateExtractionVisitor()); return predicates.stream() diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index f6fb76cf86..d35c89bc89 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -64,7 +64,7 @@ public void testFullTableLoad() throws Exception { QueryEngine engine = new SQLQueryEngine(em, dictionary); Query query = Query.builder() - .schema(playerStatsSchema) + .entityClass(PlayerStats.class) .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) @@ -100,8 +100,8 @@ public void testDegenerateDimensionFilter() throws Exception { Query query = Query.builder() .entityClass(PlayerStats.class) - .metric(playerStatsSchema.getMetric("lowScore"), new Sum()) - .metric(playerStatsSchema.getMetric("highScore"), new Sum()) + .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) .whereFilter(filterParser.parseFilterExpression("overallRating==Great", @@ -129,8 +129,8 @@ public void testFilterJoin() throws Exception { Query query = Query.builder() .entityClass(PlayerStats.class) - .metric(playerStatsSchema.getMetric("lowScore"), new Sum()) - .metric(playerStatsSchema.getMetric("highScore"), new Sum()) + .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) .whereFilter(filterParser.parseFilterExpression("country.name=='United States'", @@ -164,7 +164,7 @@ public void testSubqueryFilterJoin() throws Exception { Query query = Query.builder() .entityClass(PlayerStatsView.class) - .metric(playerStatsSchema.getMetric("highScore"), new Sum()) + .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) .whereFilter(filterParser.parseFilterExpression("player.name=='Jane Doe'", PlayerStatsView.class, false)) .build(); @@ -186,7 +186,7 @@ public void testSubqueryLoad() throws Exception { Query query = Query.builder() .entityClass(PlayerStatsView.class) - .metric(playerStatsSchema.getMetric("highScore"), new Sum()) + .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) .build(); List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) From 0ea81821be18b0423e55a8c19c2afef3a062df86 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sun, 14 Jul 2019 20:15:28 -0500 Subject: [PATCH 15/47] Added logic for ID generation --- .../datastores/aggregation/engine/SQLQueryEngine.java | 7 ++++--- .../datastores/aggregation/engine/SQLQueryEngineTest.java | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 755b649574..1f2c327c2f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -34,6 +34,7 @@ import org.apache.calcite.sql.dialect.H2SqlDialect; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.commons.lang3.mutable.MutableInt; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; @@ -155,11 +156,11 @@ public Iterable executeQuery(Query query) { log.debug("Running native SQL query: {}", nativeSql); - //Coerce the results into entity objects. MutableInt counter = new MutableInt(0); + return results.stream() .map((result) -> { return result instanceof Object[] ? (Object []) result : new Object[] { result }; }) - .map((result) -> coerceObjectToEntity(query, result)) + .map((result) -> coerceObjectToEntity(query, result, counter)) .collect(Collectors.toList()); } @@ -215,7 +216,7 @@ protected SQLQuery toSQL(Query query) { return builder.build(); } - protected Object coerceObjectToEntity(Query query, Object[] result) { + protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt counter) { Class entityClass = query.getEntityClass(); List projections = query.getMetrics().entrySet().stream() diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index d35c89bc89..3e3fe4cbe7 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -83,6 +83,7 @@ public void testFullTableLoad() throws Exception { stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); PlayerStats stats2 = new PlayerStats(); + stats2.setId("1"); stats2.setLowScore(241); stats2.setHighScore(2412); stats2.setOverallRating("Great"); @@ -112,6 +113,7 @@ public void testDegenerateDimensionFilter() throws Exception { .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); stats1.setLowScore(241); stats1.setHighScore(2412); stats1.setOverallRating("Great"); @@ -141,12 +143,14 @@ public void testFilterJoin() throws Exception { .collect(Collectors.toList()); PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); stats1.setLowScore(72); stats1.setHighScore(1234); stats1.setOverallRating("Good"); stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); PlayerStats stats2 = new PlayerStats(); + stats2.setId("1"); stats2.setLowScore(241); stats2.setHighScore(2412); stats2.setOverallRating("Great"); @@ -173,6 +177,7 @@ public void testSubqueryFilterJoin() throws Exception { .collect(Collectors.toList()); PlayerStatsView stats2 = new PlayerStatsView(); + stats2.setId("0"); stats2.setHighScore(2412); Assert.assertEquals(results.size(), 1); @@ -193,6 +198,7 @@ public void testSubqueryLoad() throws Exception { .collect(Collectors.toList()); PlayerStatsView stats2 = new PlayerStatsView(); + stats2.setId("0"); stats2.setHighScore(2412); Assert.assertEquals(results.size(), 1); From adaa4a6044f97f1bc5f481fc2f4cebfaefc35371 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Sun, 14 Jul 2019 22:23:23 -0500 Subject: [PATCH 16/47] Added sorting logic and test --- .../aggregation/engine/SQLQueryEngine.java | 80 ++++++++----------- .../engine/SQLQueryEngineTest.java | 38 ++++++++- 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 1f2c327c2f..a7299e3a89 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -14,7 +14,6 @@ import com.yahoo.elide.core.filter.HQLFilterOperation; import com.yahoo.elide.core.filter.expression.FilterExpression; import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor; -import com.yahoo.elide.core.pagination.Pagination; import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; @@ -39,8 +38,7 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashSet; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -93,7 +91,6 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) } public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { - //this(entityManager, dictionary, CalciteSqlDialect.DEFAULT); this(entityManager, dictionary, new H2SqlDialect(SqlDialect.EMPTY_CONTEXT)); } @@ -107,6 +104,15 @@ public Iterable executeQuery(Query query) { //Translate the query into SQL. SQLQuery sql = toSQL(query); + Map sortClauses = (query.getSorting() == null) + ? new HashMap<>() + : query.getSorting().getValidSortingRules(query.getEntityClass(), dictionary); + + String joinClause = ""; + String whereClause = ""; + String orderByClause = ""; + String groupByClause = ""; + List metricProjections = query.getMetrics().entrySet().stream() .map((entry) -> entry.getKey()) .map(Metric::getName) @@ -130,25 +136,30 @@ public Iterable executeQuery(Query query) { .collect(Collectors.joining(",")); } - //Supply the query parameters to the query - supplyFilterQueryParameters(query, pageTotalQuery); - //Run the Pagination query and log the time spent. - long total = new TimedFunction<>(() -> { - return CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); - }, "Running Query: " + paginationSQL).get(); - - pagination.setPageTotals(total); - } + if (query.getWhereFilter() != null) { + joinClause = " " + extractJoin(query.getWhereFilter()); + whereClause = " " + translateFilterExpression(schema, query.getWhereFilter()); } if (!dimensionProjections.isEmpty()) { - sql += " GROUP BY "; - sql += dimensionProjections.stream() + groupByClause = " GROUP BY "; + groupByClause += dimensionProjections.stream() .map((name) -> tableAlias + "." + name) .collect(Collectors.joining(",")); } + if (query.getSorting() != null) { + orderByClause = " " + extractOrderBy(query.getEntityClass(), sortClauses); + joinClause += " " + extractJoin(sortClauses); + } + + String sql = String.format("SELECT %s FROM %s AS %s", projectionClause, tableName, tableAlias) + + joinClause + + whereClause + + groupByClause + + orderByClause; + String nativeSql = translateSqlToNative(sql, dialect); nativeSql = expandMetricTemplates(nativeSql, query.getMetrics()); @@ -335,29 +346,19 @@ private String extractJoin(Path.PathElement pathElement) { relationshipIdField); } - /** - * Given a list of columns to sort on, extracts any entity relationship traversals that require joins. - * @param sortClauses The list of sort columns and their sort order (ascending or descending). - * @return A set of path elements that capture a relationship traversal. - */ - private Set extractPathElements(Map sortClauses) { + private String extractJoin(Map sortClauses) { if (sortClauses.isEmpty()) { - return new LinkedHashSet<>(); + return ""; } return sortClauses.entrySet().stream() .map(Map.Entry::getKey) .flatMap((path) -> path.getPathElements().stream()) - .filter((element) -> dictionary.isRelation(element.getType(), element.getFieldName())) - .collect(Collectors.toCollection(LinkedHashSet::new)); + .filter((predicate) -> dictionary.isRelation(predicate.getType(), predicate.getFieldName())) + .map(this::extractJoin) + .collect(Collectors.joining(" ")); } - /** - * Given a list of columns to sort on, constructs an ORDER BY clause in SQL. - * @param entityClass The class to sort. - * @param sortClauses The list of sort columns and their sort order (ascending or descending). - * @return A SQL expression - */ private String extractOrderBy(Class entityClass, Map sortClauses) { if (sortClauses.isEmpty()) { return ""; @@ -377,22 +378,9 @@ private String extractOrderBy(Class entityClass, Map }).collect(Collectors.joining(",")); } - /** - * Given a JPA query, replaces any parameters with their values from client query. - * @param query The client query - * @param jpaQuery The JPA query - */ - private void supplyFilterQueryParameters(Query query, - javax.persistence.Query jpaQuery) { - - Collection predicates = new ArrayList<>(); - if (query.getWhereFilter() != null) { - predicates.addAll(query.getWhereFilter().accept(new PredicateExtractionVisitor())); - } - - if (query.getHavingFilter() != null) { - predicates.addAll(query.getHavingFilter().accept(new PredicateExtractionVisitor())); - } + private void supplyFilterQueryParameters(FilterExpression expression, + javax.persistence.Query query) { + Collection predicates = expression.accept(new PredicateExtractionVisitor()); for (FilterPredicate filterPredicate : predicates) { if (filterPredicate.getOperator().isParameterized()) { diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 3e3fe4cbe7..25e51c6f65 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -8,6 +8,7 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.filter.dialect.RSQLFilterDialect; +import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.dimension.TimeDimension; @@ -25,7 +26,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import javax.persistence.EntityManager; @@ -204,4 +204,40 @@ public void testSubqueryLoad() throws Exception { Assert.assertEquals(results.size(), 1); Assert.assertEquals(results.get(0), stats2); } + + @Test + public void testSortJoin() throws Exception { + EntityManager em = emf.createEntityManager(); + QueryEngine engine = new SQLQueryEngine(em, dictionary); + + Map sortMap = new HashMap<>(); + sortMap.put("player.name", Sorting.SortOrder.asc); + + Query query = Query.builder() + .entityClass(PlayerStats.class) + .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) + .groupDimension(playerStatsSchema.getDimension("overallRating")) + .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) + .sorting(new Sorting(sortMap)) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); + stats1.setLowScore(241); + stats1.setOverallRating("Great"); + stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); + + PlayerStats stats2 = new PlayerStats(); + stats2.setId("1"); + stats2.setLowScore(72); + stats2.setOverallRating("Good"); + stats2.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); + + Assert.assertEquals(results.size(), 2); + Assert.assertEquals(results.get(0), stats1); + Assert.assertEquals(results.get(1), stats2); + } } From e749cca48d22696d7d4094f934799675f061d7b2 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Mon, 15 Jul 2019 11:31:43 -0500 Subject: [PATCH 17/47] Added pagination support and testing --- .../elide/datastores/aggregation/Query.java | 9 +- .../aggregation/engine/SQLQueryEngine.java | 171 +++++------------- .../engine/SQLQueryEngineTest.java | 33 ++++ 3 files changed, 85 insertions(+), 128 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java index f27a68738b..f1e7052316 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java @@ -19,7 +19,7 @@ import lombok.Data; import lombok.Singular; -import java.util.LinkedHashSet; +import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -55,7 +55,10 @@ public class Query { public Set getDimensions() { return Stream.concat(getGroupDimensions().stream(), getTimeDimensions().stream()) .collect( - Collectors.toCollection(LinkedHashSet::new) + Collectors.collectingAndThen( + Collectors.toSet(), + Collections::unmodifiableSet + ) ); } -} \ No newline at end of file +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index a7299e3a89..e0edb7ccd9 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -14,6 +14,7 @@ import com.yahoo.elide.core.filter.HQLFilterOperation; import com.yahoo.elide.core.filter.expression.FilterExpression; import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor; +import com.yahoo.elide.core.pagination.Pagination; import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; @@ -24,7 +25,6 @@ import com.yahoo.elide.datastores.aggregation.engine.schema.SQLSchema; import com.yahoo.elide.datastores.aggregation.metric.Aggregation; import com.yahoo.elide.datastores.aggregation.metric.Metric; -import com.yahoo.elide.datastores.aggregation.schema.Schema; import com.yahoo.elide.utils.coerce.CoerceUtil; import com.google.common.base.Preconditions; @@ -154,7 +154,9 @@ public Iterable executeQuery(Query query) { joinClause += " " + extractJoin(sortClauses); } - String sql = String.format("SELECT %s FROM %s AS %s", projectionClause, tableName, tableAlias) + String fromClause = String.format("%s AS %s", tableName, tableAlias); + + String sql = String.format("SELECT %s FROM %s", projectionClause, fromClause) + joinClause + whereClause + groupByClause @@ -166,6 +168,15 @@ public Iterable executeQuery(Query query) { log.debug("Running native SQL query: {}", nativeSql); + javax.persistence.Query jpaQuery = entityManager.createNativeQuery(nativeSql); + + paginate(query, jpaQuery, fromClause, joinClause, whereClause); + + if (query.getWhereFilter() != null) { + supplyFilterQueryParameters(query.getWhereFilter(), jpaQuery); + } + + List results = jpaQuery.getResultList(); MutableInt counter = new MutableInt(0); @@ -417,137 +428,47 @@ private String getColumnName(Class entityClass, String fieldName) { } /** - * Takes a SQLQuery and creates a new clone that instead returns the total number of records of the original - * query. - * @param sql The original query - * @return A new query that returns the total number of records. - */ - private SQLQuery toPageTotalSQL(SQLQuery sql) { - Query clientQuery = sql.getClientQuery(); - - String groupByDimensions = clientQuery.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name)) - .collect(Collectors.joining(",")); - - String projectionClause = String.format("COUNT(DISTINCT(%s))", groupByDimensions); - - return SQLQuery.builder() - .clientQuery(sql.getClientQuery()) - .projectionClause(projectionClause) - .fromClause(sql.getFromClause()) - .joinClause(sql.getJoinClause()) - .whereClause(sql.getWhereClause()) - .build(); - } - - /** - * Given a client query, constructs the list of columns to project from a database table. - * @param query The client query - * @return A SQL fragment to use in the SELECT .. statement. + * Paginates the query if requested. + * @param query The QueryEngine query + * @param jpaQuery The JPA query */ - private String extractProjection(Query query) { - List metricProjections = query.getMetrics().entrySet().stream() - .map((entry) -> { - Metric metric = entry.getKey(); - Class agg = entry.getValue(); - return metric.getMetricExpression(Optional.of(agg)) + " AS " + metric.getName(); - }) - .collect(Collectors.toList()); - - List dimensionProjections = query.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(query.getSchema().getEntityClass(), name)) - .collect(Collectors.toList()); - - String projectionClause = metricProjections.stream() - .collect(Collectors.joining(",")); - - if (!dimensionProjections.isEmpty()) { - projectionClause = projectionClause + "," + dimensionProjections.stream() - .map((name) -> query.getSchema().getAlias() + "." + name) - .collect(Collectors.joining(",")); + private void paginate(Query query, + javax.persistence.Query jpaQuery, + String fromClause, + String joinClause, + String whereClause) { + Pagination pagination = query.getPagination(); + if (pagination == null) { + return; } - - return projectionClause; - } - - /** - * Extracts a GROUP BY SQL clause. - * @param query A client query - * @return The SQL GROUP BY clause - */ - private String extractGroupBy(Query query) { - List dimensionProjections = query.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(query.getSchema().getEntityClass(), name)) - .collect(Collectors.toList()); - - return "GROUP BY " + dimensionProjections.stream() - .map((name) -> query.getSchema().getAlias() + "." + name) + jpaQuery.setFirstResult(pagination.getOffset()); + jpaQuery.setMaxResults(pagination.getLimit()); + + /* + * TODO - this is a naive implementation. We should run these in parallel or combine the queries + * with a windowing function (if the DB supports that). + */ + if (pagination.isGenerateTotals()) { + String groupByDimensions = query.getDimensions().stream() + .map(Dimension::getName) + .map((name) -> getColumnName(query.getEntityClass(), name)) .collect(Collectors.joining(",")); - } + String sql = String.format("SELECT COUNT(DISTINCT(%s)) FROM %s %s %s", + groupByDimensions, + fromClause, + joinClause, + whereClause); - /** - * Converts a filter predicate into a SQL WHERE clause column reference. - * @param predicate The predicate to convert - * @return A SQL fragment that references a database column - */ - private String generateWhereClauseColumnReference(FilterPredicate predicate) { - Path.PathElement last = predicate.getPath().lastElement().get(); - Class lastClass = last.getType(); + javax.persistence.Query pageTotalQuery = entityManager.createNativeQuery(sql); - return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); - } - - /** - * Converts a filter predicate into a SQL HAVING clause column reference. - * @param predicate The predicate to convert - * @return A SQL fragment that references a database column - */ - private String generateHavingClauseColumnReference(FilterPredicate predicate, Query query) { - Path.PathElement last = predicate.getPath().lastElement().get(); - Class lastClass = last.getType(); - - if (!lastClass.equals(query.getSchema().getEntityClass())) { - throw new InvalidPredicateException("The having clause can only reference fact table aggregations."); - } - - Schema schema = schemas.get(lastClass); - Metric metric = schema.getMetric(last.getFieldName()); - Class agg = query.getMetrics().get(metric); - - return metric.getMetricExpression(Optional.of(agg)); - } - - protected Object coerceObjectToEntity(Class entityClass, List projections, Object[] result) { - SQLSchema schema = schemas.get(entityClass); - - Preconditions.checkNotNull(schema); - Preconditions.checkArgument(result.length == projections.size()); - - Object entityInstance; - try { - entityInstance = entityClass.newInstance(); - } catch (InstantiationException | IllegalAccessException e) { - throw new IllegalStateException(e); - } - - for (int idx = 0; idx < result.length; idx++) { - Object value = result[idx]; - String fieldName = projections.get(idx); - - Dimension dim = schema.getDimension(fieldName); - if (dim != null && dim.getDimensionType() == DimensionType.ENTITY) { - - //We don't hydrate relationships here. - continue; + if (query.getWhereFilter() != null) { + supplyFilterQueryParameters(query.getWhereFilter(), pageTotalQuery); } - dictionary.setValue(entityInstance, fieldName, value); - } + long total = CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); - return entityInstance; + pagination.setPageTotals(total); + } } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 25e51c6f65..6f672ed8e0 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -8,6 +8,7 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.filter.dialect.RSQLFilterDialect; +import com.yahoo.elide.core.pagination.Pagination; import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; @@ -240,4 +241,36 @@ public void testSortJoin() throws Exception { Assert.assertEquals(results.get(0), stats1); Assert.assertEquals(results.get(1), stats2); } + + @Test + public void testPagination() throws Exception { + EntityManager em = emf.createEntityManager(); + QueryEngine engine = new SQLQueryEngine(em, dictionary); + + Pagination pagination = Pagination.fromOffsetAndLimit(1, 0, true); + + Query query = Query.builder() + .entityClass(PlayerStats.class) + .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) + .groupDimension(playerStatsSchema.getDimension("overallRating")) + .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) + .pagination(pagination) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 + PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); + stats1.setLowScore(72); + stats1.setHighScore(1234); + stats1.setOverallRating("Good"); + stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); + + Assert.assertEquals(results.size(), 1, "Number of records returned does not match"); + Assert.assertEquals(results.get(0), stats1, "Returned record does not match"); + Assert.assertEquals(pagination.getPageTotals(), 2, "Page totals does not match"); + } } From 1414bb2ed34862c4a2376a0a3467f20f1a2712be Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Mon, 15 Jul 2019 20:37:38 -0500 Subject: [PATCH 18/47] All column references use proper name now for SQL --- .../elide/datastores/aggregation/Query.java | 7 +-- .../aggregation/engine/SQLQueryEngine.java | 30 +++++-------- .../elide/core/filter/HQLFilterOperation.java | 44 +++++++++---------- 3 files changed, 36 insertions(+), 45 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java index f1e7052316..70c50379af 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Query.java @@ -19,7 +19,7 @@ import lombok.Data; import lombok.Singular; -import java.util.Collections; +import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -55,10 +55,7 @@ public class Query { public Set getDimensions() { return Stream.concat(getGroupDimensions().stream(), getTimeDimensions().stream()) .collect( - Collectors.collectingAndThen( - Collectors.toSet(), - Collections::unmodifiableSet - ) + Collectors.toCollection(LinkedHashSet::new) ); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index e0edb7ccd9..c085cb3eef 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -64,13 +64,12 @@ public class SQLQueryEngine implements QueryEngine { private static final String SUBQUERY = "__SUBQUERY__"; - //Function to return the alias to apply to a filter predicate expression. - private static final Function ALIAS_PROVIDER = (predicate) -> { - List elements = predicate.getPath().getPathElements(); + //Converts a filter predicate into a SQL column reference + private final Function columnGenerator = (predicate) -> { + Path.PathElement last = predicate.getPath().lastElement().get(); + Class lastClass = last.getType(); - Path.PathElement last = elements.get(elements.size() - 1); - - return FilterPredicate.getTypeAlias(last.getType()); + return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); }; public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { @@ -116,17 +115,15 @@ public Iterable executeQuery(Query query) { List metricProjections = query.getMetrics().entrySet().stream() .map((entry) -> entry.getKey()) .map(Metric::getName) + .map((name) -> getColumnName(query.getEntityClass(), name)) .map((name) -> "__" + name.toUpperCase(Locale.ENGLISH) + "__") .collect(Collectors.toList()); - List dimensionProjections = query.getGroupDimensions().stream() + List dimensionProjections = query.getDimensions().stream() .map(Dimension::getName) + .map((name) -> getColumnName(query.getEntityClass(), name)) .collect(Collectors.toList()); - dimensionProjections.addAll(query.getTimeDimensions().stream() - .map(Dimension::getName) - .collect(Collectors.toList())); - String projectionClause = metricProjections.stream() .collect(Collectors.joining(",")); @@ -136,7 +133,6 @@ public Iterable executeQuery(Query query) { .collect(Collectors.joining(",")); } - if (query.getWhereFilter() != null) { joinClause = " " + extractJoin(query.getWhereFilter()); whereClause = " " + translateFilterExpression(schema, query.getWhereFilter()); @@ -246,11 +242,7 @@ protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt c .map(Metric::getName) .collect(Collectors.toList()); - projections.addAll(query.getGroupDimensions().stream() - .map(Dimension::getName) - .collect(Collectors.toList())); - - projections.addAll(query.getTimeDimensions().stream() + projections.addAll(query.getDimensions().stream() .map(Dimension::getName) .collect(Collectors.toList())); @@ -300,7 +292,9 @@ private String translateFilterExpression(SQLSchema schema, Function columnGenerator) { HQLFilterOperation filterVisitor = new HQLFilterOperation(); - return filterVisitor.apply(expression, columnGenerator); + String whereClause = filterVisitor.apply(expression, columnGenerator); + + return whereClause.replaceAll("(:[a-zA-Z0-9_]+)", "\'$1\'"); } private String expandMetricTemplates(String sql, Map> metrics) { diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java index d24f439fec..bd9863437b 100644 --- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java +++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java @@ -18,7 +18,6 @@ import com.google.common.base.Strings; import java.util.List; -import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -32,26 +31,27 @@ public class HQLFilterOperation implements FilterOperation { public static final Function LOWERED_PARAMETER = p -> String.format("lower(%s)", p.getPlaceholder()); + public static final Function GENERATE_HQL_COLUMN_NO_ALIAS = (predicate) -> { + return predicate.getFieldPath(); + }; + + public static final Function GENERATE_HQL_COLUMN_WITH_ALIAS = (predicate) -> { + return predicate.getAlias() + "." + predicate.getField(); + }; + @Override public String apply(FilterPredicate filterPredicate) { - return apply(filterPredicate, Optional.empty()); + return apply(filterPredicate, GENERATE_HQL_COLUMN_NO_ALIAS); } /** * Transforms a filter predicate into a HQL query fragment. * @param filterPredicate The predicate to transform. - * @param aliasProvider Function which supplies an alias to append to the predicate fields. - * This is useful for table aliases referenced in HQL for some kinds of joins. + * @param columnGenerator Function which supplies a HQL fragment which represents the column in the predicate. * @return The hql query fragment. */ - protected String apply(FilterPredicate filterPredicate, Optional> aliasProvider) { - String fieldPath = filterPredicate.getFieldPath(); - - if (aliasProvider.isPresent()) { - fieldPath = aliasProvider.get().apply(filterPredicate) + "." + filterPredicate.getField(); - } - - //HQL doesn't support 'this', but it does support aliases. + protected String apply(FilterPredicate filterPredicate, Function columnGenerator) { + String fieldPath = columnGenerator.apply(filterPredicate); fieldPath = fieldPath.replaceAll("\\.this", ""); List params = filterPredicate.getParameters(); @@ -161,22 +161,22 @@ private void assertValidValues(String fieldPath, String alias) { * @return A JPQL filter fragment. */ public String apply(FilterExpression filterExpression, boolean prefixWithAlias) { - Optional> aliasProvider = Optional.empty(); + Function columnGenerator = GENERATE_HQL_COLUMN_NO_ALIAS; if (prefixWithAlias) { - aliasProvider = Optional.of((predicate) -> predicate.getAlias()); + columnGenerator = GENERATE_HQL_COLUMN_WITH_ALIAS; } - return apply(filterExpression, aliasProvider); + return apply(filterExpression, columnGenerator); } /** * Translates the filterExpression to a JPQL filter fragment. * @param filterExpression The filterExpression to translate - * @param aliasProvider Optionally appends the predicates clause with an alias. + * @param columnGenerator Generates a HQL fragment that represents a column in the predicate * @return A JPQL filter fragment. */ - public String apply(FilterExpression filterExpression, Optional> aliasProvider) { - HQLQueryVisitor visitor = new HQLQueryVisitor(aliasProvider); + public String apply(FilterExpression filterExpression, Function columnGenerator) { + HQLQueryVisitor visitor = new HQLQueryVisitor(columnGenerator); return "WHERE " + filterExpression.accept(visitor); } @@ -186,15 +186,15 @@ public String apply(FilterExpression filterExpression, Optional { public static final String TWO_NON_FILTERING_EXPRESSIONS = "Cannot build a filter from two non-filtering expressions"; - private Optional> aliasProvider; + private Function columnGenerator; - public HQLQueryVisitor(Optional> aliasProvider) { - this.aliasProvider = aliasProvider; + public HQLQueryVisitor(Function columnGenerator) { + this.columnGenerator = columnGenerator; } @Override public String visitPredicate(FilterPredicate filterPredicate) { - return apply(filterPredicate, aliasProvider); + return apply(filterPredicate, columnGenerator); } @Override From fd6e9ee715ae968e0909a43a5d9252ee023f4dd6 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Mon, 15 Jul 2019 20:48:00 -0500 Subject: [PATCH 19/47] Removed calcite as a query engine --- .../aggregation/engine/SQLQueryEngine.java | 98 ++----------------- 1 file changed, 8 insertions(+), 90 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index c085cb3eef..3701030e0c 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -28,11 +28,6 @@ import com.yahoo.elide.utils.coerce.CoerceUtil; import com.google.common.base.Preconditions; -import org.apache.calcite.sql.SqlDialect; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.dialect.H2SqlDialect; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.calcite.sql.parser.SqlParser; import org.apache.commons.lang3.mutable.MutableInt; import lombok.extern.slf4j.Slf4j; @@ -40,7 +35,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -75,8 +69,6 @@ public class SQLQueryEngine implements QueryEngine { public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { this.entityManager = entityManager; this.dictionary = dictionary; - - // Construct the list of schemas that will be managed by this query engine. schemas = dictionary.getBindings() .stream() .filter((clazz) -> @@ -89,10 +81,6 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) )); } - public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { - this(entityManager, dictionary, new H2SqlDialect(SqlDialect.EMPTY_CONTEXT)); - } - @Override public Iterable executeQuery(Query query) { SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); @@ -113,10 +101,11 @@ public Iterable executeQuery(Query query) { String groupByClause = ""; List metricProjections = query.getMetrics().entrySet().stream() - .map((entry) -> entry.getKey()) - .map(Metric::getName) - .map((name) -> getColumnName(query.getEntityClass(), name)) - .map((name) -> "__" + name.toUpperCase(Locale.ENGLISH) + "__") + .map((entry) -> { + Metric metric = entry.getKey(); + Class agg = entry.getValue(); + return metric.getMetricExpression(Optional.of(agg)) + " AS " + metric.getName(); + }) .collect(Collectors.toList()); List dimensionProjections = query.getDimensions().stream() @@ -158,13 +147,9 @@ public Iterable executeQuery(Query query) { + groupByClause + orderByClause; - String nativeSql = translateSqlToNative(sql, dialect); - - nativeSql = expandMetricTemplates(nativeSql, query.getMetrics()); - - log.debug("Running native SQL query: {}", nativeSql); + log.debug("Running native SQL query: {}", sql); - javax.persistence.Query jpaQuery = entityManager.createNativeQuery(nativeSql); + javax.persistence.Query jpaQuery = entityManager.createNativeQuery(sql); paginate(query, jpaQuery, fromClause, joinClause, whereClause); @@ -182,58 +167,6 @@ public Iterable executeQuery(Query query) { .collect(Collectors.toList()); } - /** - * Translates the client query into SQL. - * @param query the client query. - * @return the SQL query. - */ - protected SQLQuery toSQL(Query query) { - SQLSchema schema = (SQLSchema) query.getSchema(); - Class entityClass = schema.getEntityClass(); - - SQLQuery.SQLQueryBuilder builder = SQLQuery.builder().clientQuery(query); - - String tableName = schema.getTableDefinition(); - String tableAlias = schema.getAlias(); - - builder.projectionClause(extractProjection(query)); - - Set joinPredicates = new HashSet<>(); - - if (query.getWhereFilter() != null) { - joinPredicates.addAll(extractPathElements(query.getWhereFilter())); - builder.whereClause("WHERE " + translateFilterExpression(schema, query.getWhereFilter(), - this::generateWhereClauseColumnReference)); - } - - if (query.getHavingFilter() != null) { - builder.havingClause("HAVING " + translateFilterExpression(schema, query.getHavingFilter(), - (predicate) -> { return generateHavingClauseColumnReference(predicate, query); })); - } - - if (! query.getDimensions().isEmpty()) { - builder.groupByClause(extractGroupBy(query)); - } - - if (query.getSorting() != null) { - Map sortClauses = query.getSorting().getValidSortingRules(entityClass, dictionary); - - builder.orderByClause(extractOrderBy(entityClass, sortClauses)); - - joinPredicates.addAll(extractPathElements(sortClauses)); - } - - String joinClause = joinPredicates.stream() - .map(this::extractJoin) - .collect(Collectors.joining(" ")); - - builder.joinClause(joinClause); - - builder.fromClause(String.format("%s AS %s", tableName, tableAlias)); - - return builder.build(); - } - protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt counter) { Class entityClass = query.getEntityClass(); @@ -292,22 +225,7 @@ private String translateFilterExpression(SQLSchema schema, Function columnGenerator) { HQLFilterOperation filterVisitor = new HQLFilterOperation(); - String whereClause = filterVisitor.apply(expression, columnGenerator); - - return whereClause.replaceAll("(:[a-zA-Z0-9_]+)", "\'$1\'"); - } - - private String expandMetricTemplates(String sql, Map> metrics) { - String expanded = sql; - for (Map.Entry entry : metrics.entrySet()) { - Metric metric = (Metric) entry.getKey(); - Class agg = (Class) entry.getValue(); - - expanded = expanded.replaceFirst( - "__" + metric.getName().toUpperCase(Locale.ENGLISH) + "__", - metric.getMetricExpression(Optional.of(agg)) + " AS " + metric.getName()); - } - return expanded; + return filterVisitor.apply(expression, columnGenerator); } private String extractJoin(FilterExpression expression) { From 284a57ea000420c359d23e6e066eabe67314cec5 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 16 Jul 2019 13:29:16 -0500 Subject: [PATCH 20/47] Refactored HQLFilterOperation so it can be used for Having and Where clause generaiton --- .../aggregation/engine/SQLQueryEngine.java | 35 +++++++++++++------ .../elide/core/filter/HQLFilterOperation.java | 2 +- .../hql/RootCollectionFetchQueryBuilder.java | 2 +- .../RootCollectionPageTotalsQueryBuilder.java | 2 +- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 3701030e0c..b6433f6f34 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -18,6 +18,7 @@ import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; +import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.dimension.Dimension; import com.yahoo.elide.datastores.aggregation.dimension.DimensionType; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromSubquery; @@ -58,14 +59,6 @@ public class SQLQueryEngine implements QueryEngine { private static final String SUBQUERY = "__SUBQUERY__"; - //Converts a filter predicate into a SQL column reference - private final Function columnGenerator = (predicate) -> { - Path.PathElement last = predicate.getPath().lastElement().get(); - Class lastClass = last.getType(); - - return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); - }; - public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { this.entityManager = entityManager; this.dictionary = dictionary; @@ -124,7 +117,7 @@ public Iterable executeQuery(Query query) { if (query.getWhereFilter() != null) { joinClause = " " + extractJoin(query.getWhereFilter()); - whereClause = " " + translateFilterExpression(schema, query.getWhereFilter()); + whereClause = " WHERE " + translateFilterExpression(schema, query.getWhereFilter()); } if (!dimensionProjections.isEmpty()) { @@ -225,7 +218,7 @@ private String translateFilterExpression(SQLSchema schema, Function columnGenerator) { HQLFilterOperation filterVisitor = new HQLFilterOperation(); - return filterVisitor.apply(expression, columnGenerator); + return filterVisitor.apply(expression, this::generateWhereClauseColumnReference); } private String extractJoin(FilterExpression expression) { @@ -383,4 +376,26 @@ private void paginate(Query query, pagination.setPageTotals(total); } } + + //Converts a filter predicate into a SQL WHERE clause column reference + private String generateWhereClauseColumnReference(FilterPredicate predicate) { + Path.PathElement last = predicate.getPath().lastElement().get(); + Class lastClass = last.getType(); + + return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); + } + + //Converts a filter predicate into a SQL HAVING clause column reference + private String generateHavingClauseColumnReference(FilterPredicate predicate) { + Path.PathElement last = predicate.getPath().lastElement().get(); + Class lastClass = last.getType(); + Schema schema = schemas.get(lastClass); + + Preconditions.checkNotNull(schema); + Metric metric = schema.getMetric(last.getFieldName()); + + Preconditions.checkNotNull(metric); + + return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); + } } diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java index bd9863437b..625d3af97d 100644 --- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java +++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java @@ -177,7 +177,7 @@ public String apply(FilterExpression filterExpression, boolean prefixWithAlias) */ public String apply(FilterExpression filterExpression, Function columnGenerator) { HQLQueryVisitor visitor = new HQLQueryVisitor(columnGenerator); - return "WHERE " + filterExpression.accept(visitor); + return filterExpression.accept(visitor); } /** diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java index 9978721da6..e57aa1dfdb 100644 --- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java +++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java @@ -44,7 +44,7 @@ public Query build() { Collection predicates = filterExpression.get().accept(extractor); //Build the WHERE clause - String filterClause = WHERE + new FilterTranslator().apply(filterExpression.get(), USE_ALIAS); + String filterClause = WHERE + new HQLFilterOperation().apply(filterExpression.get(), USE_ALIAS); //Build the JOIN clause String joinClause = getJoinClauseFromFilters(filterExpression.get()) diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java index 3bca1ac0a5..267d27c0ea 100644 --- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java +++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java @@ -67,7 +67,7 @@ public Query build() { predicates = filterExpression.get().accept(extractor); //Build the WHERE clause - filterClause = WHERE + new FilterTranslator().apply(filterExpression.get(), USE_ALIAS); + filterClause = WHERE + new HQLFilterOperation().apply(filterExpression.get(), USE_ALIAS); //Build the JOIN clause joinClause = getJoinClauseFromFilters(filterExpression.get()); From 009c2c5b44d66d46f86c8237f226055289bee7bd Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 16 Jul 2019 14:13:59 -0500 Subject: [PATCH 21/47] Added HAVING clause support --- .../aggregation/engine/SQLQueryEngine.java | 56 ++++++++++--------- .../engine/SQLQueryEngineTest.java | 24 ++++++++ 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index b6433f6f34..d0606295af 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -8,7 +8,6 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.Path; -import com.yahoo.elide.core.TimedFunction; import com.yahoo.elide.core.exceptions.InvalidPredicateException; import com.yahoo.elide.core.filter.FilterPredicate; import com.yahoo.elide.core.filter.HQLFilterOperation; @@ -92,6 +91,7 @@ public Iterable executeQuery(Query query) { String whereClause = ""; String orderByClause = ""; String groupByClause = ""; + String havingClause = ""; List metricProjections = query.getMetrics().entrySet().stream() .map((entry) -> { @@ -117,7 +117,13 @@ public Iterable executeQuery(Query query) { if (query.getWhereFilter() != null) { joinClause = " " + extractJoin(query.getWhereFilter()); - whereClause = " WHERE " + translateFilterExpression(schema, query.getWhereFilter()); + whereClause = " WHERE " + translateFilterExpression(schema, query.getWhereFilter(), + this::generateWhereClauseColumnReference); + } + + if (query.getHavingFilter() != null) { + havingClause = " HAVING " + translateFilterExpression(schema, query.getHavingFilter(), + (predicate) -> { return generateHavingClauseColumnReference(predicate, query); }); } if (!dimensionProjections.isEmpty()) { @@ -138,6 +144,7 @@ public Iterable executeQuery(Query query) { + joinClause + whereClause + groupByClause + + havingClause + orderByClause; log.debug("Running native SQL query: {}", sql); @@ -146,9 +153,7 @@ public Iterable executeQuery(Query query) { paginate(query, jpaQuery, fromClause, joinClause, whereClause); - if (query.getWhereFilter() != null) { - supplyFilterQueryParameters(query.getWhereFilter(), jpaQuery); - } + supplyFilterQueryParameters(query, jpaQuery); List results = jpaQuery.getResultList(); @@ -206,19 +211,12 @@ protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt c return entityInstance; } - /** - * Translates a filter expression into SQL. - * @param schema The schema being queried. - * @param expression The filter expression - * @param columnGenerator A function which generates a column reference in SQL from a FilterPredicate. - * @return A SQL expression - */ private String translateFilterExpression(SQLSchema schema, FilterExpression expression, Function columnGenerator) { HQLFilterOperation filterVisitor = new HQLFilterOperation(); - return filterVisitor.apply(expression, this::generateWhereClauseColumnReference); + return filterVisitor.apply(expression, columnGenerator); } private String extractJoin(FilterExpression expression) { @@ -294,9 +292,17 @@ private String extractOrderBy(Class entityClass, Map }).collect(Collectors.joining(",")); } - private void supplyFilterQueryParameters(FilterExpression expression, - javax.persistence.Query query) { - Collection predicates = expression.accept(new PredicateExtractionVisitor()); + private void supplyFilterQueryParameters(Query query, + javax.persistence.Query jpaQuery) { + + Collection predicates = new ArrayList<>(); + if (query.getWhereFilter() != null) { + predicates.addAll(query.getWhereFilter().accept(new PredicateExtractionVisitor())); + } + + if (query.getHavingFilter() != null) { + predicates.addAll(query.getHavingFilter().accept(new PredicateExtractionVisitor())); + } for (FilterPredicate filterPredicate : predicates) { if (filterPredicate.getOperator().isParameterized()) { @@ -367,9 +373,7 @@ private void paginate(Query query, javax.persistence.Query pageTotalQuery = entityManager.createNativeQuery(sql); - if (query.getWhereFilter() != null) { - supplyFilterQueryParameters(query.getWhereFilter(), pageTotalQuery); - } + supplyFilterQueryParameters(query, pageTotalQuery); long total = CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); @@ -386,16 +390,18 @@ private String generateWhereClauseColumnReference(FilterPredicate predicate) { } //Converts a filter predicate into a SQL HAVING clause column reference - private String generateHavingClauseColumnReference(FilterPredicate predicate) { + private String generateHavingClauseColumnReference(FilterPredicate predicate, Query query) { Path.PathElement last = predicate.getPath().lastElement().get(); Class lastClass = last.getType(); - Schema schema = schemas.get(lastClass); - Preconditions.checkNotNull(schema); - Metric metric = schema.getMetric(last.getFieldName()); + if (! lastClass.equals(query.getEntityClass())) { + throw new InvalidPredicateException("The having clause can only reference fact table aggregations."); + } - Preconditions.checkNotNull(metric); + Schema schema = schemas.get(lastClass); + Metric metric = schema.getMetric(last.getFieldName()); + Class agg = query.getMetrics().get(metric); - return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); + return metric.getMetricExpression(Optional.of(agg)); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 6f672ed8e0..93cfd4fcf1 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -273,4 +273,28 @@ public void testPagination() throws Exception { Assert.assertEquals(results.get(0), stats1, "Returned record does not match"); Assert.assertEquals(pagination.getPageTotals(), 2, "Page totals does not match"); } + + @Test + public void testHavingClause() throws Exception { + EntityManager em = emf.createEntityManager(); + QueryEngine engine = new SQLQueryEngine(em, dictionary); + + Query query = Query.builder() + .entityClass(PlayerStats.class) + .metric(playerStatsSchema.getMetric("highScore"), Sum.class) + .havingFilter(filterParser.parseFilterExpression("highScore > 300", + PlayerStats.class, false)) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00 + PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); + stats1.setHighScore(3646); + + Assert.assertEquals(results.size(), 1); + Assert.assertEquals(results.get(0), stats1); + } } From 1fbb6fcbf6389e6494a573fa59d09938fa0d463d Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 16 Jul 2019 15:13:20 -0500 Subject: [PATCH 22/47] Changed Query to take schema instead of entityClass --- .../aggregation/engine/SQLQueryEngine.java | 14 +++++++------- .../aggregation/engine/SQLQueryEngineTest.java | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index d0606295af..7b728ab97d 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -79,13 +79,14 @@ public Iterable executeQuery(Query query) { //Make sure we actually manage this schema. Preconditions.checkNotNull(schema); + Class entityClass = schema.getEntityClass(); //Translate the query into SQL. SQLQuery sql = toSQL(query); Map sortClauses = (query.getSorting() == null) ? new HashMap<>() - : query.getSorting().getValidSortingRules(query.getEntityClass(), dictionary); + : query.getSorting().getValidSortingRules(entityClass, dictionary); String joinClause = ""; String whereClause = ""; @@ -103,7 +104,7 @@ public Iterable executeQuery(Query query) { List dimensionProjections = query.getDimensions().stream() .map(Dimension::getName) - .map((name) -> getColumnName(query.getEntityClass(), name)) + .map((name) -> getColumnName(entityClass, name)) .collect(Collectors.toList()); String projectionClause = metricProjections.stream() @@ -134,7 +135,7 @@ public Iterable executeQuery(Query query) { } if (query.getSorting() != null) { - orderByClause = " " + extractOrderBy(query.getEntityClass(), sortClauses); + orderByClause = " " + extractOrderBy(entityClass, sortClauses); joinClause += " " + extractJoin(sortClauses); } @@ -166,8 +167,7 @@ public Iterable executeQuery(Query query) { } protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt counter) { - - Class entityClass = query.getEntityClass(); + Class entityClass = query.getSchema().getEntityClass(); List projections = query.getMetrics().entrySet().stream() .map(Map.Entry::getKey) .map(Metric::getName) @@ -362,7 +362,7 @@ private void paginate(Query query, if (pagination.isGenerateTotals()) { String groupByDimensions = query.getDimensions().stream() .map(Dimension::getName) - .map((name) -> getColumnName(query.getEntityClass(), name)) + .map((name) -> getColumnName(query.getSchema().getEntityClass(), name)) .collect(Collectors.joining(",")); String sql = String.format("SELECT COUNT(DISTINCT(%s)) FROM %s %s %s", @@ -394,7 +394,7 @@ private String generateHavingClauseColumnReference(FilterPredicate predicate, Qu Path.PathElement last = predicate.getPath().lastElement().get(); Class lastClass = last.getType(); - if (! lastClass.equals(query.getEntityClass())) { + if (! lastClass.equals(query.getSchema().getEntityClass())) { throw new InvalidPredicateException("The having clause can only reference fact table aggregations."); } diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 93cfd4fcf1..7871985665 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -65,7 +65,7 @@ public void testFullTableLoad() throws Exception { QueryEngine engine = new SQLQueryEngine(em, dictionary); Query query = Query.builder() - .entityClass(PlayerStats.class) + .schema(playerStatsSchema) .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) @@ -101,7 +101,7 @@ public void testDegenerateDimensionFilter() throws Exception { QueryEngine engine = new SQLQueryEngine(em, dictionary); Query query = Query.builder() - .entityClass(PlayerStats.class) + .schema(playerStatsSchema) .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) @@ -131,7 +131,7 @@ public void testFilterJoin() throws Exception { QueryEngine engine = new SQLQueryEngine(em, dictionary); Query query = Query.builder() - .entityClass(PlayerStats.class) + .schema(playerStatsSchema) .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) @@ -168,7 +168,7 @@ public void testSubqueryFilterJoin() throws Exception { QueryEngine engine = new SQLQueryEngine(em, dictionary); Query query = Query.builder() - .entityClass(PlayerStatsView.class) + .schema(playerStatsViewSchema) .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) .whereFilter(filterParser.parseFilterExpression("player.name=='Jane Doe'", PlayerStatsView.class, false)) @@ -191,7 +191,7 @@ public void testSubqueryLoad() throws Exception { QueryEngine engine = new SQLQueryEngine(em, dictionary); Query query = Query.builder() - .entityClass(PlayerStatsView.class) + .schema(playerStatsViewSchema) .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) .build(); @@ -215,7 +215,7 @@ public void testSortJoin() throws Exception { sortMap.put("player.name", Sorting.SortOrder.asc); Query query = Query.builder() - .entityClass(PlayerStats.class) + .schema(playerStatsSchema) .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) @@ -250,7 +250,7 @@ public void testPagination() throws Exception { Pagination pagination = Pagination.fromOffsetAndLimit(1, 0, true); Query query = Query.builder() - .entityClass(PlayerStats.class) + .schema(playerStatsSchema) .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) @@ -280,7 +280,7 @@ public void testHavingClause() throws Exception { QueryEngine engine = new SQLQueryEngine(em, dictionary); Query query = Query.builder() - .entityClass(PlayerStats.class) + .schema(playerStatsSchema) .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .havingFilter(filterParser.parseFilterExpression("highScore > 300", PlayerStats.class, false)) From 080ae92ce5b9d41428359780999233963a5c4024 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 16 Jul 2019 17:13:29 -0500 Subject: [PATCH 23/47] First pass at cleanup --- .../aggregation/engine/SQLQuery.java | 3 - .../aggregation/engine/SQLQueryEngine.java | 199 +++++++++--------- .../engine/SQLQueryEngineTest.java | 1 + 3 files changed, 102 insertions(+), 101 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQuery.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQuery.java index 537adac4a8..ec2cb5242c 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQuery.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQuery.java @@ -11,9 +11,6 @@ import lombok.Data; import lombok.NonNull; -/** - * Aids in constructing a SQL query from String fragments. - */ @Data @Builder public class SQLQuery { diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 7b728ab97d..93aaa19758 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -33,7 +33,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -79,91 +78,81 @@ public Iterable executeQuery(Query query) { //Make sure we actually manage this schema. Preconditions.checkNotNull(schema); - Class entityClass = schema.getEntityClass(); - //Translate the query into SQL. SQLQuery sql = toSQL(query); - Map sortClauses = (query.getSorting() == null) - ? new HashMap<>() - : query.getSorting().getValidSortingRules(entityClass, dictionary); + log.debug("Running native SQL query: {}", sql); - String joinClause = ""; - String whereClause = ""; - String orderByClause = ""; - String groupByClause = ""; - String havingClause = ""; + javax.persistence.Query jpaQuery = entityManager.createNativeQuery(sql.toString()); - List metricProjections = query.getMetrics().entrySet().stream() - .map((entry) -> { - Metric metric = entry.getKey(); - Class agg = entry.getValue(); - return metric.getMetricExpression(Optional.of(agg)) + " AS " + metric.getName(); - }) - .collect(Collectors.toList()); + Pagination pagination = query.getPagination(); + if (pagination != null) { + jpaQuery.setFirstResult(pagination.getOffset()); + jpaQuery.setMaxResults(pagination.getLimit()); - List dimensionProjections = query.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(entityClass, name)) - .collect(Collectors.toList()); + if (pagination.isGenerateTotals()) { + javax.persistence.Query pageTotalQuery = + entityManager.createNativeQuery(toPageTotalSQL(sql).toString()); - String projectionClause = metricProjections.stream() - .collect(Collectors.joining(",")); + supplyFilterQueryParameters(query, pageTotalQuery); - if (!dimensionProjections.isEmpty()) { - projectionClause = projectionClause + "," + dimensionProjections.stream() - .map((name) -> tableAlias + "." + name) - .collect(Collectors.joining(",")); + pagination.setPageTotals(CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class)); + } } + supplyFilterQueryParameters(query, jpaQuery); + + List results = jpaQuery.getResultList(); + + MutableInt counter = new MutableInt(0); + + return results.stream() + .map((result) -> { return result instanceof Object[] ? (Object []) result : new Object[] { result }; }) + .map((result) -> coerceObjectToEntity(query, result, counter)) + .collect(Collectors.toList()); + } + + protected SQLQuery toSQL(Query query) { + SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); + Class entityClass = schema.getEntityClass(); + + Preconditions.checkNotNull(schema); + + SQLQuery.SQLQueryBuilder builder = SQLQuery.builder().clientQuery(query); + + String tableName = schema.getTableDefinition(); + String tableAlias = schema.getAlias(); + + String joinClause = ""; + builder.projectionClause(extractProjection(query)); + if (query.getWhereFilter() != null) { - joinClause = " " + extractJoin(query.getWhereFilter()); - whereClause = " WHERE " + translateFilterExpression(schema, query.getWhereFilter(), - this::generateWhereClauseColumnReference); + joinClause = extractJoin(query.getWhereFilter()); + builder.whereClause("WHERE " + translateFilterExpression(schema, query.getWhereFilter(), + this::generateWhereClauseColumnReference)); } if (query.getHavingFilter() != null) { - havingClause = " HAVING " + translateFilterExpression(schema, query.getHavingFilter(), - (predicate) -> { return generateHavingClauseColumnReference(predicate, query); }); + builder.havingClause("HAVING " + translateFilterExpression(schema, query.getHavingFilter(), + (predicate) -> { return generateHavingClauseColumnReference(predicate, query); })); } - if (!dimensionProjections.isEmpty()) { - groupByClause = " GROUP BY "; - groupByClause += dimensionProjections.stream() - .map((name) -> tableAlias + "." + name) - .collect(Collectors.joining(",")); + if (! query.getDimensions().isEmpty()) { + builder.groupByClause(extractGroupBy(query)); } if (query.getSorting() != null) { - orderByClause = " " + extractOrderBy(entityClass, sortClauses); - joinClause += " " + extractJoin(sortClauses); - } - - String fromClause = String.format("%s AS %s", tableName, tableAlias); + Map sortClauses = query.getSorting().getValidSortingRules(entityClass, dictionary); - String sql = String.format("SELECT %s FROM %s", projectionClause, fromClause) - + joinClause - + whereClause - + groupByClause - + havingClause - + orderByClause; - - log.debug("Running native SQL query: {}", sql); - - javax.persistence.Query jpaQuery = entityManager.createNativeQuery(sql); - - paginate(query, jpaQuery, fromClause, joinClause, whereClause); - - supplyFilterQueryParameters(query, jpaQuery); + builder.orderByClause(extractOrderBy(entityClass, sortClauses)); + joinClause += extractJoin(sortClauses); + } - List results = jpaQuery.getResultList(); + builder.joinClause(joinClause); - MutableInt counter = new MutableInt(0); + builder.fromClause(String.format("%s AS %s", tableName, tableAlias)); - return results.stream() - .map((result) -> { return result instanceof Object[] ? (Object []) result : new Object[] { result }; }) - .map((result) -> coerceObjectToEntity(query, result, counter)) - .collect(Collectors.toList()); + return builder.build(); } protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt counter) { @@ -338,47 +327,61 @@ private String getColumnName(Class entityClass, String fieldName) { } } - /** - * Paginates the query if requested. - * @param query The QueryEngine query - * @param jpaQuery The JPA query - */ - private void paginate(Query query, - javax.persistence.Query jpaQuery, - String fromClause, - String joinClause, - String whereClause) { - Pagination pagination = query.getPagination(); - if (pagination == null) { - return; - } - jpaQuery.setFirstResult(pagination.getOffset()); - jpaQuery.setMaxResults(pagination.getLimit()); - - /* - * TODO - this is a naive implementation. We should run these in parallel or combine the queries - * with a windowing function (if the DB supports that). - */ - if (pagination.isGenerateTotals()) { - String groupByDimensions = query.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(query.getSchema().getEntityClass(), name)) - .collect(Collectors.joining(",")); + private SQLQuery toPageTotalSQL(SQLQuery sql) { + Query clientQuery = sql.getClientQuery(); - String sql = String.format("SELECT COUNT(DISTINCT(%s)) FROM %s %s %s", - groupByDimensions, - fromClause, - joinClause, - whereClause); + String groupByDimensions = clientQuery.getDimensions().stream() + .map(Dimension::getName) + .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name)) + .collect(Collectors.joining(",")); - javax.persistence.Query pageTotalQuery = entityManager.createNativeQuery(sql); + String projectionClause = String.format("SELECT COUNT(DISTINCT(%s))", groupByDimensions); - supplyFilterQueryParameters(query, pageTotalQuery); + return SQLQuery.builder() + .clientQuery(sql.getClientQuery()) + .projectionClause(projectionClause) + .fromClause(sql.getFromClause()) + .joinClause(sql.getJoinClause()) + .whereClause(sql.getWhereClause()) + .build(); + } - long total = CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); + private String extractProjection(Query query) { + List metricProjections = query.getMetrics().entrySet().stream() + .map((entry) -> { + Metric metric = entry.getKey(); + Class agg = entry.getValue(); + return metric.getMetricExpression(Optional.of(agg)) + " AS " + metric.getName(); + }) + .collect(Collectors.toList()); - pagination.setPageTotals(total); + List dimensionProjections = query.getDimensions().stream() + .map(Dimension::getName) + .map((name) -> getColumnName(query.getSchema().getEntityClass(), name)) + .collect(Collectors.toList()); + + String projectionClause = metricProjections.stream() + .collect(Collectors.joining(",")); + + if (!dimensionProjections.isEmpty()) { + projectionClause = projectionClause + "," + dimensionProjections.stream() + .map((name) -> query.getSchema().getAlias() + "." + name) + .collect(Collectors.joining(",")); } + + return projectionClause; + } + + private String extractGroupBy(Query query) { + List dimensionProjections = query.getDimensions().stream() + .map(Dimension::getName) + .map((name) -> getColumnName(query.getSchema().getEntityClass(), name)) + .collect(Collectors.toList()); + + return "GROUP BY " + dimensionProjections.stream() + .map((name) -> query.getSchema().getAlias() + "." + name) + .collect(Collectors.joining(",")); + } //Converts a filter predicate into a SQL WHERE clause column reference diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 7871985665..c8e60360a5 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -36,6 +36,7 @@ public class SQLQueryEngineTest { private EntityManagerFactory emf; + private Schema playerStatsSchema; private Schema playerStatsViewSchema; private EntityDictionary dictionary; From 2699dffcde6aaa7bbe49127e0336c86fc170687f Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 16 Jul 2019 17:19:32 -0500 Subject: [PATCH 24/47] Fixed checkstyles --- .../yahoo/elide/datastores/aggregation/engine/SQLQuery.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQuery.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQuery.java index ec2cb5242c..537adac4a8 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQuery.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQuery.java @@ -11,6 +11,9 @@ import lombok.Data; import lombok.NonNull; +/** + * Aids in constructing a SQL query from String fragments. + */ @Data @Builder public class SQLQuery { From 763dfa8fddb522e50b275fd14c153f414ff48440 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 16 Jul 2019 22:00:16 -0500 Subject: [PATCH 25/47] Cleanup --- .../aggregation/engine/SQLQueryEngine.java | 116 +++++++++++++++--- 1 file changed, 100 insertions(+), 16 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 93aaa19758..2620f27ad1 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -8,6 +8,7 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.Path; +import com.yahoo.elide.core.TimedFunction; import com.yahoo.elide.core.exceptions.InvalidPredicateException; import com.yahoo.elide.core.filter.FilterPredicate; import com.yahoo.elide.core.filter.HQLFilterOperation; @@ -29,6 +30,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang3.mutable.MutableInt; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; @@ -52,14 +54,15 @@ public class SQLQueryEngine implements QueryEngine { private EntityManager entityManager; private EntityDictionary dictionary; + @Getter private Map, SQLSchema> schemas; - private static final String SUBQUERY = "__SUBQUERY__"; - public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) { this.entityManager = entityManager; this.dictionary = dictionary; + + // Construct the list of schemas that will be managed by this query engine. schemas = dictionary.getBindings() .stream() .filter((clazz) -> @@ -79,10 +82,9 @@ public Iterable executeQuery(Query query) { //Make sure we actually manage this schema. Preconditions.checkNotNull(schema); + //Translate the query into SQL. SQLQuery sql = toSQL(query); - log.debug("Running native SQL query: {}", sql); - javax.persistence.Query jpaQuery = entityManager.createNativeQuery(sql.toString()); Pagination pagination = query.getPagination(); @@ -94,30 +96,44 @@ public Iterable executeQuery(Query query) { javax.persistence.Query pageTotalQuery = entityManager.createNativeQuery(toPageTotalSQL(sql).toString()); + //Supply the query parameters to the query supplyFilterQueryParameters(query, pageTotalQuery); - pagination.setPageTotals(CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class)); + //Run the Pagination query and log the time spent. + long total = new TimedFunction<>(() -> { + return CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); + }, "Pagination Query").get(); + + pagination.setPageTotals(total); } } + //Supply the query parameters to the query supplyFilterQueryParameters(query, jpaQuery); - List results = jpaQuery.getResultList(); + //Run the primary query and log the time spent. + List results = new TimedFunction<>(() -> { + return jpaQuery.getResultList(); + }, "Primary Query").get(); - MutableInt counter = new MutableInt(0); + //Coerce the results into entity objects. + MutableInt counter = new MutableInt(0); return results.stream() .map((result) -> { return result instanceof Object[] ? (Object []) result : new Object[] { result }; }) .map((result) -> coerceObjectToEntity(query, result, counter)) .collect(Collectors.toList()); } + /** + * Translates the client query into SQL. + * @param query the client query. + * @return the SQL query. + */ protected SQLQuery toSQL(Query query) { - SQLSchema schema = schemas.get(query.getSchema().getEntityClass()); + SQLSchema schema = (SQLSchema) query.getSchema(); Class entityClass = schema.getEntityClass(); - Preconditions.checkNotNull(schema); - SQLQuery.SQLQueryBuilder builder = SQLQuery.builder().clientQuery(query); String tableName = schema.getTableDefinition(); @@ -155,8 +171,17 @@ protected SQLQuery toSQL(Query query) { return builder.build(); } + /** + * Coerces results from a JPA query into an Object. + * @param query The client query + * @param result A row from the results. + * @param counter Monotonically increasing number to generate IDs. + * @return A hydrated entity object. + */ protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt counter) { Class entityClass = query.getSchema().getEntityClass(); + + //Get all the projections from the client query. List projections = query.getMetrics().entrySet().stream() .map(Map.Entry::getKey) .map(Metric::getName) @@ -166,8 +191,6 @@ protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt c .map(Dimension::getName) .collect(Collectors.toList())); - SQLSchema schema = schemas.get(entityClass); - Preconditions.checkArgument(result.length == projections.size()); SQLSchema schema = (SQLSchema) query.getSchema(); @@ -200,6 +223,13 @@ protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt c return entityInstance; } + /** + * Translates a filter expression into SQL. + * @param schema The schema being queried. + * @param expression The filter expression + * @param columnGenerator A function which generates a column reference in SQL from a FilterPredicate. + * @return A SQL expression + */ private String translateFilterExpression(SQLSchema schema, FilterExpression expression, Function columnGenerator) { @@ -208,13 +238,27 @@ private String translateFilterExpression(SQLSchema schema, return filterVisitor.apply(expression, columnGenerator); } + /** + * Given a filter expression, extracts any table joins that are required to perform the filter. + * @param expression The filter expression + * @return A SQL expression + */ private String extractJoin(FilterExpression expression) { Collection predicates = expression.accept(new PredicateExtractionVisitor()); return predicates.stream() .filter(predicate -> predicate.getPath().getPathElements().size() > 1) - .map(FilterPredicate::getPath) - .flatMap((path) -> path.getPathElements().stream()) + .map(this::extractJoin) + .collect(Collectors.joining(" ")); + } + + /** + * Given a filter predicate, extracts any table joins that are required to perform the filter. + * @param predicate The filter predicate + * @return A SQL expression + */ + private String extractJoin(FilterPredicate predicate) { + return predicate.getPath().getPathElements().stream() .filter((p) -> dictionary.isRelation(p.getType(), p.getFieldName())) .collect(Collectors.toCollection(LinkedHashSet::new)); } @@ -249,6 +293,11 @@ private String extractJoin(Path.PathElement pathElement) { relationshipIdField); } + /** + * Given a list of columns to sort on, extracts any required table joins to perform the sort. + * @param sortClauses The list of sort columns and their sort order (ascending or descending). + * @return A SQL expression + */ private String extractJoin(Map sortClauses) { if (sortClauses.isEmpty()) { return ""; @@ -262,6 +311,12 @@ private String extractJoin(Map sortClauses) { .collect(Collectors.joining(" ")); } + /** + * Given a list of columns to sort on, constructs an ORDER BY clause in SQL. + * @param entityClass The class to sort. + * @param sortClauses The list of sort columns and their sort order (ascending or descending). + * @return A SQL expression + */ private String extractOrderBy(Class entityClass, Map sortClauses) { if (sortClauses.isEmpty()) { return ""; @@ -281,6 +336,11 @@ private String extractOrderBy(Class entityClass, Map }).collect(Collectors.joining(",")); } + /** + * Given a JPA query, replaces any parameters with their values from client query. + * @param query The client query + * @param jpaQuery The JPA query + */ private void supplyFilterQueryParameters(Query query, javax.persistence.Query jpaQuery) { @@ -327,6 +387,12 @@ private String getColumnName(Class entityClass, String fieldName) { } } + /** + * Takes a SQLQuery and creates a new clone that instead returns the total number of records of the original + * query. + * @param sql The original query + * @return A new query that returns the total number of records. + */ private SQLQuery toPageTotalSQL(SQLQuery sql) { Query clientQuery = sql.getClientQuery(); @@ -346,6 +412,11 @@ private SQLQuery toPageTotalSQL(SQLQuery sql) { .build(); } + /** + * Given a client query, constructs the list of columns to project from a database table. + * @param query The client query + * @return A SQL fragment to use in the SELECT .. statement. + */ private String extractProjection(Query query) { List metricProjections = query.getMetrics().entrySet().stream() .map((entry) -> { @@ -372,6 +443,11 @@ private String extractProjection(Query query) { return projectionClause; } + /** + * Extracts a GROUP BY SQL clause. + * @param query A client query + * @return The SQL GROUP BY clause + */ private String extractGroupBy(Query query) { List dimensionProjections = query.getDimensions().stream() .map(Dimension::getName) @@ -384,7 +460,11 @@ private String extractGroupBy(Query query) { } - //Converts a filter predicate into a SQL WHERE clause column reference + /** + * Converts a filter predicate into a SQL WHERE clause column reference. + * @param predicate The predicate to convert + * @return A SQL fragment that references a database column + */ private String generateWhereClauseColumnReference(FilterPredicate predicate) { Path.PathElement last = predicate.getPath().lastElement().get(); Class lastClass = last.getType(); @@ -392,7 +472,11 @@ private String generateWhereClauseColumnReference(FilterPredicate predicate) { return FilterPredicate.getTypeAlias(lastClass) + "." + getColumnName(lastClass, last.getFieldName()); } - //Converts a filter predicate into a SQL HAVING clause column reference + /** + * Converts a filter predicate into a SQL HAVING clause column reference. + * @param predicate The predicate to convert + * @return A SQL fragment that references a database column + */ private String generateHavingClauseColumnReference(FilterPredicate predicate, Query query) { Path.PathElement last = predicate.getPath().lastElement().get(); Class lastClass = last.getType(); From 29329f9bc8651904c0002be5b545d53a616a54d6 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 16 Jul 2019 22:02:46 -0500 Subject: [PATCH 26/47] Cleanup --- .../engine/SQLQueryEngineTest.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index c8e60360a5..971f79c2c0 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -298,4 +298,27 @@ public void testHavingClause() throws Exception { Assert.assertEquals(results.size(), 1); Assert.assertEquals(results.get(0), stats1); } + + @Test + public void testTheEverythingQuery() throws Exception { + EntityManager em = emf.createEntityManager(); + QueryEngine engine = new SQLQueryEngine(em, dictionary); + + Query query = Query.builder() + .schema(playerStatsViewSchema) + .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) + .whereFilter(filterParser.parseFilterExpression("player.name=='Jane Doe'", + PlayerStatsView.class, false)) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStatsView stats2 = new PlayerStatsView(); + stats2.setId("0"); + stats2.setHighScore(2412); + + Assert.assertEquals(results.size(), 1); + Assert.assertEquals(results.get(0), stats2); + } } From cd169dbd1ab3cf3e6b5b992c0a8df4b43d8eaefe Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Tue, 16 Jul 2019 22:34:36 -0500 Subject: [PATCH 27/47] Added a complex SQL expression test and fixed bugs --- .../aggregation/engine/SQLQueryEngine.java | 46 +++++++++---------- .../engine/SQLQueryEngineTest.java | 9 ++++ 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 2620f27ad1..6e5712f67e 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -35,6 +35,8 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -139,11 +141,12 @@ protected SQLQuery toSQL(Query query) { String tableName = schema.getTableDefinition(); String tableAlias = schema.getAlias(); - String joinClause = ""; builder.projectionClause(extractProjection(query)); + Set joinPredicates = new HashSet<>(); + if (query.getWhereFilter() != null) { - joinClause = extractJoin(query.getWhereFilter()); + joinPredicates.addAll(extractPathElements(query.getWhereFilter())); builder.whereClause("WHERE " + translateFilterExpression(schema, query.getWhereFilter(), this::generateWhereClauseColumnReference)); } @@ -161,9 +164,14 @@ protected SQLQuery toSQL(Query query) { Map sortClauses = query.getSorting().getValidSortingRules(entityClass, dictionary); builder.orderByClause(extractOrderBy(entityClass, sortClauses)); - joinClause += extractJoin(sortClauses); + + joinPredicates.addAll(extractPathElements(sortClauses)); } + String joinClause = joinPredicates.stream() + .map(this::extractJoin) + .collect(Collectors.joining(" ")); + builder.joinClause(joinClause); builder.fromClause(String.format("%s AS %s", tableName, tableAlias)); @@ -239,26 +247,17 @@ private String translateFilterExpression(SQLSchema schema, } /** - * Given a filter expression, extracts any table joins that are required to perform the filter. + * Given a filter expression, extracts any entity relationship traversals that require joins. * @param expression The filter expression - * @return A SQL expression + * @return A set of path elements that capture a relationship traversal. */ - private String extractJoin(FilterExpression expression) { + private Set extractPathElements(FilterExpression expression) { Collection predicates = expression.accept(new PredicateExtractionVisitor()); return predicates.stream() .filter(predicate -> predicate.getPath().getPathElements().size() > 1) - .map(this::extractJoin) - .collect(Collectors.joining(" ")); - } - - /** - * Given a filter predicate, extracts any table joins that are required to perform the filter. - * @param predicate The filter predicate - * @return A SQL expression - */ - private String extractJoin(FilterPredicate predicate) { - return predicate.getPath().getPathElements().stream() + .map(FilterPredicate::getPath) + .flatMap((path) -> path.getPathElements().stream()) .filter((p) -> dictionary.isRelation(p.getType(), p.getFieldName())) .collect(Collectors.toCollection(LinkedHashSet::new)); } @@ -294,21 +293,20 @@ private String extractJoin(Path.PathElement pathElement) { } /** - * Given a list of columns to sort on, extracts any required table joins to perform the sort. + * Given a list of columns to sort on, extracts any entity relationship traversals that require joins. * @param sortClauses The list of sort columns and their sort order (ascending or descending). - * @return A SQL expression + * @return A set of path elements that capture a relationship traversal. */ - private String extractJoin(Map sortClauses) { + private Set extractPathElements(Map sortClauses) { if (sortClauses.isEmpty()) { - return ""; + return new LinkedHashSet<>(); } return sortClauses.entrySet().stream() .map(Map.Entry::getKey) .flatMap((path) -> path.getPathElements().stream()) - .filter((predicate) -> dictionary.isRelation(predicate.getType(), predicate.getFieldName())) - .map(this::extractJoin) - .collect(Collectors.joining(" ")); + .filter((element) -> dictionary.isRelation(element.getType(), element.getFieldName())) + .collect(Collectors.toCollection(LinkedHashSet::new)); } /** diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 971f79c2c0..b68700c77f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -304,11 +304,18 @@ public void testTheEverythingQuery() throws Exception { EntityManager em = emf.createEntityManager(); QueryEngine engine = new SQLQueryEngine(em, dictionary); + Map sortMap = new HashMap<>(); + sortMap.put("player.name", Sorting.SortOrder.asc); + Query query = Query.builder() .schema(playerStatsViewSchema) .metric(playerStatsViewSchema.getMetric("highScore"), Sum.class) + .groupDimension(playerStatsViewSchema.getDimension("countryName")) .whereFilter(filterParser.parseFilterExpression("player.name=='Jane Doe'", PlayerStatsView.class, false)) + .havingFilter(filterParser.parseFilterExpression("highScore > 300", + PlayerStatsView.class, false)) + .sorting(new Sorting(sortMap)) .build(); List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) @@ -317,6 +324,8 @@ public void testTheEverythingQuery() throws Exception { PlayerStatsView stats2 = new PlayerStatsView(); stats2.setId("0"); stats2.setHighScore(2412); + stats2.setCountryName("United States"); + Assert.assertEquals(results.size(), 1); Assert.assertEquals(results.get(0), stats2); From 50ac2063625e2f5dcb834a83bbd2ec7a2c0be459 Mon Sep 17 00:00:00 2001 From: Aaron Klish Date: Wed, 17 Jul 2019 12:34:43 -0500 Subject: [PATCH 28/47] Fixed merge issues. Added another test. Added better logging --- .../dimension/DegenerateDimension.java | 1 - .../dimension/EntityDimension.java | 1 - .../aggregation/dimension/TimeDimension.java | 1 - .../aggregation/engine/SQLQueryEngine.java | 10 +++-- .../aggregation/metric/AggregatedMetric.java | 1 - .../engine/SQLQueryEngineTest.java | 42 ++++++++++++++++++- 6 files changed, 46 insertions(+), 10 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java index 84fa2c51c7..22f84dc885 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/DegenerateDimension.java @@ -6,7 +6,6 @@ package com.yahoo.elide.datastores.aggregation.dimension; import com.yahoo.elide.core.EntityDictionary; -import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; import com.yahoo.elide.datastores.aggregation.annotation.Meta; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java index 21eeff5569..f95726862f 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java @@ -7,7 +7,6 @@ import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.datastores.aggregation.Column; -import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.annotation.Cardinality; import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; import com.yahoo.elide.datastores.aggregation.annotation.FriendlyName; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java index 44d01f0f17..5e3331ff04 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/TimeDimension.java @@ -5,7 +5,6 @@ */ package com.yahoo.elide.datastores.aggregation.dimension; -import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; import com.yahoo.elide.datastores.aggregation.annotation.Meta; import com.yahoo.elide.datastores.aggregation.schema.Schema; diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 6e5712f67e..f6b7d9d03a 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -18,7 +18,6 @@ import com.yahoo.elide.core.sort.Sorting; import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; -import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.dimension.Dimension; import com.yahoo.elide.datastores.aggregation.dimension.DimensionType; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromSubquery; @@ -26,6 +25,7 @@ import com.yahoo.elide.datastores.aggregation.engine.schema.SQLSchema; import com.yahoo.elide.datastores.aggregation.metric.Aggregation; import com.yahoo.elide.datastores.aggregation.metric.Metric; +import com.yahoo.elide.datastores.aggregation.schema.Schema; import com.yahoo.elide.utils.coerce.CoerceUtil; import com.google.common.base.Preconditions; @@ -95,8 +95,10 @@ public Iterable executeQuery(Query query) { jpaQuery.setMaxResults(pagination.getLimit()); if (pagination.isGenerateTotals()) { + + SQLQuery paginationSQL = toPageTotalSQL(sql); javax.persistence.Query pageTotalQuery = - entityManager.createNativeQuery(toPageTotalSQL(sql).toString()); + entityManager.createNativeQuery(paginationSQL.toString()); //Supply the query parameters to the query supplyFilterQueryParameters(query, pageTotalQuery); @@ -104,7 +106,7 @@ public Iterable executeQuery(Query query) { //Run the Pagination query and log the time spent. long total = new TimedFunction<>(() -> { return CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); - }, "Pagination Query").get(); + }, "Running Query: " + paginationSQL).get(); pagination.setPageTotals(total); } @@ -116,7 +118,7 @@ public Iterable executeQuery(Query query) { //Run the primary query and log the time spent. List results = new TimedFunction<>(() -> { return jpaQuery.getResultList(); - }, "Primary Query").get(); + }, "Running Query: " + sql).get(); //Coerce the results into entity objects. diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java index a3d2c92405..f8149c1208 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java @@ -6,7 +6,6 @@ package com.yahoo.elide.datastores.aggregation.metric; import com.yahoo.elide.datastores.aggregation.Column; -import com.yahoo.elide.datastores.aggregation.Schema; import com.yahoo.elide.datastores.aggregation.annotation.Meta; import com.yahoo.elide.datastores.aggregation.annotation.MetricAggregation; diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index b68700c77f..8330910245 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import javax.persistence.EntityManager; @@ -212,7 +213,7 @@ public void testSortJoin() throws Exception { EntityManager em = emf.createEntityManager(); QueryEngine engine = new SQLQueryEngine(em, dictionary); - Map sortMap = new HashMap<>(); + Map sortMap = new TreeMap<>(); sortMap.put("player.name", Sorting.SortOrder.asc); Query query = Query.builder() @@ -304,7 +305,7 @@ public void testTheEverythingQuery() throws Exception { EntityManager em = emf.createEntityManager(); QueryEngine engine = new SQLQueryEngine(em, dictionary); - Map sortMap = new HashMap<>(); + Map sortMap = new TreeMap<>(); sortMap.put("player.name", Sorting.SortOrder.asc); Query query = Query.builder() @@ -330,4 +331,41 @@ public void testTheEverythingQuery() throws Exception { Assert.assertEquals(results.size(), 1); Assert.assertEquals(results.get(0), stats2); } + + @Test + public void testSortByMultipleColumns() throws Exception { + EntityManager em = emf.createEntityManager(); + QueryEngine engine = new SQLQueryEngine(em, dictionary); + + Map sortMap = new TreeMap<>(); + sortMap.put("lowScore", Sorting.SortOrder.desc); + sortMap.put("player.name", Sorting.SortOrder.asc); + + Query query = Query.builder() + .schema(playerStatsSchema) + .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) + .groupDimension(playerStatsSchema.getDimension("overallRating")) + .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) + .sorting(new Sorting(sortMap)) + .build(); + + List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) + .collect(Collectors.toList()); + + PlayerStats stats1 = new PlayerStats(); + stats1.setId("0"); + stats1.setLowScore(241); + stats1.setOverallRating("Great"); + stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); + + PlayerStats stats2 = new PlayerStats(); + stats2.setId("1"); + stats2.setLowScore(72); + stats2.setOverallRating("Good"); + stats2.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); + + Assert.assertEquals(results.size(), 2); + Assert.assertEquals(results.get(0), stats1); + Assert.assertEquals(results.get(1), stats2); + } } From 848463f24077009737e8a41631ef3a03db356f49 Mon Sep 17 00:00:00 2001 From: Jiaqi Liu <2257440489@qq.com> Date: Mon, 15 Jul 2019 16:55:13 -0700 Subject: [PATCH 29/47] Hydrate Relationship --- .../engine/AbstractEntityHydrator.java | 158 ++++++++++++++++++ .../aggregation/engine/SQLEntityHydrator.java | 84 ++++++++++ .../aggregation/engine/SQLQueryEngine.java | 87 ++-------- .../aggregation/engine/StitchList.java | 125 ++++++++++++++ .../engine/SQLQueryEngineTest.java | 29 +++- .../aggregation/example/Country.java | 32 ++++ 6 files changed, 434 insertions(+), 81 deletions(-) create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java create mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java new file mode 100644 index 0000000000..2f2c04d5f7 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java @@ -0,0 +1,158 @@ +package com.yahoo.elide.datastores.aggregation.engine; + +import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.datastores.aggregation.Query; +import com.yahoo.elide.datastores.aggregation.dimension.Dimension; +import com.yahoo.elide.datastores.aggregation.dimension.DimensionType; +import com.yahoo.elide.datastores.aggregation.engine.schema.SQLSchema; +import com.yahoo.elide.datastores.aggregation.metric.Metric; + +import com.google.common.base.Preconditions; + +import org.apache.commons.lang3.mutable.MutableInt; + +import lombok.AccessLevel; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * + */ +public abstract class AbstractEntityHydrator { + + @Getter(AccessLevel.PROTECTED) + private final EntityDictionary entityDictionary; + + @Getter(AccessLevel.PRIVATE) + private final StitchList stitchList; + + @Getter(AccessLevel.PRIVATE) + private final List results; + + @Getter(AccessLevel.PRIVATE) + private final Query query; + + public AbstractEntityHydrator(List results, Query query, EntityDictionary entityDictionary) { + this.stitchList = new StitchList(entityDictionary); + this.results = new ArrayList<>(results); + this.query = query; + this.entityDictionary = entityDictionary; + } + + /** + * Returns a list of relationship objects whose ID matches a specified list of ID's + *

+ * Note the relationship cannot be toMany. Regardless of ID duplicates, this method returns the list of relationship + * objects in the same size as the specified list of ID's, i.e. duplicate ID mapps to the same object in the + * returned list. For example: + *

+ * if {@code entityClass = Country.java}, {@code joinField = country}, and {@code joinFieldIds = [840, 840, 344]}, + * then this method returns {@code Country(id:840), Country(id:840), Country(id:344)}. + *

+ * If the ID list is empty or no matching objects are found, this method returns {@link Collections#emptyList()}. + * + * @param entityClass The type of relationship + * @param joinField The relationship field name + * @param joinFieldIds The specified list of join ID's against the relationshiop + * + * @return a list of hydrating values + */ + public abstract Map getRelationshipValues( + Class entityClass, + String joinField, + List joinFieldIds + ); + + public Iterable hydrate() { + //Coerce the results into entity objects. + MutableInt counter = new MutableInt(0); + + List queryResults = getResults().stream() + .map((result) -> { return result instanceof Object[] ? (Object []) result : new Object[] { result }; }) + .map((result) -> coerceObjectToEntity(result, counter)) + .collect(Collectors.toList()); + + if (getStitchList().shouldStitch()) { + // relationship is requested, stitch relationship then + populateObjectLookupTable(); + getStitchList().stitch(); + } + + return queryResults; + } + + /** + * Coerces results from a {@link Query} into an Object. + * + * @param counter Monotonically increasing number to generate IDs. + * @return A hydrated entity object. + */ + protected Object coerceObjectToEntity(Object[] result, MutableInt counter) { + Class entityClass = query.getSchema().getEntityClass(); + + //Get all the projections from the client query. + List projections = query.getMetrics().entrySet().stream() + .map(Map.Entry::getKey) + .map(Metric::getName) + .collect(Collectors.toList()); + + projections.addAll(query.getDimensions().stream() + .map(Dimension::getName) + .collect(Collectors.toList())); + + Preconditions.checkArgument(result.length == projections.size()); + + SQLSchema schema = (SQLSchema) query.getSchema(); + + //Construct the object. + Object entityInstance; + try { + entityInstance = entityClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new IllegalStateException(e); + } + + //Populate all of the fields. + for (int idx = 0; idx < result.length; idx++) { + Object value = result[idx]; + String fieldName = projections.get(idx); + + Dimension dim = schema.getDimension(fieldName); + if (dim != null && dim.getDimensionType() == DimensionType.ENTITY) { + getStitchList().todo(entityInstance, fieldName, value); + //We don't hydrate relationships here. + continue; + } + + getEntityDictionary().setValue(entityInstance, fieldName, value); + } + + //Set the ID (it must be coerced from an integer) + getEntityDictionary().setValue( + entityInstance, + getEntityDictionary().getIdFieldName(entityClass), + counter.getAndIncrement() + ); + + return entityInstance; + } + + private void populateObjectLookupTable() { + // mapping: relationship field name -> join ID's + Map> hydrationIdsByRelationship = getStitchList().getHydrationMapping(); + Class entityType = getQuery().getSchema().getEntityClass(); + + // hydrate each relationship + for (Map.Entry> entry : hydrationIdsByRelationship.entrySet()) { + String joinField = entry.getKey(); + List joinFieldIds = entry.getValue(); + + getStitchList().populateLookup(entityType, getRelationshipValues(entityType, joinField, joinFieldIds)); + } + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java new file mode 100644 index 0000000000..31e0eaaa22 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java @@ -0,0 +1,84 @@ +package com.yahoo.elide.datastores.aggregation.engine; + +import com.yahoo.elide.core.EntityDictionary; +import com.yahoo.elide.datastores.aggregation.Query; + +import lombok.AccessLevel; +import lombok.Getter; + +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import javax.persistence.EntityManager; + +public class SQLEntityHydrator extends AbstractEntityHydrator { + + @Getter(AccessLevel.PRIVATE) + private final EntityManager entityManager; + + public SQLEntityHydrator( + final List results, + final Query query, + final EntityDictionary entityDictionary, + final EntityManager entityManager + ) { + super(results, query, entityDictionary); + this.entityManager = entityManager; + } + + /** + * Returns a list of relationship objects whose ID matches a specified list of ID's + *

+ * Note the relationship cannot be toMany. Regardless of ID duplicates, this method returns the list of relationship + * objects in the same size as the specified list of ID's, i.e. duplicate ID mapps to the same object in the + * returned list. For example: + *

+ * if {@code entityClass = Country.java}, {@code joinField = country}, and {@code joinFieldIds = [840, 840, 344]}, + * then this method returns {@code Country(id:840), Country(id:840), Country(id:344)}. + *

+ * If the ID list is empty or no matching objects are found, this method returns {@link Collections#emptyList()}. + * + * @param entityClass The type of relationship + * @param joinField The relationship field name + * @param joinFieldIds The specified list of join ID's against the relationshiop + * + * @return a list of hydrating values + */ + @Override + public Map getRelationshipValues(Class entityClass, String joinField, List joinFieldIds) { + if (joinFieldIds.isEmpty()) { + return Collections.emptyMap(); + } + + List uniqueIds = joinFieldIds.stream().distinct().collect(Collectors.toCollection(LinkedList::new)); + + List loaded = getEntityManager() + .createQuery( + String.format( + "SELECT %s FROM %s WHERE %s IN :idList", + entityClass.getCanonicalName(), + entityClass.getCanonicalName(), + getEntityDictionary().getIdFieldName(entityClass) + ) + ) + .setParameter("idList", uniqueIds) + .getResultList(); + + // returns a mapping as [joinId(0) -> loaded(0), joinId(1) -> loaded(1), ...] + return IntStream.range(0, loaded.size()) + .boxed() + .map(i -> new AbstractMap.SimpleImmutableEntry<>(uniqueIds.get(i), loaded.get(i))) + .collect( + Collectors.collectingAndThen( + Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue), + Collections::unmodifiableMap) + ); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index f6b7d9d03a..8f1073e039 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -19,7 +19,6 @@ import com.yahoo.elide.datastores.aggregation.Query; import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.dimension.Dimension; -import com.yahoo.elide.datastores.aggregation.dimension.DimensionType; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromSubquery; import com.yahoo.elide.datastores.aggregation.engine.annotation.FromTable; import com.yahoo.elide.datastores.aggregation.engine.schema.SQLSchema; @@ -29,7 +28,7 @@ import com.yahoo.elide.utils.coerce.CoerceUtil; import com.google.common.base.Preconditions; -import org.apache.commons.lang3.mutable.MutableInt; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -69,7 +68,7 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) .stream() .filter((clazz) -> dictionary.getAnnotation(clazz, FromTable.class) != null - || dictionary.getAnnotation(clazz, FromSubquery.class) != null + || dictionary.getAnnotation(clazz, FromSubquery.class) != null ) .collect(Collectors.toMap( Function.identity(), @@ -105,8 +104,8 @@ public Iterable executeQuery(Query query) { //Run the Pagination query and log the time spent. long total = new TimedFunction<>(() -> { - return CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); - }, "Running Query: " + paginationSQL).get(); + return CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); + }, "Running Query: " + paginationSQL).get(); pagination.setPageTotals(total); } @@ -118,15 +117,9 @@ public Iterable executeQuery(Query query) { //Run the primary query and log the time spent. List results = new TimedFunction<>(() -> { return jpaQuery.getResultList(); - }, "Running Query: " + sql).get(); - + }, "Running Query: " + sql).get(); - //Coerce the results into entity objects. - MutableInt counter = new MutableInt(0); - return results.stream() - .map((result) -> { return result instanceof Object[] ? (Object []) result : new Object[] { result }; }) - .map((result) -> coerceObjectToEntity(query, result, counter)) - .collect(Collectors.toList()); + return new SQLEntityHydrator(results, query, dictionary, entityManager).hydrate(); } /** @@ -181,58 +174,6 @@ protected SQLQuery toSQL(Query query) { return builder.build(); } - /** - * Coerces results from a JPA query into an Object. - * @param query The client query - * @param result A row from the results. - * @param counter Monotonically increasing number to generate IDs. - * @return A hydrated entity object. - */ - protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt counter) { - Class entityClass = query.getSchema().getEntityClass(); - - //Get all the projections from the client query. - List projections = query.getMetrics().entrySet().stream() - .map(Map.Entry::getKey) - .map(Metric::getName) - .collect(Collectors.toList()); - - projections.addAll(query.getDimensions().stream() - .map(Dimension::getName) - .collect(Collectors.toList())); - - Preconditions.checkArgument(result.length == projections.size()); - - SQLSchema schema = (SQLSchema) query.getSchema(); - - //Construct the object. - Object entityInstance; - try { - entityInstance = entityClass.newInstance(); - } catch (InstantiationException | IllegalAccessException e) { - throw new IllegalStateException(e); - } - - //Populate all of the fields. - for (int idx = 0; idx < result.length; idx++) { - Object value = result[idx]; - String fieldName = projections.get(idx); - - Dimension dim = schema.getDimension(fieldName); - if (dim != null && dim.getDimensionType() == DimensionType.ENTITY) { - //We don't hydrate relationships here. - continue; - } - - dictionary.setValue(entityInstance, fieldName, value); - } - - //Set the ID (it must be coerced from an integer) - dictionary.setValue(entityInstance, dictionary.getIdFieldName(entityClass), counter.getAndIncrement()); - - return entityInstance; - } - /** * Translates a filter expression into SQL. * @param schema The schema being queried. @@ -241,8 +182,8 @@ protected Object coerceObjectToEntity(Query query, Object[] result, MutableInt c * @return A SQL expression */ private String translateFilterExpression(SQLSchema schema, - FilterExpression expression, - Function columnGenerator) { + FilterExpression expression, + Function columnGenerator) { HQLFilterOperation filterVisitor = new HQLFilterOperation(); return filterVisitor.apply(expression, columnGenerator); @@ -342,7 +283,7 @@ private String extractOrderBy(Class entityClass, Map * @param jpaQuery The JPA query */ private void supplyFilterQueryParameters(Query query, - javax.persistence.Query jpaQuery) { + javax.persistence.Query jpaQuery) { Collection predicates = new ArrayList<>(); if (query.getWhereFilter() != null) { @@ -397,9 +338,9 @@ private SQLQuery toPageTotalSQL(SQLQuery sql) { Query clientQuery = sql.getClientQuery(); String groupByDimensions = clientQuery.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name)) - .collect(Collectors.joining(",")); + .map(Dimension::getName) + .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name)) + .collect(Collectors.joining(",")); String projectionClause = String.format("SELECT COUNT(DISTINCT(%s))", groupByDimensions); @@ -455,8 +396,8 @@ private String extractGroupBy(Query query) { .collect(Collectors.toList()); return "GROUP BY " + dimensionProjections.stream() - .map((name) -> query.getSchema().getAlias() + "." + name) - .collect(Collectors.joining(",")); + .map((name) -> query.getSchema().getAlias() + "." + name) + .collect(Collectors.joining(",")); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java new file mode 100644 index 0000000000..88c93b9c16 --- /dev/null +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java @@ -0,0 +1,125 @@ +package com.yahoo.elide.datastores.aggregation.engine; + +import com.yahoo.elide.core.EntityDictionary; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * An auxiliary class for {@link AbstractEntityHydrator} and is responsible for setting relationship values of an entity + * instance. + *

+ * {@link StitchList} should not be subclassed. + *

+ * Concurrency note: {@link #stitch()} operation generally do not block, so may overlap with update operations ( + * {@link #todo(Object, String, Object)} and {@link #populateLookup(Class, Map)}). {@link #stitch()} reflects the + * results of the most recently completed update operations holding upon their onset. (More formally, an update + * operation bears a happens-before relation with a {@link #stitch()} operation.) + */ +public final class StitchList { + + /** + * A representation of an element in a {@link StitchList}. + */ + @Data + public static class Todo { + private final Object entityInstance; + private final String relationshipName; + private final Object foreignKey; + } + + /** + * Maps an relationship entity class to a Map of object ID to object instance. + *

+ * For example, [Country.class: [340: Country(id:340), 100: Country(id:100)]] + */ + @Getter(AccessLevel.PRIVATE) + private final Map, Map> objectLookups; + + /** + * List of relationships to hydrate + */ + @Getter(AccessLevel.PRIVATE) + private final List todoList; + + @Getter(AccessLevel.PRIVATE) + private final EntityDictionary entityDictionary; + + public StitchList(EntityDictionary entityDictionary) { + this.objectLookups = new ConcurrentHashMap<>(); + this.todoList = Collections.synchronizedList(new ArrayList<>()); + this.entityDictionary = entityDictionary; + } + + public boolean shouldStitch() { + return !getTodoList().isEmpty(); + } + + public void todo(Object entityInstance, String fieldName, Object value) { + getTodoList().add(new Todo(entityInstance, fieldName, value)); + } + + /** + * Any existing values will be overwritten. + * + * @param relationshipType + * @param idToInstance + */ + public void populateLookup(Class relationshipType, Map idToInstance) { + getObjectLookups().put(relationshipType, idToInstance); + } + + public void stitch() { + for (Todo todo : getTodoList()) { + Object entityInstance = todo.getEntityInstance(); + String relationshipName = todo.getRelationshipName(); + Object foreignKey = todo.getForeignKey(); + + Object relationshipValue = getObjectLookups().get(entityInstance.getClass()).get(foreignKey); + + getEntityDictionary().setValue(entityInstance, relationshipName, relationshipValue); + } + } + + /** + * For example, given the following {@code todoList}: + * 

+     * {@code
+     *     [PlayerStats, country, 344]
+     *     [PlayerStats, country, 840]
+     *     [PlayerStats, country, 344]
+     *     [PlayerStats, player, 1]
+     *     [PlayerStats, player, 1]
+     *     [PlayerStats, player, 1]
+     * }
+     *
+ * this method returns a map of the following: + * 
+     *     [
+     *         "country": [344, 840]
+     *         "player": [1]
+     *     ]
+     *
+ * + * @return a mapping from relationship name to an ordered list of relationship join ID's + */ + public Map> getHydrationMapping() { + return getTodoList().stream() + .collect( + Collectors.groupingBy( + StitchList.Todo::getRelationshipName, + Collectors.mapping(StitchList.Todo::getForeignKey, Collectors.toCollection(LinkedList::new)) + ) + ); + } +} diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java index 8330910245..653e578775 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java @@ -137,6 +137,7 @@ public void testFilterJoin() throws Exception { .metric(playerStatsSchema.getMetric("lowScore"), Sum.class) .metric(playerStatsSchema.getMetric("highScore"), Sum.class) .groupDimension(playerStatsSchema.getDimension("overallRating")) + .groupDimension(playerStatsSchema.getDimension("country")) .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate")) .whereFilter(filterParser.parseFilterExpression("country.name=='United States'", PlayerStats.class, false)) @@ -145,23 +146,35 @@ public void testFilterJoin() throws Exception { List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false) .collect(Collectors.toList()); + Country expectedCountry = new Country(); + expectedCountry.setId("840"); + expectedCountry.setIsoCode("USA"); + expectedCountry.setName("United States"); + + PlayerStats stats1 = new PlayerStats(); stats1.setId("0"); - stats1.setLowScore(72); - stats1.setHighScore(1234); - stats1.setOverallRating("Good"); - stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); + stats1.setLowScore(241); + stats1.setHighScore(2412); + stats1.setOverallRating("Great"); + stats1.setCountry(expectedCountry); + stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); PlayerStats stats2 = new PlayerStats(); stats2.setId("1"); - stats2.setLowScore(241); - stats2.setHighScore(2412); - stats2.setOverallRating("Great"); - stats2.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00")); + stats2.setLowScore(72); + stats2.setHighScore(1234); + stats2.setOverallRating("Good"); + stats2.setCountry(expectedCountry); + stats2.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00")); Assert.assertEquals(results.size(), 2); Assert.assertEquals(results.get(0), stats1); Assert.assertEquals(results.get(1), stats2); + + // test join + PlayerStats actualStats1 = (PlayerStats) results.get(0); + Assert.assertNotNull(actualStats1.getCountry()); } @Test diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java index f9804b308b..647d4390a6 100644 --- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java +++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java @@ -10,6 +10,9 @@ import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; import com.yahoo.elide.datastores.aggregation.annotation.FriendlyName; +import java.util.Objects; +import java.util.StringJoiner; + import javax.persistence.Entity; import javax.persistence.Id; @@ -52,4 +55,33 @@ public String getName() { public void setName(final String name) { this.name = name; } + + @Override + public boolean equals(final Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + + final Country country = (Country) other; + return getId().equals(country.getId()) + && getIsoCode().equals(country.getIsoCode()) + && getName().equals(country.getName()); + } + + @Override + public int hashCode() { + return Objects.hash(getId(), getIsoCode(), getName()); + } + + @Override + public String toString() { + return new StringJoiner(", ", Country.class.getSimpleName() + "[", "]") + .add("id='" + getId() + "'") + .add("isoCode='" + getIsoCode() + "'") + .add("name='" + getName() + "'") + .toString(); + } } From dae947e326f0a3f4e43383063bdb3cae4d6e8e59 Mon Sep 17 00:00:00 2001 From: Jiaqi Liu <2257440489@qq.com> Date: Thu, 18 Jul 2019 17:55:37 -0700 Subject: [PATCH 30/47] Self-review --- .../engine/AbstractEntityHydrator.java | 2 +- .../aggregation/engine/SQLEntityHydrator.java | 21 +------------------ .../aggregation/engine/StitchList.java | 3 ++- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java index 2f2c04d5f7..c845f35f53 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java @@ -145,12 +145,12 @@ protected Object coerceObjectToEntity(Object[] result, MutableInt counter) { private void populateObjectLookupTable() { // mapping: relationship field name -> join ID's Map> hydrationIdsByRelationship = getStitchList().getHydrationMapping(); - Class entityType = getQuery().getSchema().getEntityClass(); // hydrate each relationship for (Map.Entry> entry : hydrationIdsByRelationship.entrySet()) { String joinField = entry.getKey(); List joinFieldIds = entry.getValue(); + Class entityType = getEntityDictionary().getType(getQuery().getSchema().getEntityClass(), joinField); getStitchList().populateLookup(entityType, getRelationshipValues(entityType, joinField, joinFieldIds)); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java index 31e0eaaa22..f640aacfd4 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java @@ -33,24 +33,6 @@ public SQLEntityHydrator( this.entityManager = entityManager; } - /** - * Returns a list of relationship objects whose ID matches a specified list of ID's - *

- * Note the relationship cannot be toMany. Regardless of ID duplicates, this method returns the list of relationship - * objects in the same size as the specified list of ID's, i.e. duplicate ID mapps to the same object in the - * returned list. For example: - *

- * if {@code entityClass = Country.java}, {@code joinField = country}, and {@code joinFieldIds = [840, 840, 344]}, - * then this method returns {@code Country(id:840), Country(id:840), Country(id:344)}. - *

- * If the ID list is empty or no matching objects are found, this method returns {@link Collections#emptyList()}. - * - * @param entityClass The type of relationship - * @param joinField The relationship field name - * @param joinFieldIds The specified list of join ID's against the relationshiop - * - * @return a list of hydrating values - */ @Override public Map getRelationshipValues(Class entityClass, String joinField, List joinFieldIds) { if (joinFieldIds.isEmpty()) { @@ -62,8 +44,7 @@ public Map getRelationshipValues(Class entityClass, String jo List loaded = getEntityManager() .createQuery( String.format( - "SELECT %s FROM %s WHERE %s IN :idList", - entityClass.getCanonicalName(), + "SELECT e FROM %s e WHERE %s IN (:idList)", entityClass.getCanonicalName(), getEntityDictionary().getIdFieldName(entityClass) ) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java index 88c93b9c16..870c7173e6 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java @@ -85,7 +85,8 @@ public void stitch() { String relationshipName = todo.getRelationshipName(); Object foreignKey = todo.getForeignKey(); - Object relationshipValue = getObjectLookups().get(entityInstance.getClass()).get(foreignKey); + Class relationshipType = getEntityDictionary().getType(entityInstance, relationshipName); + Object relationshipValue = getObjectLookups().get(relationshipType).get(foreignKey); getEntityDictionary().setValue(entityInstance, relationshipName, relationshipValue); } From 75b03ff7a48bdc9e828533950b4e33fcdfa07b17 Mon Sep 17 00:00:00 2001 From: Jiaqi Liu <2257440489@qq.com> Date: Thu, 18 Jul 2019 19:37:35 -0700 Subject: [PATCH 31/47] Self-review --- .../engine/AbstractEntityHydrator.java | 57 ++++++++++++++++--- .../aggregation/engine/SQLEntityHydrator.java | 31 +++++++--- .../aggregation/engine/SQLQueryEngine.java | 24 ++++---- .../aggregation/engine/StitchList.java | 51 ++++++++++++++--- .../datastores/aggregation/metric/Metric.java | 2 + .../aggregation/example/Country.java | 32 +---------- 6 files changed, 132 insertions(+), 65 deletions(-) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java index c845f35f53..9c84021536 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java @@ -1,7 +1,13 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ package com.yahoo.elide.datastores.aggregation.engine; import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.datastores.aggregation.Query; +import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.dimension.Dimension; import com.yahoo.elide.datastores.aggregation.dimension.DimensionType; import com.yahoo.elide.datastores.aggregation.engine.schema.SQLSchema; @@ -21,7 +27,10 @@ import java.util.stream.Collectors; /** - * + * {@link AbstractEntityHydrator} hydrates the entity loaded by {@link QueryEngine#executeQuery(Query)}. + *

+ * {@link AbstractEntityHydrator} is not thread-safe and should be accessed by only 1 thread in this application, + * because it uses {@link StitchList}. See {@link StitchList} for more details. */ public abstract class AbstractEntityHydrator { @@ -37,6 +46,13 @@ public abstract class AbstractEntityHydrator { @Getter(AccessLevel.PRIVATE) private final Query query; + /** + * Constructor. + * + * @param results The loaded objects from {@link QueryEngine#executeQuery(Query)} + * @param query The query passed to {@link QueryEngine#executeQuery(Query)} to load the objects + * @param entityDictionary An object that sets entity instance values and provides entity metadata info + */ public AbstractEntityHydrator(List results, Query query, EntityDictionary entityDictionary) { this.stitchList = new StitchList(entityDictionary); this.results = new ArrayList<>(results); @@ -45,16 +61,37 @@ public AbstractEntityHydrator(List results, Query query, EntityDictionar } /** - * Returns a list of relationship objects whose ID matches a specified list of ID's + * Loads a map of relationship object ID to relationship object instance. *

- * Note the relationship cannot be toMany. Regardless of ID duplicates, this method returns the list of relationship - * objects in the same size as the specified list of ID's, i.e. duplicate ID mapps to the same object in the - * returned list. For example: + * Note the relationship cannot be toMany. This method will be invoked for every relationship field of the + * requested entity. Its implementation should return the result of the following query *

- * if {@code entityClass = Country.java}, {@code joinField = country}, and {@code joinFieldIds = [840, 840, 344]}, - * then this method returns {@code Country(id:840), Country(id:840), Country(id:344)}. + * Given a relationship {@code joinField} in an entity of type {@code entityClass}, loads all relationship + * objects whose foreign keys are one of the specified list, {@code joinFieldIds}. *

- * If the ID list is empty or no matching objects are found, this method returns {@link Collections#emptyList()}. + * For example, when the relationship is loaded from SQL and we have the following example identity: + * 

+     * {@code
+     * public class PlayerStats {
+     *     private String id;
+     *     private Country country;
+     *
+     *     @OneToOne
+     *     @JoinColumn(name = "country_id")
+     *     public Country getCountry() {
+     *         return country;
+     * }
+     *
+ * In this case {@code entityClass = PlayerStats.class}; {@code joinField = "country"}. If {@code country} is + * requested in {@code PlayerStats} query and 3 stats, for example, are found in database whose country ID's are + * {@code joinFieldIds = [840, 344, 840]}, then this method should effectively run the following query (JPQL as + * example) + * 
+     * {@code
+     *     SELECT e FROM country_table e WHERE country_id IN (840, 344);
+     * }
+     *
+ * and returns the map of [840: Country(id:840), 344: Country(id:344)] * * @param entityClass The type of relationship * @param joinField The relationship field name @@ -142,6 +179,10 @@ protected Object coerceObjectToEntity(Object[] result, MutableInt counter) { return entityInstance; } + /** + * Foe each requested relationship, run a single query to load all relationship objects whose ID's are involved in + * the request. + */ private void populateObjectLookupTable() { // mapping: relationship field name -> join ID's Map> hydrationIdsByRelationship = getStitchList().getHydrationMapping(); diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java index f640aacfd4..ca9ce5f6a7 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java @@ -1,7 +1,13 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ package com.yahoo.elide.datastores.aggregation.engine; import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.datastores.aggregation.Query; +import com.yahoo.elide.datastores.aggregation.QueryEngine; import lombok.AccessLevel; import lombok.Getter; @@ -18,16 +24,27 @@ import javax.persistence.EntityManager; +/** + * {@link SQLEntityHydrator} hydrates the entity loaded by {@link SQLQueryEngine#executeQuery(Query)}. + */ public class SQLEntityHydrator extends AbstractEntityHydrator { @Getter(AccessLevel.PRIVATE) private final EntityManager entityManager; + /** + * Constructor. + * + * @param results The loaded objects from {@link SQLQueryEngine#executeQuery(Query)} + * @param query The query passed to {@link SQLQueryEngine#executeQuery(Query)} to load the objects + * @param entityDictionary An object that sets entity instance values and provides entity metadata info + * @param entityManager An service that issues JPQL queries to load relationship objects + */ public SQLEntityHydrator( - final List results, - final Query query, - final EntityDictionary entityDictionary, - final EntityManager entityManager + List results, + Query query, + EntityDictionary entityDictionary, + EntityManager entityManager ) { super(results, query, entityDictionary); this.entityManager = entityManager; @@ -56,10 +73,6 @@ public Map getRelationshipValues(Class entityClass, String jo return IntStream.range(0, loaded.size()) .boxed() .map(i -> new AbstractMap.SimpleImmutableEntry<>(uniqueIds.get(i), loaded.get(i))) - .collect( - Collectors.collectingAndThen( - Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue), - Collections::unmodifiableMap) - ); + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java index 8f1073e039..f5c837ae60 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java @@ -68,7 +68,7 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary) .stream() .filter((clazz) -> dictionary.getAnnotation(clazz, FromTable.class) != null - || dictionary.getAnnotation(clazz, FromSubquery.class) != null + || dictionary.getAnnotation(clazz, FromSubquery.class) != null ) .collect(Collectors.toMap( Function.identity(), @@ -104,8 +104,8 @@ public Iterable executeQuery(Query query) { //Run the Pagination query and log the time spent. long total = new TimedFunction<>(() -> { - return CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); - }, "Running Query: " + paginationSQL).get(); + return CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class); + }, "Running Query: " + paginationSQL).get(); pagination.setPageTotals(total); } @@ -117,7 +117,7 @@ public Iterable executeQuery(Query query) { //Run the primary query and log the time spent. List results = new TimedFunction<>(() -> { return jpaQuery.getResultList(); - }, "Running Query: " + sql).get(); + }, "Running Query: " + sql).get(); return new SQLEntityHydrator(results, query, dictionary, entityManager).hydrate(); } @@ -182,8 +182,8 @@ protected SQLQuery toSQL(Query query) { * @return A SQL expression */ private String translateFilterExpression(SQLSchema schema, - FilterExpression expression, - Function columnGenerator) { + FilterExpression expression, + Function columnGenerator) { HQLFilterOperation filterVisitor = new HQLFilterOperation(); return filterVisitor.apply(expression, columnGenerator); @@ -283,7 +283,7 @@ private String extractOrderBy(Class entityClass, Map * @param jpaQuery The JPA query */ private void supplyFilterQueryParameters(Query query, - javax.persistence.Query jpaQuery) { + javax.persistence.Query jpaQuery) { Collection predicates = new ArrayList<>(); if (query.getWhereFilter() != null) { @@ -338,9 +338,9 @@ private SQLQuery toPageTotalSQL(SQLQuery sql) { Query clientQuery = sql.getClientQuery(); String groupByDimensions = clientQuery.getDimensions().stream() - .map(Dimension::getName) - .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name)) - .collect(Collectors.joining(",")); + .map(Dimension::getName) + .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name)) + .collect(Collectors.joining(",")); String projectionClause = String.format("SELECT COUNT(DISTINCT(%s))", groupByDimensions); @@ -396,8 +396,8 @@ private String extractGroupBy(Query query) { .collect(Collectors.toList()); return "GROUP BY " + dimensionProjections.stream() - .map((name) -> query.getSchema().getAlias() + "." + name) - .collect(Collectors.joining(",")); + .map((name) -> query.getSchema().getAlias() + "." + name) + .collect(Collectors.joining(",")); } diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java index 870c7173e6..c6d58456f8 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java @@ -1,3 +1,8 @@ +/* + * Copyright 2019, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ package com.yahoo.elide.datastores.aggregation.engine; import com.yahoo.elide.core.EntityDictionary; @@ -29,7 +34,7 @@ public final class StitchList { /** - * A representation of an element in a {@link StitchList}. + * A representation of an TODO item in a {@link StitchList}. */ @Data public static class Todo { @@ -39,7 +44,7 @@ public static class Todo { } /** - * Maps an relationship entity class to a Map of object ID to object instance. + * Maps an relationship entity class to a map of object ID to object instance. *

* For example, [Country.class: [340: Country(id:340), 100: Country(id:100)]] */ @@ -55,30 +60,54 @@ public static class Todo { @Getter(AccessLevel.PRIVATE) private final EntityDictionary entityDictionary; + /** + * Constructor. + * + * @param entityDictionary An object that sets entity instance values and provides entity metadata info + */ public StitchList(EntityDictionary entityDictionary) { this.objectLookups = new ConcurrentHashMap<>(); this.todoList = Collections.synchronizedList(new ArrayList<>()); this.entityDictionary = entityDictionary; } + /** + * Returns whether or not the entity instances in this {@link StitchList} have relationships that are unset. + * + * @return {@code true} if the entity instances in this {@link StitchList} should be further hydrated because they + * have one or more relationship fields. + */ public boolean shouldStitch() { return !getTodoList().isEmpty(); } + /** + * Enqueues an entity instance which will be further hydrated on one of its relationship fields later + * + * @param entityInstance The entity instance to be hydrated + * @param fieldName The relationship field to hydrate in the entity instance + * @param value The foreign key between the entity instance and the field entity. + */ public void todo(Object entityInstance, String fieldName, Object value) { getTodoList().add(new Todo(entityInstance, fieldName, value)); } /** - * Any existing values will be overwritten. + * Sets all the relationship values of an requested entity. + *

+ * Values associated with the existing key will be overwritten. * - * @param relationshipType - * @param idToInstance + * @param relationshipType The type of the relationship to set + * @param idToInstance A map from relationship ID to the actual relationship instance with that ID */ public void populateLookup(Class relationshipType, Map idToInstance) { getObjectLookups().put(relationshipType, idToInstance); } + /** + * Stitch all entity instances currently in this {@link StitchList} by setting their relationship fields whose + * values are determined by relationship ID's. + */ public void stitch() { for (Todo todo : getTodoList()) { Object entityInstance = todo.getEntityInstance(); @@ -93,6 +122,8 @@ public void stitch() { } /** + * Returns a mapping from relationship name to an immutable list of foreign key ID objects. + *

* For example, given the following {@code todoList}: * 

      * {@code
@@ -118,8 +149,14 @@ public Map> getHydrationMapping() {
         return getTodoList().stream()
                 .collect(
                         Collectors.groupingBy(
-                                StitchList.Todo::getRelationshipName,
-                                Collectors.mapping(StitchList.Todo::getForeignKey, Collectors.toCollection(LinkedList::new))
+                                Todo::getRelationshipName,
+                                Collectors.mapping(
+                                        Todo::getForeignKey,
+                                        Collectors.collectingAndThen(
+                                                Collectors.toCollection(LinkedList::new),
+                                                Collections::unmodifiableList
+                                        )
+                                )
                         )
                 );
     }
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
index 049df39e0f..23afb9c009 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
@@ -9,6 +9,8 @@
 import com.yahoo.elide.datastores.aggregation.annotation.MetricAggregation;
 import com.yahoo.elide.datastores.aggregation.annotation.MetricComputation;
 
+import lombok.Data;
+
 import java.io.Serializable;
 import java.util.List;
 import java.util.Optional;
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java
index 647d4390a6..3cc198cd1e 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java
@@ -10,6 +10,8 @@
 import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize;
 import com.yahoo.elide.datastores.aggregation.annotation.FriendlyName;
 
+import lombok.Data;
+
 import java.util.Objects;
 import java.util.StringJoiner;
 
@@ -19,6 +21,7 @@
 /**
  * A root level entity for testing AggregationDataStore.
  */
+@Data
 @Entity
 @Include(rootLevel = true)
 @Cardinality(size = CardinalitySize.SMALL)
@@ -55,33 +58,4 @@ public String getName() {
     public void setName(final String name) {
         this.name = name;
     }
-
-    @Override
-    public boolean equals(final Object other) {
-        if (this == other) {
-            return true;
-        }
-        if (other == null || getClass() != other.getClass()) {
-            return false;
-        }
-
-        final Country country = (Country) other;
-        return getId().equals(country.getId())
-                && getIsoCode().equals(country.getIsoCode())
-                && getName().equals(country.getName());
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(getId(), getIsoCode(), getName());
-    }
-
-    @Override
-    public String toString() {
-        return new StringJoiner(", ", Country.class.getSimpleName() + "[", "]")
-                .add("id='" + getId() + "'")
-                .add("isoCode='" + getIsoCode() + "'")
-                .add("name='" + getName() + "'")
-                .toString();
-    }
 }

From c8ff674702b24b43ee02b8cd7dda783ab4d8836f Mon Sep 17 00:00:00 2001
From: Jiaqi Liu <2257440489@qq.com>
Date: Thu, 18 Jul 2019 19:40:02 -0700
Subject: [PATCH 32/47] Self-review

---
 .../datastores/aggregation/engine/AbstractEntityHydrator.java | 1 -
 .../datastores/aggregation/engine/SQLEntityHydrator.java      | 3 ---
 .../elide/datastores/aggregation/engine/SQLQueryEngine.java   | 4 ++--
 .../yahoo/elide/datastores/aggregation/engine/StitchList.java | 1 -
 .../com/yahoo/elide/datastores/aggregation/metric/Metric.java | 2 --
 .../yahoo/elide/datastores/aggregation/example/Country.java   | 3 ---
 6 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
index 9c84021536..f2ce541a78 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
@@ -21,7 +21,6 @@
 import lombok.Getter;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
index ca9ce5f6a7..40a64af053 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
@@ -7,15 +7,12 @@
 
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.datastores.aggregation.Query;
-import com.yahoo.elide.datastores.aggregation.QueryEngine;
 
 import lombok.AccessLevel;
 import lombok.Getter;
 
 import java.util.AbstractMap;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
index f5c837ae60..49f948fa61 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
@@ -396,8 +396,8 @@ private String extractGroupBy(Query query) {
                 .collect(Collectors.toList());
 
         return "GROUP BY " + dimensionProjections.stream()
-            .map((name) -> query.getSchema().getAlias() + "." + name)
-            .collect(Collectors.joining(","));
+                    .map((name) -> query.getSchema().getAlias() + "." + name)
+                    .collect(Collectors.joining(","));
 
     }
 
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
index c6d58456f8..b84c2428f0 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
@@ -13,7 +13,6 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
index 23afb9c009..049df39e0f 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
@@ -9,8 +9,6 @@
 import com.yahoo.elide.datastores.aggregation.annotation.MetricAggregation;
 import com.yahoo.elide.datastores.aggregation.annotation.MetricComputation;
 
-import lombok.Data;
-
 import java.io.Serializable;
 import java.util.List;
 import java.util.Optional;
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java
index 3cc198cd1e..6da056c26f 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/example/Country.java
@@ -12,9 +12,6 @@
 
 import lombok.Data;
 
-import java.util.Objects;
-import java.util.StringJoiner;
-
 import javax.persistence.Entity;
 import javax.persistence.Id;
 

From 1123067605d6491ae7957a351c3979f4229d81b0 Mon Sep 17 00:00:00 2001
From: Jiaqi Liu <2257440489@qq.com>
Date: Thu, 18 Jul 2019 20:01:10 -0700
Subject: [PATCH 33/47] Self-review

---
 .../datastores/aggregation/engine/AbstractEntityHydrator.java   | 2 +-
 .../elide/datastores/aggregation/engine/SQLEntityHydrator.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
index f2ce541a78..7040d7dec5 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
@@ -98,7 +98,7 @@ public AbstractEntityHydrator(List results, Query query, EntityDictionar
      *
      * @return a list of hydrating values
      */
-    public abstract Map getRelationshipValues(
+    protected abstract Map getRelationshipValues(
             Class entityClass,
             String joinField,
             List joinFieldIds
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
index 40a64af053..0dde38a000 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
@@ -48,7 +48,7 @@ public SQLEntityHydrator(
     }
 
     @Override
-    public Map getRelationshipValues(Class entityClass, String joinField, List joinFieldIds) {
+    protected Map getRelationshipValues(Class entityClass, String joinField, List joinFieldIds) {
         if (joinFieldIds.isEmpty()) {
             return Collections.emptyMap();
         }

From 316c79b79ca460dafd41609d8a432aedfd052bfb Mon Sep 17 00:00:00 2001
From: Jiaqi Liu <2257440489@qq.com>
Date: Thu, 18 Jul 2019 22:09:21 -0700
Subject: [PATCH 34/47] Self-review

---
 .../datastores/aggregation/engine/SQLEntityHydrator.java    | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
index 0dde38a000..58e3f7ed97 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
@@ -48,7 +48,11 @@ public SQLEntityHydrator(
     }
 
     @Override
-    protected Map getRelationshipValues(Class entityClass, String joinField, List joinFieldIds) {
+    protected Map getRelationshipValues(
+            Class entityClass,
+            String joinField,
+            List joinFieldIds
+    ) {
         if (joinFieldIds.isEmpty()) {
             return Collections.emptyMap();
         }

From 673774f53c3707ef9f17045f7b895a5836161600 Mon Sep 17 00:00:00 2001
From: Jiaqi Liu <2257440489@qq.com>
Date: Fri, 19 Jul 2019 12:36:43 -0700
Subject: [PATCH 35/47] Address comments from @aklish

---
 .../datastores/aggregation/engine/StitchList.java      | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
index b84c2428f0..83abdf116f 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
@@ -13,6 +13,7 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -24,11 +25,6 @@
  * instance.
  * 

  * {@link StitchList} should not be subclassed.
- * 

- * Concurrency note: {@link #stitch()} operation generally do not block, so may overlap with update operations (
- * {@link #todo(Object, String, Object)} and {@link #populateLookup(Class, Map)}). {@link #stitch()} reflects the
- * results of the most recently completed update operations holding upon their onset. (More formally, an update
- * operation bears a happens-before relation with a {@link #stitch()} operation.)
  */
 public final class StitchList {
 
@@ -65,8 +61,8 @@ public static class Todo {
      * @param entityDictionary  An object that sets entity instance values and provides entity metadata info
      */
     public StitchList(EntityDictionary entityDictionary) {
-        this.objectLookups = new ConcurrentHashMap<>();
-        this.todoList = Collections.synchronizedList(new ArrayList<>());
+        this.objectLookups = new HashMap<>();
+        this.todoList = new ArrayList<>();
         this.entityDictionary = entityDictionary;
     }
 

From 686c1560cc642298ebc06bd35bdad3ada4433cc7 Mon Sep 17 00:00:00 2001
From: Han Chen 
Date: Thu, 15 Aug 2019 15:18:33 -0500
Subject: [PATCH 36/47] Refactor EntityHydrator (#893)

---
 .../engine/AbstractEntityHydrator.java        | 67 ++++++++++---------
 .../aggregation/engine/StitchList.java        |  2 -
 2 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
index 7040d7dec5..b0c1da55f0 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
@@ -10,7 +10,6 @@
 import com.yahoo.elide.datastores.aggregation.QueryEngine;
 import com.yahoo.elide.datastores.aggregation.dimension.Dimension;
 import com.yahoo.elide.datastores.aggregation.dimension.DimensionType;
-import com.yahoo.elide.datastores.aggregation.engine.schema.SQLSchema;
 import com.yahoo.elide.datastores.aggregation.metric.Metric;
 
 import com.google.common.base.Preconditions;
@@ -21,6 +20,7 @@
 import lombok.Getter;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -39,8 +39,8 @@ public abstract class AbstractEntityHydrator {
     @Getter(AccessLevel.PRIVATE)
     private final StitchList stitchList;
 
-    @Getter(AccessLevel.PRIVATE)
-    private final List results;
+    @Getter(AccessLevel.PROTECTED)
+    private final List> results = new ArrayList<>();
 
     @Getter(AccessLevel.PRIVATE)
     private final Query query;
@@ -54,9 +54,34 @@ public abstract class AbstractEntityHydrator {
      */
     public AbstractEntityHydrator(List results, Query query, EntityDictionary entityDictionary) {
         this.stitchList = new StitchList(entityDictionary);
-        this.results = new ArrayList<>(results);
         this.query = query;
         this.entityDictionary = entityDictionary;
+
+        //Get all the projections from the client query.
+        List projections = this.query.getMetrics().keySet().stream()
+                .map(Metric::getName)
+                .collect(Collectors.toList());
+
+        projections.addAll(this.query.getDimensions().stream()
+                .map(Dimension::getName)
+                .collect(Collectors.toList()));
+
+
+        results.forEach(result -> {
+            Map row = new HashMap<>();
+
+            Object[] resultValues = result instanceof Object[] ? (Object[]) result : new Object[] { result };
+
+            Preconditions.checkArgument(projections.size() == resultValues.length);
+
+            for (int idx = 0; idx < resultValues.length; idx++) {
+                Object value = resultValues[idx];
+                String fieldName = projections.get(idx);
+                row.put(fieldName, value);
+            }
+
+            this.results.add(row);
+        });
     }
 
     /**
@@ -109,7 +134,6 @@ public Iterable hydrate() {
         MutableInt counter = new MutableInt(0);
 
         List queryResults = getResults().stream()
-                .map((result) -> { return result instanceof Object[] ? (Object []) result : new Object[] { result }; })
                 .map((result) -> coerceObjectToEntity(result, counter))
                 .collect(Collectors.toList());
 
@@ -128,23 +152,9 @@ public Iterable hydrate() {
      * @param counter Monotonically increasing number to generate IDs.
      * @return A hydrated entity object.
      */
-    protected Object coerceObjectToEntity(Object[] result, MutableInt counter) {
+    protected Object coerceObjectToEntity(Map result, MutableInt counter) {
         Class entityClass = query.getSchema().getEntityClass();
 
-        //Get all the projections from the client query.
-        List projections = query.getMetrics().entrySet().stream()
-                .map(Map.Entry::getKey)
-                .map(Metric::getName)
-                .collect(Collectors.toList());
-
-        projections.addAll(query.getDimensions().stream()
-                .map(Dimension::getName)
-                .collect(Collectors.toList()));
-
-        Preconditions.checkArgument(result.length == projections.size());
-
-        SQLSchema schema = (SQLSchema) query.getSchema();
-
         //Construct the object.
         Object entityInstance;
         try {
@@ -153,20 +163,15 @@ protected Object coerceObjectToEntity(Object[] result, MutableInt counter) {
             throw new IllegalStateException(e);
         }
 
-        //Populate all of the fields.
-        for (int idx = 0; idx < result.length; idx++) {
-            Object value = result[idx];
-            String fieldName = projections.get(idx);
+        result.forEach((fieldName, value) -> {
+            Dimension dim = query.getSchema().getDimension(fieldName);
 
-            Dimension dim = schema.getDimension(fieldName);
             if (dim != null && dim.getDimensionType() == DimensionType.ENTITY) {
-                getStitchList().todo(entityInstance, fieldName, value);
-                //We don't hydrate relationships here.
-                continue;
+                getStitchList().todo(entityInstance, fieldName, value); // We don't hydrate relationships here.
+            } else {
+                getEntityDictionary().setValue(entityInstance, fieldName, value);
             }
-
-            getEntityDictionary().setValue(entityInstance, fieldName, value);
-        }
+        });
 
         //Set the ID (it must be coerced from an integer)
         getEntityDictionary().setValue(
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
index 83abdf116f..6d0e4464be 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
@@ -6,7 +6,6 @@
 package com.yahoo.elide.datastores.aggregation.engine;
 
 import com.yahoo.elide.core.EntityDictionary;
-
 import lombok.AccessLevel;
 import lombok.Data;
 import lombok.Getter;
@@ -17,7 +16,6 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
 /**

From 968556aaf3adc2b9d0ac9e3425adb352feca0d2a Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Thu, 26 Sep 2019 15:54:08 -0500
Subject: [PATCH 37/47] rebase

---
 .../elide/core/EntityDictionaryTest.java      |  12 -
 .../elide/datastores/aggregation/Schema.java  | 309 ------------------
 .../aggregation/engine/schema/SQLSchema.java  |   3 -
 .../engine/SQLQueryEngineTest.java            |   6 -
 .../hql/RootCollectionFetchQueryBuilder.java  |   2 +-
 .../RootCollectionPageTotalsQueryBuilder.java |   2 +-
 pom.xml                                       |   4 +-
 7 files changed, 4 insertions(+), 334 deletions(-)
 delete mode 100644 elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java

diff --git a/elide-core/src/test/java/com/yahoo/elide/core/EntityDictionaryTest.java b/elide-core/src/test/java/com/yahoo/elide/core/EntityDictionaryTest.java
index 0d3554a220..34c3657a0e 100644
--- a/elide-core/src/test/java/com/yahoo/elide/core/EntityDictionaryTest.java
+++ b/elide-core/src/test/java/com/yahoo/elide/core/EntityDictionaryTest.java
@@ -511,16 +511,4 @@ public void testIsValidField() {
         assertTrue(isValidField(Job.class, "title"));
         assertFalse(isValidField(Job.class, "foo"));
     }
-
-    @Test
-    public void testAttributeOrRelationAnnotationExists() {
-        Assert.assertTrue(attributeOrRelationAnnotationExists(Job.class, "jobId", Id.class));
-        Assert.assertFalse(attributeOrRelationAnnotationExists(Job.class, "title", OneToOne.class));
-    }
-
-    @Test
-    public void testIsValidField() {
-        Assert.assertTrue(isValidField(Job.class, "title"));
-        Assert.assertFalse(isValidField(Job.class, "foo"));
-    }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java
deleted file mode 100644
index dfdae6fd04..0000000000
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/Schema.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Copyright 2019, Yahoo Inc.
- * Licensed under the Apache License, Version 2.0
- * See LICENSE file in project root for terms.
- */
-package com.yahoo.elide.datastores.aggregation;
-
-import com.yahoo.elide.core.DataStore;
-import com.yahoo.elide.core.EntityDictionary;
-import com.yahoo.elide.core.filter.FilterPredicate;
-import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize;
-import com.yahoo.elide.datastores.aggregation.annotation.Meta;
-import com.yahoo.elide.datastores.aggregation.annotation.MetricAggregation;
-import com.yahoo.elide.datastores.aggregation.annotation.MetricComputation;
-import com.yahoo.elide.datastores.aggregation.annotation.Temporal;
-import com.yahoo.elide.datastores.aggregation.dimension.DegenerateDimension;
-import com.yahoo.elide.datastores.aggregation.dimension.Dimension;
-import com.yahoo.elide.datastores.aggregation.dimension.EntityDimension;
-import com.yahoo.elide.datastores.aggregation.dimension.TimeDimension;
-import com.yahoo.elide.datastores.aggregation.metric.AggregatedMetric;
-import com.yahoo.elide.datastores.aggregation.metric.Aggregation;
-import com.yahoo.elide.datastores.aggregation.metric.Metric;
-
-import lombok.AccessLevel;
-import lombok.Getter;
-import lombok.Singular;
-import lombok.extern.slf4j.Slf4j;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-/**
- * {@link Schema} keeps track of table, metrics, and dimensions of an entity for AggregationDataStore.
- * 

- * On calling {@link DataStore#populateEntityDictionary(EntityDictionary)}, one {@link Schema} will be created for each
- * entity.
- * 

- * By overriding {@link #constructDimension(String, Class, EntityDictionary)} and
- * {@link #constructMetric(String, Class, EntityDictionary)}, people can have new schema backed by their own defined
- * {@link Dimension}s and {@link Metric}s.
- * 

- * {@link Schema} is thread-safe and can be accessed by multiple-threads.
- */
-@Slf4j
-public class Schema {
-
-    @Getter
-    private final Class entityClass;
-
-    @Singular
-    @Getter
-    private final Set metrics;
-
-    @Singular
-    @Getter
-    private final Set dimensions;
-    @Getter(value = AccessLevel.PRIVATE)
-    private final EntityDictionary entityDictionary;
-
-    /**
-     * Constructor
-     * 

-     * This constructor calls {@link #constructDimension(String, Class, EntityDictionary)} and
-     * {@link #constructMetric(String, Class, EntityDictionary)} ()} to construct all {@link Dimension}s and
-     * {@link Metric}s associated with the entity class passed in.
-     *
-     * @param cls  The type of the entity, whose {@link Schema} is to be constructed
-     * @param entityDictionary  The meta info object that helps to construct {@link Schema}
-     *
-     * @throws NullPointerException if anyone of the arguments is {@code null}
-     */
-    public Schema(Class cls, EntityDictionary entityDictionary) {
-        this.entityClass = Objects.requireNonNull(cls, "cls");
-        this.entityDictionary = Objects.requireNonNull(entityDictionary, "entityDictionary");
-
-        this.metrics = getAllMetrics();
-        this.dimensions = getAllDimensions();
-    }
-
-    /**
-     * Returns an immutable view of all {@link Dimension}s and {@link Metric}s described by this {@link Schema}.
-     *
-     * @return union of all {@link Dimension}s and {@link Metric}s under this {@link Schema}
-     */
-    public Set getAllColumns() {
-        return Stream.concat(getMetrics().stream(), getDimensions().stream())
-                .map(item -> (Column) item)
-                .collect(
-                        Collectors.collectingAndThen(
-                                Collectors.toSet(),
-                                Collections::unmodifiableSet
-                        )
-                );
-    }
-
-    /**
-     * Finds the {@link Dimension} by name.
-     *
-     * @param dimensionName  The entity field name associated with the searched {@link Dimension}
-     *
-     * @return {@link Dimension} found or {@code null} if not found
-     */
-    public Dimension getDimension(String dimensionName) {
-        return getDimensions().stream()
-                .filter(dimension -> dimension.getName().equals(dimensionName))
-                .findAny()
-                .orElse(null);
-    }
-
-    /**
-     * Finds the {@link Metric} by name.
-     *
-     * @param metricName  The entity field name associated with the searched {@link Metric}
-     *
-     * @return {@link Metric} found or {@code null} if not found
-     */
-    public Metric getMetric(String metricName) {
-        return getMetrics().stream()
-                .filter(metric -> metric.getName().equals(metricName))
-                .findAny()
-                .orElse(null);
-    }
-
-    /**
-     * Returns whether or not an entity field is a metric field.
-     * 

-     * A field is a metric field iff that field is annotated by at least one of
-     * 
    
-     *     
  1.  {@link MetricAggregation}
-     *     
  2.  {@link MetricComputation}
-     * 

-     *
-     * @param fieldName  The entity field
-     *
-     * @return {@code true} if the field is a metric field
-     */
-    public boolean isMetricField(String fieldName) {
-        return getEntityDictionary().attributeOrRelationAnnotationExists(
-                getEntityClass(), fieldName, MetricAggregation.class
-        )
-                || getEntityDictionary().attributeOrRelationAnnotationExists(
-                getEntityClass(), fieldName, MetricComputation.class
-        );
-    }
-
-    /**
-     * An alias to assign this schema.
-     * @return an alias that can be used in SQL.
-     */
-    public String getAlias() {
-        return FilterPredicate.getTypeAlias(entityClass);
-    }
-
-    /**
-     * Constructs a new {@link Metric} instance.
-     *
-     * @param metricField  The entity field of the metric being constructed
-     * @param cls  The entity that contains the metric being constructed
-     * @param entityDictionary  The auxiliary object that offers binding info used to construct this
-     * {@link Metric}
-     *
-     * @return a {@link Metric}
-     */
-    protected Metric constructMetric(String metricField, Class cls, EntityDictionary entityDictionary) {
-        Meta metaData = entityDictionary.getAttributeOrRelationAnnotation(cls, Meta.class, metricField);
-        Class fieldType = entityDictionary.getType(cls, metricField);
-
-        // get all metric aggregations
-        List> aggregations = Arrays.stream(
-                entityDictionary.getAttributeOrRelationAnnotation(
-                        cls,
-                        MetricAggregation.class,
-                        metricField
-                ).aggregations()
-        )
-                .collect(
-                        Collectors.collectingAndThen(
-                                Collectors.toList(),
-                                Collections::unmodifiableList
-                        )
-                );
-
-        return new AggregatedMetric(
-                this,
-                metricField,
-                metaData,
-                fieldType,
-                aggregations
-        );
-    }
-
-    /**
-     * Constructs and returns a new instance of {@link Dimension}.
-     *
-     * @param dimensionField  The entity field of the dimension being constructed
-     * @param cls  The entity that contains the dimension being constructed
-     * @param entityDictionary  The auxiliary object that offers binding info used to construct this
-     * {@link Dimension}
-     *
-     * @return a {@link Dimension}
-     */
-    protected Dimension constructDimension(String dimensionField, Class cls, EntityDictionary entityDictionary) {
-        // field with ToMany relationship is not supported
-        if (getEntityDictionary().getRelationshipType(cls, dimensionField).isToMany()) {
-            String message = String.format("ToMany relationship is not supported in '%s'", cls.getCanonicalName());
-            log.error(message);
-            throw new IllegalStateException(message);
-        }
-
-        Meta metaData = entityDictionary.getAttributeOrRelationAnnotation(cls, Meta.class, dimensionField);
-        Class fieldType = entityDictionary.getType(cls, dimensionField);
-
-        String friendlyName = EntityDimension.getFriendlyNameField(cls, entityDictionary);
-        CardinalitySize cardinality = EntityDimension.getEstimatedCardinality(dimensionField, cls, entityDictionary);
-
-        if (entityDictionary.isRelation(cls, dimensionField)) {
-            // relationship field
-            return new EntityDimension(
-                    this,
-                    dimensionField,
-                    metaData,
-                    fieldType,
-                    cardinality,
-                    friendlyName
-            );
-        } else if (!getEntityDictionary().attributeOrRelationAnnotationExists(cls, dimensionField, Temporal.class)) {
-            // regular field
-            return new DegenerateDimension(
-                    this,
-                    dimensionField,
-                    metaData,
-                    fieldType,
-                    cardinality,
-                    friendlyName,
-                    DegenerateDimension.parseColumnType(dimensionField, cls, entityDictionary)
-            );
-        } else {
-            // temporal field
-            Temporal temporal = getEntityDictionary().getAttributeOrRelationAnnotation(
-                    cls,
-                    Temporal.class,
-                    dimensionField
-            );
-
-            return new TimeDimension(
-                    this,
-                    dimensionField,
-                    metaData,
-                    fieldType,
-                    cardinality,
-                    friendlyName,
-                    TimeZone.getTimeZone(temporal.timeZone()),
-                    temporal.timeGrain()
-            );
-
-        }
-    }
-
-    /**
-     * Constructs all metrics found in an entity.
-     * 

-     * This method calls {@link #constructMetric(String, Class, EntityDictionary)} to create each dimension inside the
-     * entity
-     *
-     * @return all metric fields as {@link Metric} objects
-     */
-    private Set getAllMetrics() {
-        return getEntityDictionary().getAllFields(getEntityClass()).stream()
-                .filter(this::isMetricField)
-                // TODO: remove the filter() below when computedMetric is supported
-                .filter(field ->
-                        getEntityDictionary()
-                                .attributeOrRelationAnnotationExists(getEntityClass(), field, MetricAggregation.class)
-                )
-                .map(field -> constructMetric(field, getEntityClass(), getEntityDictionary()))
-                .collect(
-                        Collectors.collectingAndThen(
-                                Collectors.toSet(),
-                                Collections::unmodifiableSet
-                        )
-                );
-    }
-
-    /**
-     * Constructs all dimensions found in an entity.
-     * 

-     * This method calls {@link #constructDimension(String, Class, EntityDictionary)} to create each dimension inside
-     * the entity
-     *
-     * @return all non-metric fields as {@link Dimension} objects
-     */
-    private Set getAllDimensions() {
-        return getEntityDictionary().getAllFields(getEntityClass()).stream()
-                .filter(field -> !isMetricField(field))
-                .map(field -> constructDimension(field, getEntityClass(), getEntityDictionary()))
-                .collect(
-                        Collectors.collectingAndThen(
-                                Collectors.toSet(),
-                                Collections::unmodifiableSet
-                        )
-                );
-    }
-}
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java
index 889ad24c35..8b2395f5c3 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/schema/SQLSchema.java
@@ -24,9 +24,6 @@ public class SQLSchema extends Schema {
     @Getter
     private boolean isSubquery;
 
-    @Getter
-    private boolean isSubquery;
-
     @Getter
     private String tableDefinition;
 
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java
index 653e578775..c29fce753f 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java
@@ -43,11 +43,6 @@ public class SQLQueryEngineTest {
     private EntityDictionary dictionary;
     private RSQLFilterDialect filterParser;
 
-    private Schema playerStatsSchema;
-    private Schema playerStatsViewSchema;
-    private EntityDictionary dictionary;
-    private RSQLFilterDialect filterParser;
-
     public SQLQueryEngineTest() {
         emf = Persistence.createEntityManagerFactory("aggregationStore");
         dictionary = new EntityDictionary(new HashMap<>());
@@ -124,7 +119,6 @@ public void testDegenerateDimensionFilter() throws Exception {
 
         Assert.assertEquals(results.size(), 1);
         Assert.assertEquals(results.get(0), stats1);
-        Assert.assertEquals(results.get(1), stats2);
     }
 
     @Test
diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java
index e57aa1dfdb..169f3ab43b 100644
--- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java
+++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java
@@ -7,7 +7,7 @@
 
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.core.filter.FilterPredicate;
-import com.yahoo.elide.core.filter.FilterTranslator;
+import com.yahoo.elide.core.filter.HQLFilterOperation;
 import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor;
 import com.yahoo.elide.core.hibernate.Query;
 import com.yahoo.elide.core.hibernate.Session;
diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java
index 267d27c0ea..5758e6449c 100644
--- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java
+++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java
@@ -7,7 +7,7 @@
 
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.core.filter.FilterPredicate;
-import com.yahoo.elide.core.filter.FilterTranslator;
+import com.yahoo.elide.core.filter.HQLFilterOperation;
 import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor;
 import com.yahoo.elide.core.hibernate.Query;
 import com.yahoo.elide.core.hibernate.Session;
diff --git a/pom.xml b/pom.xml
index 91f19cb60c..100bc52434 100644
--- a/pom.xml
+++ b/pom.xml
@@ -400,11 +400,11 @@
                         1.11.2
                     
 
-                   
+                    
                         org.apache.maven.scm
                         maven-scm-api
                         1.11.2
-                   
+                    
                 
                 
                     @{project.version}

From 2d0f76cf88b310bf771f12384d2a9cebcfd49417 Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Thu, 26 Sep 2019 16:05:24 -0500
Subject: [PATCH 38/47] keep Jiaqi's changes

---
 .../yahoo/elide/core/EntityDictionary.java    | 154 +++++++++---------
 .../aggregation/engine/SQLQueryEngine.java    |  24 ++-
 2 files changed, 90 insertions(+), 88 deletions(-)

diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
index 76b57fb0a8..6a14acbf37 100644
--- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
+++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
@@ -81,7 +81,7 @@ public class EntityDictionary {
     protected final BiMap> checkNames;
     protected final Injector injector;
 
-    public final static String REGULAR_ID_NAME = "id";
+    private final static String REGULAR_ID_NAME = "id";
     private final static ConcurrentHashMap SIMPLE_NAMES = new ConcurrentHashMap<>();
 
     /**
@@ -169,10 +169,12 @@ public static Method findMethod(Class entityClass, String name, Class... p
     }
 
     protected EntityBinding getEntityBinding(Class entityClass) {
-        if (isMappedInterface(entityClass)) {
-            return EMPTY_BINDING;
-        }
-        return entityBindings.getOrDefault(lookupEntityClass(entityClass), EMPTY_BINDING);
+        return entityBindings.computeIfAbsent(entityClass, cls -> {
+            if (isMappedInterface(cls)) {
+                return EMPTY_BINDING;
+            }
+            return entityBindings.getOrDefault(lookupEntityClass(cls), EMPTY_BINDING);
+        });
     }
 
     public boolean isMappedInterface(Class interfaceClass) {
@@ -246,8 +248,8 @@ public ParseTree getPermissionsForClass(Class resourceClass, Class resourceClass,
-            String field,
-            Class annotationClass) {
+                                            String field,
+                                            Class annotationClass) {
         EntityBinding binding = getEntityBinding(resourceClass);
         return binding.entityPermissions.getFieldChecksForPermission(field, annotationClass);
     }
@@ -614,6 +616,7 @@ public RelationshipType getRelationshipType(Object entity, String relation) {
      * 
      * {@code
      * public class Address {
+     *
      *     {@literal @}Id
      *     private Long id
      *
@@ -635,6 +638,7 @@ public RelationshipType getRelationshipType(Object entity, String relation) {
      * 
      * {@code
      * public class Address {
+     *
      *     {@literal @}Id
      *     private Long surrogateKey
      *
@@ -657,7 +661,7 @@ public RelationshipType getRelationshipType(Object entity, String relation) {
      * calling this method has undefined behavior
      *
      * @param entityClass Entity class
-     * @param identifier  Identifier/Field to lookup type
+     * @param identifier  Field to lookup type
      * @return Type of entity
      */
     public Class getType(Class entityClass, String identifier) {
@@ -695,7 +699,7 @@ public Class getParameterizedType(Class entityClass, String identifier) {
      * Retrieve the parameterized type for the given field.
      *
      * @param entityClass the entity class
-     * @param identifier  the identifier/field name
+     * @param identifier  the identifier
      * @param paramIndex  the index of the parameterization
      * @return Entity type for field otherwise null.
      */
@@ -811,6 +815,7 @@ public void bindEntity(Class cls) {
         if (entityBindings.getOrDefault(lookupEntityClass(cls), EMPTY_BINDING) != EMPTY_BINDING) {
             return;
         }
+        entityBindings.remove(lookupEntityClass(cls));
 
         Annotation annotation = getFirstAnnotation(cls, Arrays.asList(Include.class, Exclude.class));
         Include include = annotation instanceof Include ? (Include) annotation : null;
@@ -1209,13 +1214,43 @@ public  List walkEntityGraph(Set> entities,  Function, T
 
     /**
      * Returns whether or not a class is already bound.
-     * @param cls The class to verify.
+     * @param cls
      * @return true if the class is bound.  False otherwise.
      */
     public boolean hasBinding(Class cls) {
         return bindJsonApiToEntity.contains(cls);
     }
 
+    /**
+     * Returns whether or not a specified annotation is present on an entity field or its corresponding method.
+     *
+     * @param fieldName  The entity field
+     * @param annotationClass  The provided annotation class
+     *
+     * @param   The type of the {@code annotationClass}
+     *
+     * @return {@code true} if the field is annotated by the {@code annotationClass}
+     */
+    public  boolean attributeOrRelationAnnotationExists(
+            Class cls,
+            String fieldName,
+            Class annotationClass
+    ) {
+        return getAttributeOrRelationAnnotation(cls, annotationClass, fieldName) != null;
+    }
+
+    /**
+     * Returns whether or not a specified field exists in an entity.
+     *
+     * @param cls  The entity
+     * @param fieldName  The provided field to check
+     *
+     * @return {@code true} if the field exists in the entity
+     */
+    public boolean isValidField(Class cls, String fieldName) {
+        return getAllFields(cls).contains(fieldName);
+    }
+
     /**
      * Invoke the get[fieldName] method on the target object OR get the field with the corresponding name.
      * @param target the object to get
@@ -1243,7 +1278,6 @@ public Object getValue(Object target, String fieldName, RequestScope scope) {
         throw new InvalidAttributeException(fieldName, getJsonAliasFor(target.getClass()));
     }
 
-
     /**
      * Invoke the set[fieldName] method on the target object OR set the field with the corresponding name.
      * @param fieldName the field name to set or invoke equivalent set method
@@ -1279,6 +1313,21 @@ public void setValue(Object target, String fieldName, Object value) {
         }
     }
 
+    /**
+     * Handle an invocation target exception.
+     *
+     * @param e Exception the exception encountered while reflecting on an object's field
+     * @return Equivalent runtime exception
+     */
+    private static RuntimeException handleInvocationTargetException(InvocationTargetException e) {
+        Throwable exception = e.getTargetException();
+        if (exception instanceof HttpStatusException || exception instanceof WebApplicationException) {
+            return (RuntimeException) exception;
+        }
+        log.error("Caught an unexpected exception (rethrowing as internal server error)", e);
+        return new InternalServerErrorException("Unexpected exception caught", e);
+    }
+
     /**
      * Coerce provided value into expected class type.
      *
@@ -1287,80 +1336,18 @@ public void setValue(Object target, String fieldName, Object value) {
      * @param fieldClass expected class type
      * @return coerced value
      */
-    public Object coerce(Object target, Object value, String fieldName, Class fieldClass) {
+    Object coerce(Object target, Object value, String fieldName, Class fieldClass) {
         if (fieldClass != null && Collection.class.isAssignableFrom(fieldClass) && value instanceof Collection) {
             return coerceCollection(target, (Collection) value, fieldName, fieldClass);
         }
 
         if (fieldClass != null && Map.class.isAssignableFrom(fieldClass) && value instanceof Map) {
-            return coerceMap(target, (Map) value, fieldName);
+            return coerceMap(target, (Map) value, fieldName, fieldClass);
         }
 
         return CoerceUtil.coerce(value, fieldClass);
     }
 
-    private Map coerceMap(Object target, Map values, String fieldName) {
-        Class keyType = getParameterizedType(target, fieldName, 0);
-        Class valueType = getParameterizedType(target, fieldName, 1);
-
-        // Verify the existing Map
-        if (isValidParameterizedMap(values, keyType, valueType)) {
-            return values;
-        }
-
-        LinkedHashMap result = new LinkedHashMap<>(values.size());
-        for (Map.Entry entry : values.entrySet()) {
-            result.put(CoerceUtil.coerce(entry.getKey(), keyType), CoerceUtil.coerce(entry.getValue(), valueType));
-        }
-
-        return result;
-    }
-
-    /**
-     * Returns whether or not a specified annotation is present on an entity field or its corresponding method.
-     *
-     * @param fieldName  The entity field
-     * @param annotationClass  The provided annotation class
-     *
-     * @param   The type of the {@code annotationClass}
-     *
-     * @return {@code true} if the field is annotated by the {@code annotationClass}
-     */
-    public  boolean attributeOrRelationAnnotationExists(
-            Class cls,
-            String fieldName,
-            Class annotationClass
-    ) {
-        return getAttributeOrRelationAnnotation(cls, annotationClass, fieldName) != null;
-    }
-
-    /**
-     * Returns whether or not a specified field exists in an entity.
-     *
-     * @param cls  The entity
-     * @param fieldName  The provided field to check
-     *
-     * @return {@code true} if the field exists in the entity
-     */
-    public boolean isValidField(Class cls, String fieldName) {
-        return getAllFields(cls).contains(fieldName);
-    }
-
-    /**
-     * Handle an invocation target exception.
-     *
-     * @param e Exception the exception encountered while reflecting on an object's field
-     * @return Equivalent runtime exception
-     */
-    private static RuntimeException handleInvocationTargetException(InvocationTargetException e) {
-        Throwable exception = e.getTargetException();
-        if (exception instanceof HttpStatusException || exception instanceof WebApplicationException) {
-            return (RuntimeException) exception;
-        }
-        log.error("Caught an unexpected exception (rethrowing as internal server error)", e);
-        return new InternalServerErrorException("Unexpected exception caught", e);
-    }
-
     private Collection coerceCollection(Object target, Collection values, String fieldName, Class fieldClass) {
         Class providedType = getParameterizedType(target, fieldName);
 
@@ -1390,6 +1377,23 @@ private Collection coerceCollection(Object target, Collection values, String
         return list;
     }
 
+    private Map coerceMap(Object target, Map values, String fieldName, Class fieldClass) {
+        Class keyType = getParameterizedType(target, fieldName, 0);
+        Class valueType = getParameterizedType(target, fieldName, 1);
+
+        // Verify the existing Map
+        if (isValidParameterizedMap(values, keyType, valueType)) {
+            return values;
+        }
+
+        LinkedHashMap result = new LinkedHashMap<>(values.size());
+        for (Map.Entry entry : values.entrySet()) {
+            result.put(CoerceUtil.coerce(entry.getKey(), keyType), CoerceUtil.coerce(entry.getValue(), valueType));
+        }
+
+        return result;
+    }
+
     private boolean isValidParameterizedMap(Map values, Class keyType, Class valueType) {
         for (Map.Entry entry : values.entrySet()) {
             Object key = entry.getKey();
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
index 49f948fa61..64a004ff41 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
@@ -68,7 +68,7 @@ public SQLQueryEngine(EntityManager entityManager, EntityDictionary dictionary)
                 .stream()
                 .filter((clazz) ->
                         dictionary.getAnnotation(clazz, FromTable.class) != null
-                        || dictionary.getAnnotation(clazz, FromSubquery.class) != null
+                                || dictionary.getAnnotation(clazz, FromSubquery.class) != null
                 )
                 .collect(Collectors.toMap(
                         Function.identity(),
@@ -103,9 +103,10 @@ public Iterable executeQuery(Query query) {
                 supplyFilterQueryParameters(query, pageTotalQuery);
 
                 //Run the Pagination query and log the time spent.
-                long total = new TimedFunction<>(() -> {
-                        return CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class);
-                    }, "Running Query: " + paginationSQL).get();
+                long total = new TimedFunction<>(
+                        () -> CoerceUtil.coerce(pageTotalQuery.getSingleResult(), Long.class),
+                        "Running Query: " + paginationSQL
+                ).get();
 
                 pagination.setPageTotals(total);
             }
@@ -115,9 +116,7 @@ public Iterable executeQuery(Query query) {
         supplyFilterQueryParameters(query, jpaQuery);
 
         //Run the primary query and log the time spent.
-        List results = new TimedFunction<>(() -> {
-            return jpaQuery.getResultList();
-            }, "Running Query: " + sql).get();
+        List results = new TimedFunction<>(() -> jpaQuery.getResultList(), "Running Query: " + sql).get();
 
         return new SQLEntityHydrator(results, query, dictionary, entityManager).hydrate();
     }
@@ -338,9 +337,9 @@ private SQLQuery toPageTotalSQL(SQLQuery sql) {
         Query clientQuery = sql.getClientQuery();
 
         String groupByDimensions = clientQuery.getDimensions().stream()
-            .map(Dimension::getName)
-            .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name))
-            .collect(Collectors.joining(","));
+                .map(Dimension::getName)
+                .map((name) -> getColumnName(clientQuery.getSchema().getEntityClass(), name))
+                .collect(Collectors.joining(","));
 
         String projectionClause = String.format("SELECT COUNT(DISTINCT(%s))", groupByDimensions);
 
@@ -396,9 +395,8 @@ private String extractGroupBy(Query query) {
                 .collect(Collectors.toList());
 
         return "GROUP BY " + dimensionProjections.stream()
-                    .map((name) -> query.getSchema().getAlias() + "." + name)
-                    .collect(Collectors.joining(","));
-
+                .map((name) -> query.getSchema().getAlias() + "." + name)
+                .collect(Collectors.joining(","));
     }
 
     /**

From abee6bd42e9367591b8e9406ce198d0e7e436d06 Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Thu, 26 Sep 2019 16:24:27 -0500
Subject: [PATCH 39/47] fix id

---
 .../src/main/java/com/yahoo/elide/core/EntityDictionary.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
index 6a14acbf37..00d0b93090 100644
--- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
+++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
@@ -81,7 +81,7 @@ public class EntityDictionary {
     protected final BiMap> checkNames;
     protected final Injector injector;
 
-    private final static String REGULAR_ID_NAME = "id";
+    public final static String REGULAR_ID_NAME = "id";
     private final static ConcurrentHashMap SIMPLE_NAMES = new ConcurrentHashMap<>();
 
     /**

From 1ef36965b75f92786efa73aece9401356a89fd82 Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Fri, 27 Sep 2019 09:55:24 -0500
Subject: [PATCH 40/47] fix maven verify

---
 .../aggregation/dimension/EntityDimension.java         |  2 +-
 .../aggregation/engine/AbstractEntityHydrator.java     |  7 ++++---
 .../datastores/aggregation/engine/SQLQueryEngine.java  |  5 ++---
 .../aggregation/metric/AggregatedMetric.java           | 10 +++-------
 .../elide/datastores/aggregation/metric/Metric.java    |  4 ++--
 .../yahoo/elide/datastores/aggregation/SchemaTest.java |  3 +--
 pom.xml                                                |  4 ++--
 7 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java
index f95726862f..70d76531b6 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimension.java
@@ -140,13 +140,13 @@ public EntityDimension(
     /**
      * Constructor.
      *
+     * @param schema The schema this {@link Column} belongs to.
      * @param dimensionField  The entity field or relation that this {@link Dimension} represents
      * @param annotation  Provides static meta data about this {@link Dimension}
      * @param fieldType  The Java type for this entity field or relation
      * @param dimensionType  The physical storage structure backing this {@link Dimension}, such as a table or a column
      * @param cardinality  The estimated cardinality of this {@link Dimension} in SQL table
      * @param friendlyName  A human-readable name representing this {@link Dimension}
-     *
      * @throws NullPointerException any argument, except for {@code annotation}, is {@code null}
      */
     protected EntityDimension(
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
index b0c1da55f0..325530ecff 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
@@ -95,15 +95,15 @@ public AbstractEntityHydrator(List results, Query query, EntityDictionar
      * 

      * For example, when the relationship is loaded from SQL and we have the following example identity:
      * 
-     * {@code
      * public class PlayerStats {
      *     private String id;
      *     private Country country;
      *
-     *     @OneToOne
-     *     @JoinColumn(name = "country_id")
+     *     @OneToOne
+     *     @JoinColumn(name = "country_id")
      *     public Country getCountry() {
      *         return country;
+     *     }
      * }
      * 

      * In this case {@code entityClass = PlayerStats.class}; {@code joinField = "country"}. If {@code country} is
@@ -149,6 +149,7 @@ public Iterable hydrate() {
     /**
      * Coerces results from a {@link Query} into an Object.
      *
+     * @param result a fieldName-value map
      * @param counter Monotonically increasing number to generate IDs.
      * @return A hydrated entity object.
      */
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
index 64a004ff41..cba4fe0b65 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
@@ -38,7 +38,6 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -362,7 +361,7 @@ private String extractProjection(Query query) {
                 .map((entry) -> {
                     Metric metric = entry.getKey();
                     Class agg = entry.getValue();
-                    return metric.getMetricExpression(Optional.of(agg)) + " AS " + metric.getName();
+                    return metric.getMetricExpression(agg) + " AS " + metric.getName();
                 })
                 .collect(Collectors.toList());
 
@@ -428,6 +427,6 @@ private String generateHavingClauseColumnReference(FilterPredicate predicate, Qu
         Metric metric = schema.getMetric(last.getFieldName());
         Class agg = query.getMetrics().get(metric);
 
-        return metric.getMetricExpression(Optional.of(agg));
+        return metric.getMetricExpression(agg);
     }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java
index f8149c1208..20c3ebea3a 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetric.java
@@ -62,20 +62,16 @@ public AggregatedMetric(
     }
 
     @Override
-    public String getMetricExpression(final Optional> aggregation) {
-        if (!aggregation.isPresent()) {
-            return "";
-        }
-
+    public String getMetricExpression(final Class aggregation) {
         try {
-            Class clazz = Class.forName(aggregation.get().getCanonicalName());
+            Class clazz = Class.forName(aggregation.getCanonicalName());
             Constructor ctor = clazz.getConstructor();
             Aggregation instance = (Aggregation) ctor.newInstance();
             return String.format(instance.getAggFunctionFormat(), schema.getAlias() + "." + name);
         } catch (Exception exception) {
             String message = String.format(
                     "Cannot generate aggregation function for '%s'",
-                    aggregation.get().getCanonicalName()
+                    aggregation.getCanonicalName()
             );
             log.error(message, exception);
             throw new IllegalStateException(message, exception);
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
index 049df39e0f..d7152adefc 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metric/Metric.java
@@ -11,7 +11,6 @@
 
 import java.io.Serializable;
 import java.util.List;
-import java.util.Optional;
 
 /**
  * Elide's definition of metric.
@@ -52,9 +51,10 @@ public interface Metric extends Serializable {
     /**
      * Returns a metric expression that represents a specified aggregation.
      *
+     * @param aggregation aggregation type to be applied
      * @return a arithmetic formula for computing this {@link Metric} or default aggregation UDF on a base/simple metric
      */
-    String getMetricExpression(Optional> aggregation);
+    String getMetricExpression(Class aggregation);
 
     /**
      * Returns a list of supported aggregations with the first as the default aggregation.
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
index 1a3d989250..6c921411bb 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
@@ -20,7 +20,6 @@
 import org.testng.annotations.Test;
 
 import java.util.Collections;
-import java.util.Optional;
 
 public class SchemaTest {
 
@@ -51,7 +50,7 @@ public void testGetDimension() {
     @Test
     public void testGetMetric() {
         Assert.assertEquals(
-                playerStatsSchema.getMetric("highScore").getMetricExpression(Optional.of(Max.class)),
+                playerStatsSchema.getMetric("highScore").getMetricExpression(Max.class),
                 "MAX(com_yahoo_elide_datastores_aggregation_example_PlayerStats.highScore)"
         );
     }
diff --git a/pom.xml b/pom.xml
index 100bc52434..9ef26451e9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -79,7 +79,7 @@
         1.2.3
         9.4.19.v20190610
         2.9.0
-        2.9.9
+        2.9.10
         2.28
         3.6.10.Final
         8.0.16
@@ -246,7 +246,7 @@
             
                 com.fasterxml.jackson.core
                 jackson-databind
-                2.9.9.3
+                ${version.jackson}
             
             
                 com.jayway.restassured

From ea712b58b0819d18798f9b75dded08d218d9676d Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Fri, 27 Sep 2019 12:49:46 -0500
Subject: [PATCH 41/47] Remove HQLFilterOperation

---
 .../elide-datastore-aggregation/pom.xml       |  11 -
 .../aggregation/engine/SQLQueryEngine.java    |  12 +-
 .../elide/core/filter/FilterTranslator.java   |   4 +-
 .../elide/core/filter/HQLFilterOperation.java | 220 ------------------
 .../hql/RootCollectionFetchQueryBuilder.java  |   4 +-
 .../RootCollectionPageTotalsQueryBuilder.java |   4 +-
 6 files changed, 11 insertions(+), 244 deletions(-)
 delete mode 100644 elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java

diff --git a/elide-datastore/elide-datastore-aggregation/pom.xml b/elide-datastore/elide-datastore-aggregation/pom.xml
index 8c0e460410..c1a471088e 100644
--- a/elide-datastore/elide-datastore-aggregation/pom.xml
+++ b/elide-datastore/elide-datastore-aggregation/pom.xml
@@ -56,17 +56,6 @@
             lombok
         
 
-        
-            com.yahoo.elide
-            elide-datastore-hibernate
-            4.4.5-SNAPSHOT
-        
-
-        
-            org.projectlombok
-            lombok
-        
-
         
         
             org.hibernate.javax.persistence
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
index cba4fe0b65..170da7a723 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngine.java
@@ -11,7 +11,7 @@
 import com.yahoo.elide.core.TimedFunction;
 import com.yahoo.elide.core.exceptions.InvalidPredicateException;
 import com.yahoo.elide.core.filter.FilterPredicate;
-import com.yahoo.elide.core.filter.HQLFilterOperation;
+import com.yahoo.elide.core.filter.FilterTranslator;
 import com.yahoo.elide.core.filter.expression.FilterExpression;
 import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor;
 import com.yahoo.elide.core.pagination.Pagination;
@@ -140,12 +140,12 @@ protected SQLQuery toSQL(Query query) {
 
         if (query.getWhereFilter() != null) {
             joinPredicates.addAll(extractPathElements(query.getWhereFilter()));
-            builder.whereClause("WHERE " + translateFilterExpression(schema, query.getWhereFilter(),
+            builder.whereClause("WHERE " + translateFilterExpression(query.getWhereFilter(),
                     this::generateWhereClauseColumnReference));
         }
 
         if (query.getHavingFilter() != null) {
-            builder.havingClause("HAVING " + translateFilterExpression(schema, query.getHavingFilter(),
+            builder.havingClause("HAVING " + translateFilterExpression(query.getHavingFilter(),
                     (predicate) -> { return generateHavingClauseColumnReference(predicate, query); }));
         }
 
@@ -174,15 +174,13 @@ protected SQLQuery toSQL(Query query) {
 
     /**
      * Translates a filter expression into SQL.
-     * @param schema The schema being queried.
      * @param expression The filter expression
      * @param columnGenerator A function which generates a column reference in SQL from a FilterPredicate.
      * @return A SQL expression
      */
-    private String translateFilterExpression(SQLSchema schema,
-                                             FilterExpression expression,
+    private String translateFilterExpression(FilterExpression expression,
                                              Function columnGenerator) {
-        HQLFilterOperation filterVisitor = new HQLFilterOperation();
+        FilterTranslator filterVisitor = new FilterTranslator();
 
         return filterVisitor.apply(expression, columnGenerator);
     }
diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/FilterTranslator.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/FilterTranslator.java
index d334acb2ee..a9604ecaad 100644
--- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/FilterTranslator.java
+++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/FilterTranslator.java
@@ -202,8 +202,8 @@ public static void registerJPQLGenerator(Operator op,
      * @return Returns null if no generator is registered.
      */
     public static JPQLPredicateGenerator lookupJPQLGenerator(Operator op,
-                                           Class entityClass,
-                                           String fieldName) {
+                                                             Class entityClass,
+                                                             String fieldName) {
         return predicateOverrides.get(Triple.of(op, entityClass, fieldName));
     }
 
diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java
deleted file mode 100644
index 625d3af97d..0000000000
--- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/filter/HQLFilterOperation.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Copyright 2016, Yahoo Inc.
- * Licensed under the Apache License, Version 2.0
- * See LICENSE file in project root for terms.
- */
-package com.yahoo.elide.core.filter;
-
-import com.yahoo.elide.core.exceptions.InvalidPredicateException;
-import com.yahoo.elide.core.exceptions.InvalidValueException;
-import com.yahoo.elide.core.filter.FilterPredicate.FilterParameter;
-import com.yahoo.elide.core.filter.expression.AndFilterExpression;
-import com.yahoo.elide.core.filter.expression.FilterExpression;
-import com.yahoo.elide.core.filter.expression.FilterExpressionVisitor;
-import com.yahoo.elide.core.filter.expression.NotFilterExpression;
-import com.yahoo.elide.core.filter.expression.OrFilterExpression;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-
-import java.util.List;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-/**
- * FilterOperation that creates Hibernate query language fragments.
- */
-public class HQLFilterOperation implements FilterOperation {
-    private static final String FILTER_PATH_NOT_NULL = "Filtering field path cannot be empty.";
-    private static final String FILTER_ALIAS_NOT_NULL = "Filtering alias cannot be empty.";
-    public static final String PARAM_JOIN = ", ";
-    public static final Function LOWERED_PARAMETER = p ->
-            String.format("lower(%s)", p.getPlaceholder());
-
-    public static final Function GENERATE_HQL_COLUMN_NO_ALIAS = (predicate) -> {
-        return predicate.getFieldPath();
-    };
-
-    public static final Function GENERATE_HQL_COLUMN_WITH_ALIAS = (predicate) -> {
-        return predicate.getAlias() + "." + predicate.getField();
-    };
-
-    @Override
-    public String apply(FilterPredicate filterPredicate) {
-        return apply(filterPredicate, GENERATE_HQL_COLUMN_NO_ALIAS);
-    }
-
-    /**
-     * Transforms a filter predicate into a HQL query fragment.
-     * @param filterPredicate The predicate to transform.
-     * @param columnGenerator Function which supplies a HQL fragment which represents the column in the predicate.
-     * @return The hql query fragment.
-     */
-    protected String apply(FilterPredicate filterPredicate, Function columnGenerator) {
-        String fieldPath = columnGenerator.apply(filterPredicate);
-        fieldPath = fieldPath.replaceAll("\\.this", "");
-
-        List params = filterPredicate.getParameters();
-        String firstParam = params.size() > 0 ? params.get(0).getPlaceholder() : null;
-        switch (filterPredicate.getOperator()) {
-            case IN:
-                Preconditions.checkState(!filterPredicate.getValues().isEmpty());
-                return String.format("%s IN (%s)", fieldPath, params.stream()
-                        .map(FilterParameter::getPlaceholder)
-                        .collect(Collectors.joining(PARAM_JOIN)));
-
-            case IN_INSENSITIVE:
-                Preconditions.checkState(!filterPredicate.getValues().isEmpty());
-                return String.format("lower(%s) IN (%s)", fieldPath, params.stream()
-                        .map(LOWERED_PARAMETER)
-                        .collect(Collectors.joining(PARAM_JOIN)));
-
-            case NOT:
-                Preconditions.checkState(!filterPredicate.getValues().isEmpty());
-                return String.format("%s NOT IN (%s)", fieldPath, params.stream()
-                        .map(FilterParameter::getPlaceholder)
-                        .collect(Collectors.joining(PARAM_JOIN)));
-
-            case NOT_INSENSITIVE:
-                Preconditions.checkState(!filterPredicate.getValues().isEmpty());
-                return String.format("lower(%s) NOT IN (%s)", fieldPath, params.stream()
-                        .map(LOWERED_PARAMETER)
-                        .collect(Collectors.joining(PARAM_JOIN)));
-
-            case PREFIX:
-                return String.format("%s LIKE CONCAT(%s, '%%')", fieldPath, firstParam);
-
-            case PREFIX_CASE_INSENSITIVE:
-                assertValidValues(fieldPath, firstParam);
-                return String.format("lower(%s) LIKE CONCAT(lower(%s), '%%')", fieldPath, firstParam);
-
-            case POSTFIX:
-                return String.format("%s LIKE CONCAT('%%', %s)", fieldPath, firstParam);
-
-            case POSTFIX_CASE_INSENSITIVE:
-                assertValidValues(fieldPath, firstParam);
-                return String.format("lower(%s) LIKE CONCAT('%%', lower(%s))", fieldPath, firstParam);
-
-            case INFIX:
-                return String.format("%s LIKE CONCAT('%%', %s, '%%')", fieldPath, firstParam);
-
-            case INFIX_CASE_INSENSITIVE:
-                assertValidValues(fieldPath, firstParam);
-                return String.format("lower(%s) LIKE CONCAT('%%', lower(%s), '%%')", fieldPath, firstParam);
-
-            case LT:
-                return String.format("%s < %s", fieldPath, params.size() == 1 ? firstParam : leastClause(params));
-
-            case LE:
-                return String.format("%s <= %s", fieldPath, params.size() == 1 ? firstParam : leastClause(params));
-
-            case GT:
-                return String.format("%s > %s", fieldPath, params.size() == 1 ? firstParam : greatestClause(params));
-
-            case GE:
-                return String.format("%s >= %s", fieldPath, params.size() == 1 ? firstParam : greatestClause(params));
-
-            // Not parametric checks
-            case ISNULL:
-                return String.format("%s IS NULL", fieldPath);
-
-            case NOTNULL:
-                return String.format("%s IS NOT NULL", fieldPath);
-
-            case TRUE:
-                return "(1 = 1)";
-
-            case FALSE:
-                return "(1 = 0)";
-
-            default:
-                throw new InvalidPredicateException("Operator not implemented: " + filterPredicate.getOperator());
-        }
-    }
-
-    private String greatestClause(List params) {
-        return String.format("greatest(%s)", params.stream()
-                .map(FilterParameter::getPlaceholder)
-                .collect(Collectors.joining(PARAM_JOIN)));
-    }
-
-    private String leastClause(List params) {
-        return String.format("least(%s)", params.stream()
-                .map(FilterParameter::getPlaceholder)
-                .collect(Collectors.joining(PARAM_JOIN)));
-    }
-
-    private void assertValidValues(String fieldPath, String alias) {
-        if (Strings.isNullOrEmpty(fieldPath)) {
-            throw new InvalidValueException(FILTER_PATH_NOT_NULL);
-        }
-        if (Strings.isNullOrEmpty(alias)) {
-            throw new IllegalStateException(FILTER_ALIAS_NOT_NULL);
-        }
-    }
-
-    /**
-     * Translates the filterExpression to a JPQL filter fragment.
-     * @param filterExpression The filterExpression to translate
-     * @param prefixWithAlias If true, use the default alias provider to append the predicates with an alias.
-     *                        Otherwise, don't append aliases.
-     * @return A JPQL filter fragment.
-     */
-    public String apply(FilterExpression filterExpression, boolean prefixWithAlias) {
-        Function columnGenerator = GENERATE_HQL_COLUMN_NO_ALIAS;
-        if (prefixWithAlias) {
-            columnGenerator = GENERATE_HQL_COLUMN_WITH_ALIAS;
-        }
-
-        return apply(filterExpression, columnGenerator);
-    }
-
-    /**
-     * Translates the filterExpression to a JPQL filter fragment.
-     * @param filterExpression The filterExpression to translate
-     * @param columnGenerator Generates a HQL fragment that represents a column in the predicate
-     * @return A JPQL filter fragment.
-     */
-    public String apply(FilterExpression filterExpression, Function columnGenerator) {
-        HQLQueryVisitor visitor = new HQLQueryVisitor(columnGenerator);
-        return filterExpression.accept(visitor);
-    }
-
-    /**
-     * Filter expression visitor which builds an HQL query.
-     */
-    public class HQLQueryVisitor implements FilterExpressionVisitor {
-        public static final String TWO_NON_FILTERING_EXPRESSIONS =
-                "Cannot build a filter from two non-filtering expressions";
-        private Function columnGenerator;
-
-        public HQLQueryVisitor(Function columnGenerator) {
-            this.columnGenerator = columnGenerator;
-        }
-
-        @Override
-        public String visitPredicate(FilterPredicate filterPredicate) {
-            return apply(filterPredicate, columnGenerator);
-        }
-
-        @Override
-        public String visitAndExpression(AndFilterExpression expression) {
-            FilterExpression left = expression.getLeft();
-            FilterExpression right = expression.getRight();
-            return "(" + left.accept(this) + " AND " + right.accept(this) + ")";
-        }
-
-        @Override
-        public String visitOrExpression(OrFilterExpression expression) {
-            FilterExpression left = expression.getLeft();
-            FilterExpression right = expression.getRight();
-            return "(" + left.accept(this) + " OR " + right.accept(this) + ")";
-        }
-
-        @Override
-        public String visitNotExpression(NotFilterExpression expression) {
-            String negated = expression.getNegated().accept(this);
-            return "NOT (" + negated + ")";
-        }
-    }
-}
diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java
index 169f3ab43b..9978721da6 100644
--- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java
+++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionFetchQueryBuilder.java
@@ -7,7 +7,7 @@
 
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.core.filter.FilterPredicate;
-import com.yahoo.elide.core.filter.HQLFilterOperation;
+import com.yahoo.elide.core.filter.FilterTranslator;
 import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor;
 import com.yahoo.elide.core.hibernate.Query;
 import com.yahoo.elide.core.hibernate.Session;
@@ -44,7 +44,7 @@ public Query build() {
             Collection predicates = filterExpression.get().accept(extractor);
 
             //Build the WHERE clause
-            String filterClause = WHERE + new HQLFilterOperation().apply(filterExpression.get(), USE_ALIAS);
+            String filterClause = WHERE + new FilterTranslator().apply(filterExpression.get(), USE_ALIAS);
 
             //Build the JOIN clause
             String joinClause =  getJoinClauseFromFilters(filterExpression.get())
diff --git a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java
index 5758e6449c..3bca1ac0a5 100644
--- a/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java
+++ b/elide-datastore/elide-datastore-hibernate/src/main/java/com/yahoo/elide/core/hibernate/hql/RootCollectionPageTotalsQueryBuilder.java
@@ -7,7 +7,7 @@
 
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.core.filter.FilterPredicate;
-import com.yahoo.elide.core.filter.HQLFilterOperation;
+import com.yahoo.elide.core.filter.FilterTranslator;
 import com.yahoo.elide.core.filter.expression.PredicateExtractionVisitor;
 import com.yahoo.elide.core.hibernate.Query;
 import com.yahoo.elide.core.hibernate.Session;
@@ -67,7 +67,7 @@ public Query build() {
             predicates = filterExpression.get().accept(extractor);
 
             //Build the WHERE clause
-            filterClause = WHERE + new HQLFilterOperation().apply(filterExpression.get(), USE_ALIAS);
+            filterClause = WHERE + new FilterTranslator().apply(filterExpression.get(), USE_ALIAS);
 
             //Build the JOIN clause
             joinClause =  getJoinClauseFromFilters(filterExpression.get());

From 70729a376151d6ad581119921ffd19c429ae07b6 Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Fri, 27 Sep 2019 12:54:53 -0500
Subject: [PATCH 42/47] fix dictionary

---
 .../yahoo/elide/core/EntityDictionary.java    | 102 +++++++++---------
 1 file changed, 49 insertions(+), 53 deletions(-)

diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
index 00d0b93090..d7b805cb46 100644
--- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
+++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java
@@ -169,12 +169,10 @@ public static Method findMethod(Class entityClass, String name, Class... p
     }
 
     protected EntityBinding getEntityBinding(Class entityClass) {
-        return entityBindings.computeIfAbsent(entityClass, cls -> {
-            if (isMappedInterface(cls)) {
-                return EMPTY_BINDING;
-            }
-            return entityBindings.getOrDefault(lookupEntityClass(cls), EMPTY_BINDING);
-        });
+        if (isMappedInterface(entityClass)) {
+            return EMPTY_BINDING;
+        }
+        return entityBindings.getOrDefault(lookupEntityClass(entityClass), EMPTY_BINDING);
     }
 
     public boolean isMappedInterface(Class interfaceClass) {
@@ -616,7 +614,6 @@ public RelationshipType getRelationshipType(Object entity, String relation) {
      * 
      * {@code
      * public class Address {
-     *
      *     {@literal @}Id
      *     private Long id
      *
@@ -638,7 +635,6 @@ public RelationshipType getRelationshipType(Object entity, String relation) {
      * 
      * {@code
      * public class Address {
-     *
      *     {@literal @}Id
      *     private Long surrogateKey
      *
@@ -661,7 +657,7 @@ public RelationshipType getRelationshipType(Object entity, String relation) {
      * calling this method has undefined behavior
      *
      * @param entityClass Entity class
-     * @param identifier  Field to lookup type
+     * @param identifier  Identifier/Field to lookup type
      * @return Type of entity
      */
     public Class getType(Class entityClass, String identifier) {
@@ -699,7 +695,7 @@ public Class getParameterizedType(Class entityClass, String identifier) {
      * Retrieve the parameterized type for the given field.
      *
      * @param entityClass the entity class
-     * @param identifier  the identifier
+     * @param identifier  the identifier/field name
      * @param paramIndex  the index of the parameterization
      * @return Entity type for field otherwise null.
      */
@@ -815,7 +811,6 @@ public void bindEntity(Class cls) {
         if (entityBindings.getOrDefault(lookupEntityClass(cls), EMPTY_BINDING) != EMPTY_BINDING) {
             return;
         }
-        entityBindings.remove(lookupEntityClass(cls));
 
         Annotation annotation = getFirstAnnotation(cls, Arrays.asList(Include.class, Exclude.class));
         Include include = annotation instanceof Include ? (Include) annotation : null;
@@ -1214,43 +1209,13 @@ public  List walkEntityGraph(Set> entities,  Function, T
 
     /**
      * Returns whether or not a class is already bound.
-     * @param cls
+     * @param cls The class to verify.
      * @return true if the class is bound.  False otherwise.
      */
     public boolean hasBinding(Class cls) {
         return bindJsonApiToEntity.contains(cls);
     }
 
-    /**
-     * Returns whether or not a specified annotation is present on an entity field or its corresponding method.
-     *
-     * @param fieldName  The entity field
-     * @param annotationClass  The provided annotation class
-     *
-     * @param   The type of the {@code annotationClass}
-     *
-     * @return {@code true} if the field is annotated by the {@code annotationClass}
-     */
-    public  boolean attributeOrRelationAnnotationExists(
-            Class cls,
-            String fieldName,
-            Class annotationClass
-    ) {
-        return getAttributeOrRelationAnnotation(cls, annotationClass, fieldName) != null;
-    }
-
-    /**
-     * Returns whether or not a specified field exists in an entity.
-     *
-     * @param cls  The entity
-     * @param fieldName  The provided field to check
-     *
-     * @return {@code true} if the field exists in the entity
-     */
-    public boolean isValidField(Class cls, String fieldName) {
-        return getAllFields(cls).contains(fieldName);
-    }
-
     /**
      * Invoke the get[fieldName] method on the target object OR get the field with the corresponding name.
      * @param target the object to get
@@ -1287,28 +1252,29 @@ public void setValue(Object target, String fieldName, Object value) {
         Class targetClass = target.getClass();
         String targetType = getJsonAliasFor(targetClass);
 
+        String fieldAlias = fieldName;
         try {
             Class fieldClass = getType(targetClass, fieldName);
             String realName = getNameFromAlias(target, fieldName);
-            fieldName = (realName != null) ? realName : fieldName;
-            String setMethod = "set" + StringUtils.capitalize(fieldName);
+            fieldAlias = (realName != null) ? realName : fieldName;
+            String setMethod = "set" + StringUtils.capitalize(fieldAlias);
             Method method = EntityDictionary.findMethod(targetClass, setMethod, fieldClass);
-            method.invoke(target, coerce(target, value, fieldName, fieldClass));
+            method.invoke(target, coerce(target, value, fieldAlias, fieldClass));
         } catch (IllegalAccessException e) {
-            throw new InvalidAttributeException(fieldName, targetType, e);
+            throw new InvalidAttributeException(fieldAlias, targetType, e);
         } catch (InvocationTargetException e) {
             throw handleInvocationTargetException(e);
         } catch (IllegalArgumentException | NoSuchMethodException noMethod) {
-            AccessibleObject accessor = getAccessibleObject(target, fieldName);
+            AccessibleObject accessor = getAccessibleObject(target, fieldAlias);
             if (accessor != null && accessor instanceof Field) {
                 Field field = (Field) accessor;
                 try {
-                    field.set(target, coerce(target, value, fieldName, field.getType()));
+                    field.set(target, coerce(target, value, fieldAlias, field.getType()));
                 } catch (IllegalAccessException noField) {
-                    throw new InvalidAttributeException(fieldName, targetType, noField);
+                    throw new InvalidAttributeException(fieldAlias, targetType, noField);
                 }
             } else {
-                throw new InvalidAttributeException(fieldName, targetType);
+                throw new InvalidAttributeException(fieldAlias, targetType);
             }
         }
     }
@@ -1336,13 +1302,13 @@ private static RuntimeException handleInvocationTargetException(InvocationTarget
      * @param fieldClass expected class type
      * @return coerced value
      */
-    Object coerce(Object target, Object value, String fieldName, Class fieldClass) {
+    public Object coerce(Object target, Object value, String fieldName, Class fieldClass) {
         if (fieldClass != null && Collection.class.isAssignableFrom(fieldClass) && value instanceof Collection) {
             return coerceCollection(target, (Collection) value, fieldName, fieldClass);
         }
 
         if (fieldClass != null && Map.class.isAssignableFrom(fieldClass) && value instanceof Map) {
-            return coerceMap(target, (Map) value, fieldName, fieldClass);
+            return coerceMap(target, (Map) value, fieldName);
         }
 
         return CoerceUtil.coerce(value, fieldClass);
@@ -1377,7 +1343,7 @@ private Collection coerceCollection(Object target, Collection values, String
         return list;
     }
 
-    private Map coerceMap(Object target, Map values, String fieldName, Class fieldClass) {
+    private Map coerceMap(Object target, Map values, String fieldName) {
         Class keyType = getParameterizedType(target, fieldName, 0);
         Class valueType = getParameterizedType(target, fieldName, 1);
 
@@ -1394,6 +1360,36 @@ private Map coerceMap(Object target, Map values, String fieldName, Class  The type of the {@code annotationClass}
+     *
+     * @return {@code true} if the field is annotated by the {@code annotationClass}
+     */
+    public  boolean attributeOrRelationAnnotationExists(
+            Class cls,
+            String fieldName,
+            Class annotationClass
+    ) {
+        return getAttributeOrRelationAnnotation(cls, annotationClass, fieldName) != null;
+    }
+
+    /**
+     * Returns whether or not a specified field exists in an entity.
+     *
+     * @param cls  The entity
+     * @param fieldName  The provided field to check
+     *
+     * @return {@code true} if the field exists in the entity
+     */
+    public boolean isValidField(Class cls, String fieldName) {
+        return getAllFields(cls).contains(fieldName);
+    }
+
     private boolean isValidParameterizedMap(Map values, Class keyType, Class valueType) {
         for (Map.Entry entry : values.entrySet()) {
             Object key = entry.getKey();

From 24b774832819cef8bc6d1e0b95656fd1edea4aef Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Fri, 27 Sep 2019 14:33:34 -0500
Subject: [PATCH 43/47] fix SqlEngineTest

---
 .../elide-datastore-aggregation/pom.xml       |  26 +-
 .../aggregation/engine/SQLEntityHydrator.java |  14 +-
 .../aggregation/engine/StitchList.java        |  21 +-
 .../datastores/aggregation/SchemaTest.java    |  33 ++-
 .../aggregation/dimension/DimensionTest.java  |  32 +-
 .../dimension/EntityDimensionTest.java        |  51 ++--
 .../engine/SQLQueryEngineTest.java            | 280 +++++++++++++-----
 .../filter/visitor/FilterConstraintsTest.java |  49 +--
 .../SplitFilterExpressionVisitorTest.java     | 133 +++++----
 .../metric/AggregatedMetricTest.java          |  35 +--
 .../src/test/resources/country.csv            |   2 +-
 .../src/test/resources/player_stats.csv       |   1 +
 12 files changed, 421 insertions(+), 256 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/pom.xml b/elide-datastore/elide-datastore-aggregation/pom.xml
index c1a471088e..e8ebfadbb6 100644
--- a/elide-datastore/elide-datastore-aggregation/pom.xml
+++ b/elide-datastore/elide-datastore-aggregation/pom.xml
@@ -14,7 +14,7 @@
     
         com.yahoo.elide
         elide-datastore-parent-pom
-        4.4.5-SNAPSHOT
+        4.5.2-SNAPSHOT
     
 
     
@@ -38,6 +38,10 @@
         HEAD
     
 
+    
+        5.4.1
+    
+
     
         
             com.yahoo.elide
@@ -71,9 +75,25 @@
         
 
         
+        
+        
+            org.junit.jupiter
+            junit-jupiter-api
+            ${junit.version}
+            test
+        
+
+        
+            org.junit.jupiter
+            junit-jupiter-params
+            ${junit.version}
+            test
+        
+
         
-            org.testng
-            testng
+            org.junit.jupiter
+            junit-jupiter-engine
+            ${junit.version}
             test
         
 
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
index 58e3f7ed97..3717c36616 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
@@ -7,7 +7,6 @@
 
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.datastores.aggregation.Query;
-
 import lombok.AccessLevel;
 import lombok.Getter;
 
@@ -17,7 +16,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
 import javax.persistence.EntityManager;
 
@@ -70,10 +68,14 @@ protected Map getRelationshipValues(
                 .setParameter("idList", uniqueIds)
                 .getResultList();
 
-        // returns a mapping as [joinId(0) -> loaded(0), joinId(1) -> loaded(1), ...]
-        return IntStream.range(0, loaded.size())
-                .boxed()
-                .map(i -> new AbstractMap.SimpleImmutableEntry<>(uniqueIds.get(i), loaded.get(i)))
+        return loaded.stream()
+                .map(obj -> new AbstractMap.SimpleImmutableEntry<>((Object) getEntityDictionary().getId(obj), obj))
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        // returns a mapping as [joinId(0) -> loaded(0), joinId(1) -> loaded(1), ...]
+//        return IntStream.range(0, loaded.size())
+//                .boxed()
+//                .map(i -> new AbstractMap.SimpleImmutableEntry<>(uniqueIds.get(i), loaded.get(i)))
+//                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
index 6d0e4464be..4c40d1138b 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
@@ -25,17 +25,6 @@
  * {@link StitchList} should not be subclassed.
  */
 public final class StitchList {
-
-    /**
-     * A representation of an TODO item in a {@link StitchList}.
-     */
-    @Data
-    public static class Todo {
-        private final Object entityInstance;
-        private final String relationshipName;
-        private final Object foreignKey;
-    }
-
     /**
      * Maps an relationship entity class to a map of object ID to object instance.
      * 

@@ -53,6 +42,16 @@ public static class Todo {
     @Getter(AccessLevel.PRIVATE)
     private final EntityDictionary entityDictionary;
 
+    /**
+     * A representation of an TODO item in a {@link StitchList}.
+     */
+    @Data
+    public static class Todo {
+        private final Object entityInstance;
+        private final String relationshipName;
+        private final Object foreignKey;
+    }
+
     /**
      * Constructor.
      *
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
index 6c921411bb..f51a1a4399 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
@@ -5,6 +5,10 @@
  */
 package com.yahoo.elide.datastores.aggregation;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize;
 import com.yahoo.elide.datastores.aggregation.example.Country;
@@ -14,20 +18,18 @@
 import com.yahoo.elide.datastores.aggregation.metric.Max;
 
 import com.yahoo.elide.datastores.aggregation.schema.Schema;
-
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 
 import java.util.Collections;
 
 public class SchemaTest {
 
-    private EntityDictionary entityDictionary;
-    private Schema playerStatsSchema;
+    private static EntityDictionary entityDictionary;
+    private static Schema playerStatsSchema;
 
-    @BeforeMethod
-    public void setupEntityDictionary() {
+    @BeforeAll
+    public static void setupEntityDictionary() {
         entityDictionary = new EntityDictionary(Collections.emptyMap());
         entityDictionary.bindEntity(Country.class);
         entityDictionary.bindEntity(VideoGame.class);
@@ -37,21 +39,22 @@ public void setupEntityDictionary() {
         playerStatsSchema = new Schema(PlayerStats.class, entityDictionary);
     }
 
-    @Test void testMetricCheck() {
-        Assert.assertTrue(playerStatsSchema.isMetricField("highScore"));
-        Assert.assertFalse(playerStatsSchema.isMetricField("country"));
+    @Test
+    void testMetricCheck() {
+        assertTrue(playerStatsSchema.isMetricField("highScore"));
+        assertFalse(playerStatsSchema.isMetricField("country"));
     }
 
     @Test
     public void testGetDimension() {
-        Assert.assertEquals(playerStatsSchema.getDimension("country").getCardinality(), CardinalitySize.SMALL);
+        assertEquals(CardinalitySize.SMALL, playerStatsSchema.getDimension("country").getCardinality());
     }
 
     @Test
     public void testGetMetric() {
-        Assert.assertEquals(
-                playerStatsSchema.getMetric("highScore").getMetricExpression(Max.class),
-                "MAX(com_yahoo_elide_datastores_aggregation_example_PlayerStats.highScore)"
+        assertEquals(
+                "MAX(com_yahoo_elide_datastores_aggregation_example_PlayerStats.highScore)",
+                playerStatsSchema.getMetric("highScore").getMetricExpression(Max.class)
         );
     }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/DimensionTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/DimensionTest.java
index bbafa9db59..46e187c18f 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/DimensionTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/DimensionTest.java
@@ -5,15 +5,15 @@
  */
 package com.yahoo.elide.datastores.aggregation.dimension;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.mockito.Mockito.mock;
 
 import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize;
 import com.yahoo.elide.datastores.aggregation.example.Country;
 import com.yahoo.elide.datastores.aggregation.schema.Schema;
 import com.yahoo.elide.datastores.aggregation.time.TimeGrain;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.Test;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -55,16 +55,16 @@ public class DimensionTest {
 
     @Test
     public void testDimensionAsCollectionElement() {
-        Assert.assertEquals(ENTITY_DIMENSION, ENTITY_DIMENSION);
-        Assert.assertEquals(DEGENERATE_DIMENSION, DEGENERATE_DIMENSION);
-        Assert.assertNotEquals(ENTITY_DIMENSION, DEGENERATE_DIMENSION);
-        Assert.assertNotEquals(ENTITY_DIMENSION.hashCode(), DEGENERATE_DIMENSION.hashCode());
+        assertEquals(ENTITY_DIMENSION, ENTITY_DIMENSION);
+        assertEquals(DEGENERATE_DIMENSION, DEGENERATE_DIMENSION);
+        assertNotEquals(DEGENERATE_DIMENSION, ENTITY_DIMENSION);
+        assertNotEquals(DEGENERATE_DIMENSION.hashCode(), ENTITY_DIMENSION.hashCode());
 
         // different dimensions should be separate elements in Set
         Set dimensions = new HashSet<>();
         dimensions.add(ENTITY_DIMENSION);
 
-        Assert.assertEquals(dimensions.size(), 1);
+        assertEquals(1, dimensions.size());
 
         // a separate same object doesn't increase collection size
         Dimension sameEntityDimension = new EntityDimension(
@@ -75,35 +75,35 @@ public void testDimensionAsCollectionElement() {
                 CardinalitySize.SMALL,
                 "name"
         );
-        Assert.assertEquals(sameEntityDimension, ENTITY_DIMENSION);
+        assertEquals(ENTITY_DIMENSION, sameEntityDimension);
         dimensions.add(sameEntityDimension);
-        Assert.assertEquals(dimensions.size(), 1);
+        assertEquals(1, dimensions.size());
 
         dimensions.add(ENTITY_DIMENSION);
-        Assert.assertEquals(dimensions.size(), 1);
+        assertEquals(1, dimensions.size());
 
         dimensions.add(DEGENERATE_DIMENSION);
-        Assert.assertEquals(dimensions.size(), 2);
+        assertEquals(2, dimensions.size());
 
         dimensions.add(TIME_DIMENSION);
-        Assert.assertEquals(dimensions.size(), 3);
+        assertEquals(3, dimensions.size());
     }
 
     @Test
     public void testToString() {
         // table dimension
-        Assert.assertEquals(
+        assertEquals(
                 ENTITY_DIMENSION.toString(),
                 "EntityDimension[name='country', longName='country', description='country', dimensionType=ENTITY, dataType=Country, cardinality=SMALL, friendlyName='name']"
         );
 
         // degenerate dimension
-        Assert.assertEquals(
+        assertEquals(
                 DEGENERATE_DIMENSION.toString(),
                 "DegenerateDimension[columnType=FIELD, name='overallRating', longName='overallRating', description='overallRating', dimensionType=DEGENERATE, dataType=String, cardinality=SMALL, friendlyName='overallRating']"
         );
 
-        Assert.assertEquals(
+        assertEquals(
                 TIME_DIMENSION.toString(),
                 "TimeDimension[timeZone=Pacific Standard Time, timeGrain=DAY, columnType=TEMPORAL, name='recordedTime', longName='recordedTime', description='recordedTime', dimensionType=DEGENERATE, dataType=class java.lang.Long, cardinality=LARGE, friendlyName='recordedTime']"
         );
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimensionTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimensionTest.java
index 64b9856862..9e6942f495 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimensionTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimensionTest.java
@@ -5,6 +5,9 @@
  */
 package com.yahoo.elide.datastores.aggregation.dimension;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
 import com.yahoo.elide.annotation.Include;
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize;
@@ -12,10 +15,8 @@
 import com.yahoo.elide.datastores.aggregation.example.Country;
 import com.yahoo.elide.datastores.aggregation.example.PlayerStats;
 import com.yahoo.elide.datastores.aggregation.example.VideoGame;
-
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 
 import java.util.Collections;
 
@@ -52,10 +53,10 @@ public String getSubTitle() {
         }
     }
 
-    private EntityDictionary entityDictionary;
+    private static EntityDictionary entityDictionary;
 
-    @BeforeMethod
-    public void setupEntityDictionary() {
+    @BeforeAll
+    public static void setupEntityDictionary() {
         entityDictionary = new EntityDictionary(Collections.emptyMap());
         entityDictionary.bindEntity(PlayerStats.class);
         entityDictionary.bindEntity(Country.class);
@@ -66,44 +67,46 @@ public void setupEntityDictionary() {
     @Test
     public void testHappyPathFriendlyNameScan() {
         // 1 field with @FriendlyName
-        Assert.assertEquals(
-                EntityDimension.getFriendlyNameField(PlayerStats.class, entityDictionary),
-                "overallRating"
+        assertEquals(
+                "overallRating",
+                EntityDimension.getFriendlyNameField(PlayerStats.class, entityDictionary)
         );
 
         // no field with @FriendlyName
-        Assert.assertEquals(
-                EntityDimension.getFriendlyNameField(VideoGame.class, entityDictionary),
-                "id"
+        assertEquals(
+                "id",
+                EntityDimension.getFriendlyNameField(VideoGame.class, entityDictionary)
         );
     }
 
     /**
      * Multiple {@link FriendlyName} annotations in entity is illegal.
      */
-    @Test(expectedExceptions = IllegalStateException.class)
+    @Test
     public void testUnhappyPathFriendlyNameScan() {
-        EntityDimension.getFriendlyNameField(Book.class, entityDictionary);
+        assertThrows(
+                IllegalStateException.class,
+                () -> EntityDimension.getFriendlyNameField(Book.class, entityDictionary));
     }
 
     @Test
     public void testCardinalityScan() {
         // annotation on entity
-        Assert.assertEquals(
-                EntityDimension.getEstimatedCardinality("country", PlayerStats.class, entityDictionary),
-                CardinalitySize.SMALL
+        assertEquals(
+                CardinalitySize.SMALL,
+                EntityDimension.getEstimatedCardinality("country", PlayerStats.class, entityDictionary)
         );
 
         // annotation on field
-        Assert.assertEquals(
-                EntityDimension.getEstimatedCardinality("overallRating", PlayerStats.class, entityDictionary),
-                CardinalitySize.MEDIUM
+        assertEquals(
+                CardinalitySize.MEDIUM,
+                EntityDimension.getEstimatedCardinality("overallRating", PlayerStats.class, entityDictionary)
         );
 
         // default is used
-        Assert.assertEquals(
-                EntityDimension.getEstimatedCardinality("recordedDate", PlayerStats.class, entityDictionary),
-                EntityDimension.getDefaultCardinality()
+        assertEquals(
+                EntityDimension.getDefaultCardinality(),
+                EntityDimension.getEstimatedCardinality("recordedDate", PlayerStats.class, entityDictionary)
         );
     }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java
index c29fce753f..bc930b3a75 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/engine/SQLQueryEngineTest.java
@@ -6,6 +6,9 @@
 
 package com.yahoo.elide.datastores.aggregation.engine;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.core.filter.dialect.RSQLFilterDialect;
 import com.yahoo.elide.core.pagination.Pagination;
@@ -20,8 +23,8 @@
 import com.yahoo.elide.datastores.aggregation.example.PlayerStatsView;
 import com.yahoo.elide.datastores.aggregation.metric.Sum;
 import com.yahoo.elide.datastores.aggregation.schema.Schema;
-import org.testng.Assert;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 
 import java.sql.Timestamp;
 import java.util.HashMap;
@@ -35,15 +38,14 @@
 import javax.persistence.Persistence;
 
 public class SQLQueryEngineTest {
-
-    private EntityManagerFactory emf;
-
-    private Schema playerStatsSchema;
-    private Schema playerStatsViewSchema;
-    private EntityDictionary dictionary;
-    private RSQLFilterDialect filterParser;
-
-    public SQLQueryEngineTest() {
+    private static EntityManagerFactory emf;
+    private static Schema playerStatsSchema;
+    private static Schema playerStatsViewSchema;
+    private static EntityDictionary dictionary;
+    private static RSQLFilterDialect filterParser;
+
+    @BeforeAll
+    public static void init() {
         emf = Persistence.createEntityManagerFactory("aggregationStore");
         dictionary = new EntityDictionary(new HashMap<>());
         dictionary.bindEntity(PlayerStats.class);
@@ -56,8 +58,11 @@ public SQLQueryEngineTest() {
         playerStatsViewSchema = new SQLSchema(PlayerStatsView.class, dictionary);
     }
 
+    /**
+     * Test loading all three records from the table.
+     */
     @Test
-    public void testFullTableLoad() throws Exception {
+    public void testFullTableLoad() {
         EntityManager em = emf.createEntityManager();
         QueryEngine engine = new SQLQueryEngine(em, dictionary);
 
@@ -65,33 +70,41 @@ public void testFullTableLoad() throws Exception {
                 .schema(playerStatsSchema)
                 .metric(playerStatsSchema.getMetric("lowScore"), Sum.class)
                 .metric(playerStatsSchema.getMetric("highScore"), Sum.class)
-                .groupDimension(playerStatsSchema.getDimension("overallRating"))
                 .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate"))
                 .build();
 
         List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false)
                 .collect(Collectors.toList());
 
-        //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00
+        PlayerStats stats0 = new PlayerStats();
+        stats0.setId("0");
+        stats0.setLowScore(241);
+        stats0.setHighScore(2412);
+        stats0.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
+
         PlayerStats stats1 = new PlayerStats();
-        stats1.setId("0");
+        stats1.setId("1");
         stats1.setLowScore(72);
         stats1.setHighScore(1234);
-        stats1.setOverallRating("Good");
         stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00"));
 
         PlayerStats stats2 = new PlayerStats();
-        stats2.setId("1");
-        stats2.setLowScore(241);
-        stats2.setHighScore(2412);
-        stats2.setOverallRating("Great");
-        stats2.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
+        stats2.setId("2");
+        stats2.setLowScore(72);
+        stats2.setHighScore(1000);
+        stats2.setRecordedDate(Timestamp.valueOf("2019-07-13 00:00:00"));
 
-        Assert.assertEquals(results.size(), 2);
-        Assert.assertEquals(results.get(0), stats1);
-        Assert.assertEquals(results.get(1), stats2);
+        assertEquals(3, results.size());
+        assertEquals(stats0, results.get(0));
+        assertEquals(stats1, results.get(1));
+        assertEquals(stats2, results.get(2));
     }
 
+    /**
+     * Test group by a degenerate dimension with a filter applied.
+     *
+     * @throws Exception exception
+     */
     @Test
     public void testDegenerateDimensionFilter() throws Exception {
         EntityManager em = emf.createEntityManager();
@@ -117,10 +130,15 @@ public void testDegenerateDimensionFilter() throws Exception {
         stats1.setOverallRating("Great");
         stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
 
-        Assert.assertEquals(results.size(), 1);
-        Assert.assertEquals(results.get(0), stats1);
+        assertEquals(1, results.size());
+        assertEquals(stats1, results.get(0));
     }
 
+    /**
+     * Test filtering on a dimension attribute.
+     *
+     * @throws Exception exception
+     */
     @Test
     public void testFilterJoin() throws Exception {
         EntityManager em = emf.createEntityManager();
@@ -146,31 +164,36 @@ public void testFilterJoin() throws Exception {
         expectedCountry.setName("United States");
 
 
-        PlayerStats stats1 = new PlayerStats();
-        stats1.setId("0");
-        stats1.setLowScore(241);
-        stats1.setHighScore(2412);
-        stats1.setOverallRating("Great");
-        stats1.setCountry(expectedCountry);
-        stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
+        PlayerStats usa0 = new PlayerStats();
+        usa0.setId("0");
+        usa0.setLowScore(241);
+        usa0.setHighScore(2412);
+        usa0.setOverallRating("Great");
+        usa0.setCountry(expectedCountry);
+        usa0.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
 
-        PlayerStats stats2 = new PlayerStats();
-        stats2.setId("1");
-        stats2.setLowScore(72);
-        stats2.setHighScore(1234);
-        stats2.setOverallRating("Good");
-        stats2.setCountry(expectedCountry);
-        stats2.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00"));
+        PlayerStats usa1 = new PlayerStats();
+        usa1.setId("1");
+        usa1.setLowScore(72);
+        usa1.setHighScore(1234);
+        usa1.setOverallRating("Good");
+        usa1.setCountry(expectedCountry);
+        usa1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00"));
 
-        Assert.assertEquals(results.size(), 2);
-        Assert.assertEquals(results.get(0), stats1);
-        Assert.assertEquals(results.get(1), stats2);
+        assertEquals(2, results.size());
+        assertEquals(usa0, results.get(0));
+        assertEquals(usa1, results.get(1));
 
         // test join
         PlayerStats actualStats1 = (PlayerStats) results.get(0);
-        Assert.assertNotNull(actualStats1.getCountry());
+        assertNotNull(actualStats1.getCountry());
     }
 
+    /**
+     * Test filtering on an attribute that's not present in the query.
+     *
+     * @throws Exception exception
+     */
     @Test
     public void testSubqueryFilterJoin() throws Exception {
         EntityManager em = emf.createEntityManager();
@@ -190,10 +213,15 @@ public void testSubqueryFilterJoin() throws Exception {
         stats2.setId("0");
         stats2.setHighScore(2412);
 
-        Assert.assertEquals(results.size(), 1);
-        Assert.assertEquals(results.get(0), stats2);
+        assertEquals(1, results.size());
+        assertEquals(stats2, results.get(0));
     }
 
+    /**
+     * Test a view which filters on "stats.overallRating = 'Great'".
+     *
+     * @throws Exception exception
+     */
     @Test
     public void testSubqueryLoad() throws Exception {
         EntityManager em = emf.createEntityManager();
@@ -211,12 +239,15 @@ public void testSubqueryLoad() throws Exception {
         stats2.setId("0");
         stats2.setHighScore(2412);
 
-        Assert.assertEquals(results.size(), 1);
-        Assert.assertEquals(results.get(0), stats2);
+        assertEquals(1, results.size());
+        assertEquals(stats2, results.get(0));
     }
 
+    /**
+     * Test sorting by dimension attribute which is not present in the query.
+     */
     @Test
-    public void testSortJoin() throws Exception {
+    public void testSortJoin() {
         EntityManager em = emf.createEntityManager();
         QueryEngine engine = new SQLQueryEngine(em, dictionary);
 
@@ -234,25 +265,35 @@ public void testSortJoin() throws Exception {
         List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false)
                 .collect(Collectors.toList());
 
+        PlayerStats stats0 = new PlayerStats();
+        stats0.setId("0");
+        stats0.setLowScore(72);
+        stats0.setOverallRating("Good");
+        stats0.setRecordedDate(Timestamp.valueOf("2019-07-13 00:00:00"));
+
         PlayerStats stats1 = new PlayerStats();
-        stats1.setId("0");
+        stats1.setId("1");
         stats1.setLowScore(241);
         stats1.setOverallRating("Great");
         stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
 
         PlayerStats stats2 = new PlayerStats();
-        stats2.setId("1");
+        stats2.setId("2");
         stats2.setLowScore(72);
         stats2.setOverallRating("Good");
         stats2.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00"));
 
-        Assert.assertEquals(results.size(), 2);
-        Assert.assertEquals(results.get(0), stats1);
-        Assert.assertEquals(results.get(1), stats2);
+        assertEquals(3, results.size());
+        assertEquals(stats0, results.get(0));
+        assertEquals(stats1, results.get(1));
+        assertEquals(stats2, results.get(2));
     }
 
+    /**
+     * Test pagination.
+     */
     @Test
-    public void testPagination() throws Exception {
+    public void testPagination() {
         EntityManager em = emf.createEntityManager();
         QueryEngine engine = new SQLQueryEngine(em, dictionary);
 
@@ -278,11 +319,16 @@ public void testPagination() throws Exception {
         stats1.setOverallRating("Good");
         stats1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00"));
 
-        Assert.assertEquals(results.size(), 1, "Number of records returned does not match");
-        Assert.assertEquals(results.get(0), stats1, "Returned record does not match");
-        Assert.assertEquals(pagination.getPageTotals(), 2, "Page totals does not match");
+        assertEquals(results.size(), 1, "Number of records returned does not match");
+        assertEquals(results.get(0), stats1, "Returned record does not match");
+        assertEquals(pagination.getPageTotals(), 3, "Page totals does not match");
     }
 
+    /**
+     * Test having clause integrates with group by clause.
+     *
+     * @throws Exception exception
+     */
     @Test
     public void testHavingClause() throws Exception {
         EntityManager em = emf.createEntityManager();
@@ -291,22 +337,29 @@ public void testHavingClause() throws Exception {
         Query query = Query.builder()
                 .schema(playerStatsSchema)
                 .metric(playerStatsSchema.getMetric("highScore"), Sum.class)
-                .havingFilter(filterParser.parseFilterExpression("highScore > 300",
+                .groupDimension(playerStatsSchema.getDimension("overallRating"))
+                .havingFilter(filterParser.parseFilterExpression("highScore < 2400",
                         PlayerStats.class, false))
                 .build();
 
         List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false)
                 .collect(Collectors.toList());
 
-        //Jon Doe,1234,72,Good,840,2019-07-12 00:00:00
+        // Only "Good" rating would have total high score less than 2400
         PlayerStats stats1 = new PlayerStats();
         stats1.setId("0");
-        stats1.setHighScore(3646);
+        stats1.setOverallRating("Good");
+        stats1.setHighScore(2234);
 
-        Assert.assertEquals(results.size(), 1);
-        Assert.assertEquals(results.get(0), stats1);
+        assertEquals(1, results.size());
+        assertEquals(stats1, results.get(0));
     }
 
+    /**
+     * Test group by, having, dimension, metric at the same time.
+     *
+     * @throws Exception exception
+     */
     @Test
     public void testTheEverythingQuery() throws Exception {
         EntityManager em = emf.createEntityManager();
@@ -334,13 +387,15 @@ public void testTheEverythingQuery() throws Exception {
         stats2.setHighScore(2412);
         stats2.setCountryName("United States");
 
-
-        Assert.assertEquals(results.size(), 1);
-        Assert.assertEquals(results.get(0), stats2);
+        assertEquals(1, results.size());
+        assertEquals(stats2, results.get(0));
     }
 
+    /**
+     * Test sorting by two different columns-one metric and one dimension.
+     */
     @Test
-    public void testSortByMultipleColumns() throws Exception {
+    public void testSortByMultipleColumns() {
         EntityManager em = emf.createEntityManager();
         QueryEngine engine = new SQLQueryEngine(em, dictionary);
 
@@ -359,20 +414,95 @@ public void testSortByMultipleColumns() throws Exception {
         List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false)
                 .collect(Collectors.toList());
 
+        PlayerStats stats0 = new PlayerStats();
+        stats0.setId("0");
+        stats0.setLowScore(241);
+        stats0.setOverallRating("Great");
+        stats0.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
+
         PlayerStats stats1 = new PlayerStats();
-        stats1.setId("0");
-        stats1.setLowScore(241);
-        stats1.setOverallRating("Great");
-        stats1.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
+        stats1.setId("1");
+        stats1.setLowScore(72);
+        stats1.setOverallRating("Good");
+        stats1.setRecordedDate(Timestamp.valueOf("2019-07-13 00:00:00"));
 
         PlayerStats stats2 = new PlayerStats();
-        stats2.setId("1");
+        stats2.setId("2");
         stats2.setLowScore(72);
         stats2.setOverallRating("Good");
         stats2.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00"));
 
-        Assert.assertEquals(results.size(), 2);
-        Assert.assertEquals(results.get(0), stats1);
-        Assert.assertEquals(results.get(1), stats2);
+        assertEquals(3, results.size());
+        assertEquals(stats0, results.get(0));
+        assertEquals(stats1, results.get(1));
+        assertEquals(stats2, results.get(2));
+    }
+
+    /**
+     * Test hydrating multiple relationship values. Make sure the objects are constructed correctly.
+     */
+    @Test
+    public void testRelationshipHydration() {
+        EntityManager em = emf.createEntityManager();
+        QueryEngine engine = new SQLQueryEngine(em, dictionary);
+
+        Map sortMap = new TreeMap<>();
+        sortMap.put("country.name", Sorting.SortOrder.desc);
+
+        Query query = Query.builder()
+                .schema(playerStatsSchema)
+                .metric(playerStatsSchema.getMetric("lowScore"), Sum.class)
+                .metric(playerStatsSchema.getMetric("highScore"), Sum.class)
+                .groupDimension(playerStatsSchema.getDimension("overallRating"))
+                .groupDimension(playerStatsSchema.getDimension("country"))
+                .timeDimension((TimeDimension) playerStatsSchema.getDimension("recordedDate"))
+                .sorting(new Sorting(sortMap))
+                .build();
+
+        List results = StreamSupport.stream(engine.executeQuery(query).spliterator(), false)
+                .collect(Collectors.toList());
+
+        Country usa = new Country();
+        usa.setId("840");
+        usa.setIsoCode("USA");
+        usa.setName("United States");
+
+        Country hk = new Country();
+        hk.setId("344");
+        hk.setIsoCode("HKG");
+        hk.setName("Hong Kong");
+
+        PlayerStats usa0 = new PlayerStats();
+        usa0.setId("0");
+        usa0.setLowScore(241);
+        usa0.setHighScore(2412);
+        usa0.setOverallRating("Great");
+        usa0.setCountry(usa);
+        usa0.setRecordedDate(Timestamp.valueOf("2019-07-11 00:00:00"));
+
+        PlayerStats usa1 = new PlayerStats();
+        usa1.setId("1");
+        usa1.setLowScore(72);
+        usa1.setHighScore(1234);
+        usa1.setOverallRating("Good");
+        usa1.setCountry(usa);
+        usa1.setRecordedDate(Timestamp.valueOf("2019-07-12 00:00:00"));
+
+        PlayerStats hk2 = new PlayerStats();
+        hk2.setId("2");
+        hk2.setLowScore(72);
+        hk2.setHighScore(1000);
+        hk2.setOverallRating("Good");
+        hk2.setCountry(hk);
+        hk2.setRecordedDate(Timestamp.valueOf("2019-07-13 00:00:00"));
+
+        assertEquals(3, results.size());
+        assertEquals(usa0, results.get(0));
+        assertEquals(usa1, results.get(1));
+        assertEquals(hk2, results.get(2));
+
+        // test join
+        PlayerStats actualStats1 = (PlayerStats) results.get(0);
+        assertNotNull(actualStats1.getCountry());
     }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/filter/visitor/FilterConstraintsTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/filter/visitor/FilterConstraintsTest.java
index 89ef468efd..b0195b60ac 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/filter/visitor/FilterConstraintsTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/filter/visitor/FilterConstraintsTest.java
@@ -5,13 +5,16 @@
  */
 package com.yahoo.elide.datastores.aggregation.filter.visitor;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 import com.yahoo.elide.core.Path;
 import com.yahoo.elide.core.filter.FilterPredicate;
 import com.yahoo.elide.core.filter.Operator;
 import com.yahoo.elide.datastores.aggregation.example.PlayerStats;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.Test;
 
 import java.util.Collections;
 
@@ -30,37 +33,37 @@ public class FilterConstraintsTest {
 
     @Test
     public void testPureHaving() {
-        Assert.assertTrue(FilterConstraints.pureHaving(HAVING_PREDICATE).isPureHaving());
-        Assert.assertFalse(FilterConstraints.pureHaving(HAVING_PREDICATE).isPureWhere());
-        Assert.assertEquals(
-                FilterConstraints.pureHaving(HAVING_PREDICATE).getHavingExpression().toString(),
-                "playerStats.highScore GT [99]"
+        assertTrue(FilterConstraints.pureHaving(HAVING_PREDICATE).isPureHaving());
+        assertFalse(FilterConstraints.pureHaving(HAVING_PREDICATE).isPureWhere());
+        assertEquals(
+                "playerStats.highScore GT [99]",
+                FilterConstraints.pureHaving(HAVING_PREDICATE).getHavingExpression().toString()
         );
-        Assert.assertNull(FilterConstraints.pureHaving(HAVING_PREDICATE).getWhereExpression());
+        assertNull(FilterConstraints.pureHaving(HAVING_PREDICATE).getWhereExpression());
     }
 
     @Test
     public void testPureWhere() {
-        Assert.assertTrue(FilterConstraints.pureWhere(WHERE_PREDICATE).isPureWhere());
-        Assert.assertFalse(FilterConstraints.pureWhere(WHERE_PREDICATE).isPureHaving());
-        Assert.assertEquals(
-                FilterConstraints.pureWhere(WHERE_PREDICATE).getWhereExpression().toString(),
-                "playerStats.id IN [foo]"
+        assertTrue(FilterConstraints.pureWhere(WHERE_PREDICATE).isPureWhere());
+        assertFalse(FilterConstraints.pureWhere(WHERE_PREDICATE).isPureHaving());
+        assertEquals(
+                "playerStats.id IN [foo]",
+                FilterConstraints.pureWhere(WHERE_PREDICATE).getWhereExpression().toString()
         );
-        Assert.assertNull(FilterConstraints.pureWhere(WHERE_PREDICATE).getHavingExpression());
+        assertNull(FilterConstraints.pureWhere(WHERE_PREDICATE).getHavingExpression());
     }
 
     @Test
     public void testWithWhereAndHaving() {
-        Assert.assertFalse(FilterConstraints.withWhereAndHaving(WHERE_PREDICATE, HAVING_PREDICATE).isPureWhere());
-        Assert.assertFalse(FilterConstraints.withWhereAndHaving(WHERE_PREDICATE, HAVING_PREDICATE).isPureHaving());
-        Assert.assertEquals(
-                FilterConstraints.withWhereAndHaving(WHERE_PREDICATE, HAVING_PREDICATE).getWhereExpression().toString(),
-                "playerStats.id IN [foo]"
+        assertFalse(FilterConstraints.withWhereAndHaving(WHERE_PREDICATE, HAVING_PREDICATE).isPureWhere());
+        assertFalse(FilterConstraints.withWhereAndHaving(WHERE_PREDICATE, HAVING_PREDICATE).isPureHaving());
+        assertEquals(
+                "playerStats.id IN [foo]",
+                FilterConstraints.withWhereAndHaving(WHERE_PREDICATE, HAVING_PREDICATE).getWhereExpression().toString()
         );
-        Assert.assertEquals(
-                FilterConstraints.withWhereAndHaving(WHERE_PREDICATE, HAVING_PREDICATE).getHavingExpression().toString(),
-                "playerStats.highScore GT [99]"
+        assertEquals(
+                "playerStats.highScore GT [99]",
+                FilterConstraints.withWhereAndHaving(WHERE_PREDICATE, HAVING_PREDICATE).getHavingExpression().toString()
         );
     }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/filter/visitor/SplitFilterExpressionVisitorTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/filter/visitor/SplitFilterExpressionVisitorTest.java
index 441437b24b..834d1ec16e 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/filter/visitor/SplitFilterExpressionVisitorTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/filter/visitor/SplitFilterExpressionVisitorTest.java
@@ -5,6 +5,11 @@
  */
 package com.yahoo.elide.datastores.aggregation.filter.visitor;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
 import com.yahoo.elide.core.EntityDictionary;
 import com.yahoo.elide.core.Path;
 import com.yahoo.elide.core.filter.FilterPredicate;
@@ -17,10 +22,8 @@
 import com.yahoo.elide.datastores.aggregation.example.Player;
 import com.yahoo.elide.datastores.aggregation.example.PlayerStats;
 import com.yahoo.elide.datastores.aggregation.schema.Schema;
-
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 
 import java.util.Collections;
 
@@ -37,12 +40,12 @@ public class SplitFilterExpressionVisitorTest {
             Collections.singletonList(99)
     );
 
-    private EntityDictionary entityDictionary;
-    private Schema schema;
-    private FilterExpressionVisitor splitFilterExpressionVisitor;
+    private static EntityDictionary entityDictionary;
+    private static Schema schema;
+    private static FilterExpressionVisitor splitFilterExpressionVisitor;
 
-    @BeforeMethod
-    public void setupEntityDictionary() {
+    @BeforeAll
+    public static void setupEntityDictionary() {
         entityDictionary = new EntityDictionary(Collections.emptyMap());
         entityDictionary.bindEntity(PlayerStats.class);
         entityDictionary.bindEntity(Country.class);
@@ -54,78 +57,78 @@ public void setupEntityDictionary() {
     @Test
     public void testVisitPredicate() {
         // predicate should be a WHERE
-        Assert.assertTrue(splitFilterExpressionVisitor.visitPredicate(WHERE_PREDICATE).isPureWhere());
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitPredicate(WHERE_PREDICATE).getWhereExpression().toString(),
-                "playerStats.id IN [foo]"
+        assertTrue(splitFilterExpressionVisitor.visitPredicate(WHERE_PREDICATE).isPureWhere());
+        assertEquals(
+                "playerStats.id IN [foo]",
+                splitFilterExpressionVisitor.visitPredicate(WHERE_PREDICATE).getWhereExpression().toString()
         );
-        Assert.assertFalse(splitFilterExpressionVisitor.visitPredicate(WHERE_PREDICATE).isPureHaving());
-        Assert.assertNull(splitFilterExpressionVisitor.visitPredicate(WHERE_PREDICATE).getHavingExpression());
+        assertFalse(splitFilterExpressionVisitor.visitPredicate(WHERE_PREDICATE).isPureHaving());
+        assertNull(splitFilterExpressionVisitor.visitPredicate(WHERE_PREDICATE).getHavingExpression());
 
         // predicate should be a HAVING
-        Assert.assertTrue(splitFilterExpressionVisitor.visitPredicate(HAVING_PREDICATE).isPureHaving());
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitPredicate(HAVING_PREDICATE).getHavingExpression().toString(),
-                "playerStats.highScore GT [99]"
+        assertTrue(splitFilterExpressionVisitor.visitPredicate(HAVING_PREDICATE).isPureHaving());
+        assertEquals(
+                "playerStats.highScore GT [99]",
+                splitFilterExpressionVisitor.visitPredicate(HAVING_PREDICATE).getHavingExpression().toString()
         );
-        Assert.assertFalse(splitFilterExpressionVisitor.visitPredicate(HAVING_PREDICATE).isPureWhere());
-        Assert.assertNull(splitFilterExpressionVisitor.visitPredicate(HAVING_PREDICATE).getWhereExpression());
+        assertFalse(splitFilterExpressionVisitor.visitPredicate(HAVING_PREDICATE).isPureWhere());
+        assertNull(splitFilterExpressionVisitor.visitPredicate(HAVING_PREDICATE).getWhereExpression());
     }
 
     @Test
     public void testVisitAndExpression() {
         // pure-W AND pure-W
         AndFilterExpression filterExpression = new AndFilterExpression(WHERE_PREDICATE, WHERE_PREDICATE);
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getWhereExpression().toString(),
-                "(playerStats.id IN [foo] AND playerStats.id IN [foo])"
+        assertEquals(
+                "(playerStats.id IN [foo] AND playerStats.id IN [foo])",
+                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getWhereExpression().toString()
         );
-        Assert.assertNull(splitFilterExpressionVisitor.visitAndExpression(filterExpression).getHavingExpression());
+        assertNull(splitFilterExpressionVisitor.visitAndExpression(filterExpression).getHavingExpression());
 
         // pure-H AND pure-W
         filterExpression = new AndFilterExpression(HAVING_PREDICATE, WHERE_PREDICATE);
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getWhereExpression().toString(),
-                "playerStats.id IN [foo]"
+        assertEquals(
+                "playerStats.id IN [foo]",
+                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getWhereExpression().toString()
         );
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getHavingExpression().toString(),
-                "playerStats.highScore GT [99]"
+        assertEquals(
+                "playerStats.highScore GT [99]",
+                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getHavingExpression().toString()
         );
 
         // pure-W AND pure-H
         filterExpression = new AndFilterExpression(WHERE_PREDICATE, HAVING_PREDICATE);
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getWhereExpression().toString(),
-                "playerStats.id IN [foo]"
+        assertEquals(
+                "playerStats.id IN [foo]",
+                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getWhereExpression().toString()
         );
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getHavingExpression().toString(),
-                "playerStats.highScore GT [99]"
+        assertEquals(
+                "playerStats.highScore GT [99]",
+                splitFilterExpressionVisitor.visitAndExpression(filterExpression).getHavingExpression().toString()
         );
 
         // non-pure case - H1 AND W1 AND H2
         AndFilterExpression and1 = new AndFilterExpression(HAVING_PREDICATE, WHERE_PREDICATE);
         AndFilterExpression and2 = new AndFilterExpression(and1, HAVING_PREDICATE);
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(and2).getWhereExpression().toString(),
-                "playerStats.id IN [foo]"
+        assertEquals(
+                "playerStats.id IN [foo]",
+                splitFilterExpressionVisitor.visitAndExpression(and2).getWhereExpression().toString()
         );
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(and2).getHavingExpression().toString(),
-                "(playerStats.highScore GT [99] AND playerStats.highScore GT [99])"
+        assertEquals(
+                "(playerStats.highScore GT [99] AND playerStats.highScore GT [99])",
+                splitFilterExpressionVisitor.visitAndExpression(and2).getHavingExpression().toString()
         );
 
         // non-pure case - (H1 OR H2) AND W1
         OrFilterExpression or = new OrFilterExpression(HAVING_PREDICATE, HAVING_PREDICATE);
         AndFilterExpression and = new AndFilterExpression(or, WHERE_PREDICATE);
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(and).getWhereExpression().toString(),
-                "playerStats.id IN [foo]"
+        assertEquals(
+                "playerStats.id IN [foo]",
+                splitFilterExpressionVisitor.visitAndExpression(and).getWhereExpression().toString()
         );
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitAndExpression(and).getHavingExpression().toString(),
-                "(playerStats.highScore GT [99] OR playerStats.highScore GT [99])"
+        assertEquals(
+                "(playerStats.highScore GT [99] OR playerStats.highScore GT [99])",
+                splitFilterExpressionVisitor.visitAndExpression(and).getHavingExpression().toString()
         );
     }
 
@@ -133,27 +136,27 @@ public void testVisitAndExpression() {
     public void testVisitOrExpression() {
         // pure-W OR pure-W
         OrFilterExpression filterExpression = new OrFilterExpression(WHERE_PREDICATE, WHERE_PREDICATE);
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitOrExpression(filterExpression).getWhereExpression().toString(),
-                "(playerStats.id IN [foo] OR playerStats.id IN [foo])"
+        assertEquals(
+                "(playerStats.id IN [foo] OR playerStats.id IN [foo])",
+                splitFilterExpressionVisitor.visitOrExpression(filterExpression).getWhereExpression().toString()
         );
-        Assert.assertNull(splitFilterExpressionVisitor.visitOrExpression(filterExpression).getHavingExpression());
+        assertNull(splitFilterExpressionVisitor.visitOrExpression(filterExpression).getHavingExpression());
 
         // H1 OR W1
         OrFilterExpression or = new OrFilterExpression(HAVING_PREDICATE, WHERE_PREDICATE);
-        Assert.assertNull(splitFilterExpressionVisitor.visitOrExpression(or).getWhereExpression());
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitOrExpression(or).getHavingExpression().toString(),
-                "(playerStats.highScore GT [99] OR playerStats.id IN [foo])"
+        assertNull(splitFilterExpressionVisitor.visitOrExpression(or).getWhereExpression());
+        assertEquals(
+                "(playerStats.highScore GT [99] OR playerStats.id IN [foo])",
+                splitFilterExpressionVisitor.visitOrExpression(or).getHavingExpression().toString()
         );
 
         // (W1 AND H1) OR W2
         AndFilterExpression and = new AndFilterExpression(WHERE_PREDICATE, HAVING_PREDICATE);
         or = new OrFilterExpression(and, WHERE_PREDICATE);
-        Assert.assertNull(splitFilterExpressionVisitor.visitOrExpression(or).getWhereExpression());
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitOrExpression(or).getHavingExpression().toString(),
-                "((playerStats.id IN [foo] AND playerStats.highScore GT [99]) OR playerStats.id IN [foo])"
+        assertNull(splitFilterExpressionVisitor.visitOrExpression(or).getWhereExpression());
+        assertEquals(
+                "((playerStats.id IN [foo] AND playerStats.highScore GT [99]) OR playerStats.id IN [foo])",
+                splitFilterExpressionVisitor.visitOrExpression(or).getHavingExpression().toString()
         );
     }
 
@@ -162,10 +165,10 @@ public void testVisitNotExpression() {
         NotFilterExpression notExpression = new NotFilterExpression(
                 new AndFilterExpression(WHERE_PREDICATE, HAVING_PREDICATE)
         );
-        Assert.assertNull(splitFilterExpressionVisitor.visitNotExpression(notExpression).getWhereExpression());
-        Assert.assertEquals(
-                splitFilterExpressionVisitor.visitNotExpression(notExpression).getHavingExpression().toString(),
-                "(playerStats.id NOT [foo] OR playerStats.highScore LE [99])"
+        assertNull(splitFilterExpressionVisitor.visitNotExpression(notExpression).getWhereExpression());
+        assertEquals(
+                "(playerStats.id NOT [foo] OR playerStats.highScore LE [99])",
+                splitFilterExpressionVisitor.visitNotExpression(notExpression).getHavingExpression().toString()
         );
 
     }
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetricTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetricTest.java
index 24225a3557..51510f1f2a 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetricTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/metric/AggregatedMetricTest.java
@@ -5,11 +5,12 @@
  */
 package com.yahoo.elide.datastores.aggregation.metric;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.mockito.Mockito.mock;
 
 import com.yahoo.elide.datastores.aggregation.schema.Schema;
-import org.testng.Assert;
-import org.testng.annotations.Test;
+import org.junit.jupiter.api.Test;
 
 import java.util.Collections;
 import java.util.HashSet;
@@ -37,16 +38,16 @@ public class AggregatedMetricTest {
 
     @Test
     public void testMetricAsCollectionElement() {
-        Assert.assertEquals(SIMPLE_METRIC_1, SIMPLE_METRIC_1);
-        Assert.assertEquals(SIMPLE_METRIC_2, SIMPLE_METRIC_2);
-        Assert.assertNotEquals(SIMPLE_METRIC_1, SIMPLE_METRIC_2);
-        Assert.assertNotEquals(SIMPLE_METRIC_1.hashCode(), SIMPLE_METRIC_2.hashCode());
+        assertEquals(SIMPLE_METRIC_1, SIMPLE_METRIC_1);
+        assertEquals(SIMPLE_METRIC_2, SIMPLE_METRIC_2);
+        assertNotEquals(SIMPLE_METRIC_1, SIMPLE_METRIC_2);
+        assertNotEquals(SIMPLE_METRIC_1.hashCode(), SIMPLE_METRIC_2.hashCode());
 
         // different metrics should be separate elements in Set
         Set set = new HashSet<>();
         set.add(SIMPLE_METRIC_1);
 
-        Assert.assertEquals(set.size(), 1);
+        assertEquals(1, set.size());
 
         // a separate same object doesn't increase collection size
         Metric sameMetric = new AggregatedMetric(
@@ -56,29 +57,29 @@ public void testMetricAsCollectionElement() {
                 long.class,
                 Collections.singletonList(Max.class)
         );
-        Assert.assertEquals(sameMetric, SIMPLE_METRIC_1);
+        assertEquals(SIMPLE_METRIC_1, sameMetric);
         set.add(sameMetric);
-        Assert.assertEquals(set.size(), 1);
+        assertEquals(1, set.size());
 
         set.add(SIMPLE_METRIC_1);
-        Assert.assertEquals(set.size(), 1);
+        assertEquals(1, set.size());
 
         set.add(SIMPLE_METRIC_2);
-        Assert.assertEquals(set.size(), 2);
+        assertEquals(2, set.size());
     }
 
     @Test
     public void testToString() {
         // simple metric
-        Assert.assertEquals(
-                SIMPLE_METRIC_1.toString(),
-                "AggregatedMetric[name='highScore', longName='highScore', description='highScore', dataType=long, aggregations=Max]"
+        assertEquals(
+                "AggregatedMetric[name='highScore', longName='highScore', description='highScore', dataType=long, aggregations=Max]",
+                SIMPLE_METRIC_1.toString()
         );
 
         // computed metric
-        Assert.assertEquals(
-                SIMPLE_METRIC_2.toString(),
-                "AggregatedMetric[name='timeSpentPerGame', longName='timeSpentPerGame', description='timeSpentPerGame', dataType=class java.lang.Float, aggregations=Max]"
+        assertEquals(
+                "AggregatedMetric[name='timeSpentPerGame', longName='timeSpentPerGame', description='timeSpentPerGame', dataType=class java.lang.Float, aggregations=Max]",
+                SIMPLE_METRIC_2.toString()
         );
     }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/resources/country.csv b/elide-datastore/elide-datastore-aggregation/src/test/resources/country.csv
index 207eb92c5e..e618f4e629 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/resources/country.csv
+++ b/elide-datastore/elide-datastore-aggregation/src/test/resources/country.csv
@@ -1,3 +1,3 @@
 id,isoCode,name
-840,USA,United States
 344,HKG,Hong Kong
+840,USA,United States
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv b/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv
index 7a75dd4765..e0b18466d0 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv
+++ b/elide-datastore/elide-datastore-aggregation/src/test/resources/player_stats.csv
@@ -1,3 +1,4 @@
 id,highScore,lowScore,overallRating,country_id,player_id,recordedDate
 Jon Doe,1234,72,Good,840,1,2019-07-12 00:00:00
 Jane Doe,2412,241,Great,840,2,2019-07-11 00:00:00
+Han,1000,72,Good,344,3,2019-07-13 00:00:00

From ac0e4a163c78ecac6e8ef24d659a8ed5ad7c2f71 Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Fri, 27 Sep 2019 14:37:05 -0500
Subject: [PATCH 44/47] remove unused part

---
 .../datastores/aggregation/engine/SQLEntityHydrator.java    | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
index 3717c36616..2ffa1a0f12 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
@@ -71,11 +71,5 @@ protected Map getRelationshipValues(
         return loaded.stream()
                 .map(obj -> new AbstractMap.SimpleImmutableEntry<>((Object) getEntityDictionary().getId(obj), obj))
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
-
-        // returns a mapping as [joinId(0) -> loaded(0), joinId(1) -> loaded(1), ...]
-//        return IntStream.range(0, loaded.size())
-//                .boxed()
-//                .map(i -> new AbstractMap.SimpleImmutableEntry<>(uniqueIds.get(i), loaded.get(i)))
-//                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
 }

From 623a288e63d3d414981eae6c545d1d77c16f6a5e Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Fri, 27 Sep 2019 14:51:55 -0500
Subject: [PATCH 45/47] make codacy happy

---
 .../com/yahoo/elide/datastores/aggregation/SchemaTest.java     | 2 +-
 .../datastores/aggregation/dimension/EntityDimensionTest.java  | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
index f51a1a4399..417f711354 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/SchemaTest.java
@@ -40,7 +40,7 @@ public static void setupEntityDictionary() {
     }
 
     @Test
-    void testMetricCheck() {
+    public void testMetricCheck() {
         assertTrue(playerStatsSchema.isMetricField("highScore"));
         assertFalse(playerStatsSchema.isMetricField("country"));
     }
diff --git a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimensionTest.java b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimensionTest.java
index 9e6942f495..e3e5c17f37 100644
--- a/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimensionTest.java
+++ b/elide-datastore/elide-datastore-aggregation/src/test/java/com/yahoo/elide/datastores/aggregation/dimension/EntityDimensionTest.java
@@ -23,6 +23,7 @@
 import javax.persistence.Entity;
 
 public class EntityDimensionTest {
+    private static EntityDictionary entityDictionary;
 
     /**
      * A class for testing un-happy path on finding friendly name.
@@ -53,8 +54,6 @@ public String getSubTitle() {
         }
     }
 
-    private static EntityDictionary entityDictionary;
-
     @BeforeAll
     public static void setupEntityDictionary() {
         entityDictionary = new EntityDictionary(Collections.emptyMap());

From 293f8b529ddfc48e3af1ac72b1a481dc55edabbb Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Fri, 27 Sep 2019 16:35:01 -0500
Subject: [PATCH 46/47] should use getParametrizedType

---
 .../datastores/aggregation/engine/AbstractEntityHydrator.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
index 325530ecff..4b9b16292e 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
@@ -196,7 +196,9 @@ private void populateObjectLookupTable() {
         for (Map.Entry> entry : hydrationIdsByRelationship.entrySet()) {
             String joinField = entry.getKey();
             List joinFieldIds = entry.getValue();
-            Class entityType = getEntityDictionary().getType(getQuery().getSchema().getEntityClass(), joinField);
+            Class entityType = getEntityDictionary().getParameterizedType(
+                    getQuery().getSchema().getEntityClass(),
+                    joinField);
 
             getStitchList().populateLookup(entityType, getRelationshipValues(entityType, joinField, joinFieldIds));
         }

From 9fcaf6fd5904e53bced3485bdf41a06e1b1d5713 Mon Sep 17 00:00:00 2001
From: hchen04 
Date: Fri, 27 Sep 2019 16:51:34 -0500
Subject: [PATCH 47/47] address comments

---
 .../engine/AbstractEntityHydrator.java           | 16 +++++++---------
 .../aggregation/engine/SQLEntityHydrator.java    |  7 +++----
 .../aggregation/engine/StitchList.java           |  8 ++++++--
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
index 4b9b16292e..fa9c9ca507 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/AbstractEntityHydrator.java
@@ -90,7 +90,7 @@ public AbstractEntityHydrator(List results, Query query, EntityDictionar
      * Note the relationship cannot be toMany. This method will be invoked for every relationship field of the
      * requested entity. Its implementation should return the result of the following query
      * 

-     * Given a relationship {@code joinField} in an entity of type {@code entityClass}, loads all relationship
+     * Given a relationship with type {@code relationshipType} in an entity, loads all relationship
      * objects whose foreign keys are one of the specified list, {@code joinFieldIds}.
      * 

      * For example, when the relationship is loaded from SQL and we have the following example identity:
@@ -106,7 +106,7 @@ public AbstractEntityHydrator(List results, Query query, EntityDictionar
      *     }
      * }
      * 
-     * In this case {@code entityClass = PlayerStats.class}; {@code joinField = "country"}. If {@code country} is
+     * In this case {@code relationshipType = Country.class}. If {@code country} is
      * requested in {@code PlayerStats} query and 3 stats, for example, are found in database whose country ID's are
      * {@code joinFieldIds = [840, 344, 840]}, then this method should effectively run the following query (JPQL as
      * example)
@@ -117,15 +117,13 @@ public AbstractEntityHydrator(List results, Query query, EntityDictionar
      * 
      * and returns the map of [840: Country(id:840), 344: Country(id:344)]
      *
-     * @param entityClass  The type of relationship
-     * @param joinField  The relationship field name
-     * @param joinFieldIds  The specified list of join ID's against the relationshiop
+     * @param relationshipType  The type of relationship
+     * @param joinFieldIds  The specified list of join ID's against the relationship
      *
      * @return a list of hydrating values
      */
     protected abstract Map getRelationshipValues(
-            Class entityClass,
-            String joinField,
+            Class relationshipType,
             List joinFieldIds
     );
 
@@ -196,11 +194,11 @@ private void populateObjectLookupTable() {
         for (Map.Entry> entry : hydrationIdsByRelationship.entrySet()) {
             String joinField = entry.getKey();
             List joinFieldIds = entry.getValue();
-            Class entityType = getEntityDictionary().getParameterizedType(
+            Class relationshipType = getEntityDictionary().getParameterizedType(
                     getQuery().getSchema().getEntityClass(),
                     joinField);
 
-            getStitchList().populateLookup(entityType, getRelationshipValues(entityType, joinField, joinFieldIds));
+            getStitchList().populateLookup(relationshipType, getRelationshipValues(relationshipType, joinFieldIds));
         }
     }
 }
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
index 2ffa1a0f12..257c7c40a0 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/SQLEntityHydrator.java
@@ -47,8 +47,7 @@ public SQLEntityHydrator(
 
     @Override
     protected Map getRelationshipValues(
-            Class entityClass,
-            String joinField,
+            Class relationshipType,
             List joinFieldIds
     ) {
         if (joinFieldIds.isEmpty()) {
@@ -61,8 +60,8 @@ protected Map getRelationshipValues(
                 .createQuery(
                         String.format(
                                 "SELECT e FROM %s e WHERE %s IN (:idList)",
-                                entityClass.getCanonicalName(),
-                                getEntityDictionary().getIdFieldName(entityClass)
+                                relationshipType.getCanonicalName(),
+                                getEntityDictionary().getIdFieldName(relationshipType)
                         )
                 )
                 .setParameter("idList", uniqueIds)
diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
index 4c40d1138b..9d8c18dd2c 100644
--- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
+++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/engine/StitchList.java
@@ -93,7 +93,11 @@ public void todo(Object entityInstance, String fieldName, Object value) {
      * @param idToInstance  A map from relationship ID to the actual relationship instance with that ID
      */
     public void populateLookup(Class relationshipType, Map idToInstance) {
-        getObjectLookups().put(relationshipType, idToInstance);
+        if (getObjectLookups().containsKey(relationshipType)) {
+            getObjectLookups().get(relationshipType).putAll(idToInstance);
+        } else {
+            getObjectLookups().put(relationshipType, idToInstance);
+        }
     }
 
     /**
@@ -106,7 +110,7 @@ public void stitch() {
             String relationshipName = todo.getRelationshipName();
             Object foreignKey = todo.getForeignKey();
 
-            Class relationshipType = getEntityDictionary().getType(entityInstance, relationshipName);
+            Class relationshipType = getEntityDictionary().getParameterizedType(entityInstance, relationshipName);
             Object relationshipValue = getObjectLookups().get(relationshipType).get(foreignKey);
 
             getEntityDictionary().setValue(entityInstance, relationshipName, relationshipValue);