Merge branch 'master' into feat_glue_s3_tag_handle_virtual_views

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -30,6 +30,7 @@
 import com.linkedin.datahub.graphql.generated.CorpUserInfo;
 import com.linkedin.datahub.graphql.generated.Dashboard;
 import com.linkedin.datahub.graphql.generated.DashboardInfo;
+import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts;
 import com.linkedin.datahub.graphql.generated.DataFlow;
 import com.linkedin.datahub.graphql.generated.DataJob;
 import com.linkedin.datahub.graphql.generated.DataJobInputOutput;
@@ -81,6 +82,7 @@
 import com.linkedin.datahub.graphql.resolvers.config.AppConfigResolver;
 import com.linkedin.datahub.graphql.resolvers.container.ContainerEntitiesResolver;
 import com.linkedin.datahub.graphql.resolvers.container.ParentContainersResolver;
+import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsResolver;
 import com.linkedin.datahub.graphql.resolvers.dataset.DatasetHealthResolver;
 import com.linkedin.datahub.graphql.resolvers.deprecation.UpdateDeprecationResolver;
 import com.linkedin.datahub.graphql.resolvers.domain.CreateDomainResolver;
@@ -1015,13 +1017,19 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) {
                     })
             )
             .dataFetcher("parentContainers", new ParentContainersResolver(entityClient))
+            .dataFetcher("usageStats", new DashboardUsageStatsResolver(timeseriesAspectService))
         );
         builder.type("DashboardInfo", typeWiring -> typeWiring
             .dataFetcher("charts", new LoadableTypeBatchResolver<>(chartType,
                     (env) -> ((DashboardInfo) env.getSource()).getCharts().stream()
                         .map(Chart::getUrn)
                         .collect(Collectors.toList())))
         );
+        builder.type("DashboardUserUsageCounts", typeWiring -> typeWiring
+            .dataFetcher("user", new LoadableTypeResolver<>(
+                corpUserType,
+                (env) -> ((DashboardUserUsageCounts) env.getSource()).getUser().getUrn()))
+        );
     }
 
     /**

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardUsageMetricMapper.java

@@ -0,0 +1,34 @@
+package com.linkedin.datahub.graphql.types.dashboard.mappers;
+
+import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics;
+import com.linkedin.datahub.graphql.types.mappers.TimeSeriesAspectMapper;
+import com.linkedin.metadata.aspect.EnvelopedAspect;
+import com.linkedin.metadata.utils.GenericRecordUtils;
+import javax.annotation.Nonnull;
+
+
+public class DashboardUsageMetricMapper implements TimeSeriesAspectMapper<DashboardUsageMetrics> {
+
+  public static final DashboardUsageMetricMapper INSTANCE = new DashboardUsageMetricMapper();
+
+  public static DashboardUsageMetrics map(@Nonnull final EnvelopedAspect envelopedAspect) {
+    return INSTANCE.apply(envelopedAspect);
+  }
+
+  @Override
+  public DashboardUsageMetrics apply(EnvelopedAspect envelopedAspect) {
+    com.linkedin.dashboard.DashboardUsageStatistics gmsDashboardUsageStatistics =
+        GenericRecordUtils.deserializeAspect(envelopedAspect.getAspect().getValue(),
+            envelopedAspect.getAspect().getContentType(), com.linkedin.dashboard.DashboardUsageStatistics.class);
+
+    final com.linkedin.datahub.graphql.generated.DashboardUsageMetrics dashboardUsageMetrics =
+        new com.linkedin.datahub.graphql.generated.DashboardUsageMetrics();
+    dashboardUsageMetrics.setLastViewed(gmsDashboardUsageStatistics.getLastViewedAt());
+    dashboardUsageMetrics.setViewsCount(gmsDashboardUsageStatistics.getViewsCount());
+    dashboardUsageMetrics.setExecutionsCount(gmsDashboardUsageStatistics.getExecutionsCount());
+    dashboardUsageMetrics.setFavoritesCount(gmsDashboardUsageStatistics.getFavoritesCount());
+    dashboardUsageMetrics.setTimestampMillis(gmsDashboardUsageStatistics.getTimestampMillis());
+
+    return dashboardUsageMetrics;
+  }
+}

datahub-graphql-core/src/main/resources/entity.graphql

@@ -4121,6 +4121,12 @@ type Dashboard implements EntityWithRelationships & Entity {
     """
     lineage(input: LineageInput!): EntityLineageResult
 
+
+    """
+    Experimental (Subject to breaking change) -- Statistics about how this Dashboard is used
+    """
+    usageStats(startTimeMillis: Long, endTimeMillis: Long, limit: Int): DashboardUsageQueryResult
+
     """
     Deprecated, use properties field instead
     Additional read only information about the dashboard
@@ -5247,6 +5253,158 @@ type FieldUsageCounts {
     count: Int
 }
 
+"""
+Information about individual user usage of a Dashboard
+"""
+type DashboardUserUsageCounts {
+    """
+    The user of the Dashboard
+    """
+    user: CorpUser
+
+    """
+    number of times dashboard has been viewed by the user
+    """
+    viewsCount: Int
+
+    """
+    number of dashboard executions by the user
+    """
+    executionsCount: Int
+
+    """
+    Normalized numeric metric representing user's dashboard usage
+    Higher value represents more usage
+    """
+    usageCount: Int
+}
+
+"""
+The result of a dashboard usage query
+"""
+type DashboardUsageQueryResult {
+    """
+    A set of relevant time windows for use in displaying usage statistics
+    """
+    buckets: [DashboardUsageAggregation]
+
+    """
+    A set of rolled up aggregations about the dashboard usage
+    """
+    aggregations: DashboardUsageQueryResultAggregations
+
+    """
+    A set of absolute dashboard usage metrics
+    """
+    metrics: [DashboardUsageMetrics!]
+
+}
+
+"""
+A set of rolled up aggregations about the Dashboard usage
+"""
+type DashboardUsageQueryResultAggregations {
+    """
+    The count of unique Dashboard users within the queried time range
+    """
+    uniqueUserCount: Int
+
+    """
+    The specific per user usage counts within the queried time range
+    """
+    users: [DashboardUserUsageCounts]
+
+    """
+    The total number of dashboard views within the queried time range
+    """
+    viewsCount: Int
+
+    """
+    The total number of dashboard executions within the queried time range
+    """
+    executionsCount: Int
+
+}
+
+
+"""
+A set of absolute dashboard usage metrics
+"""
+type DashboardUsageMetrics implements TimeSeriesAspect {
+    """
+    The time at which the metrics were reported
+    """
+    timestampMillis: Long!
+
+    """
+    The total number of times dashboard has been favorited
+    FIXME: Qualifies as Popularity Metric rather than Usage Metric?
+    """
+    favoritesCount: Int
+
+    """
+    The total number of dashboard views
+    """
+    viewsCount: Int
+
+    """
+    The total number of dashboard execution
+    """
+    executionsCount: Int
+
+    """
+    The time when this dashboard was last viewed
+    """
+    lastViewed: Long
+
+}
+
+"""
+An aggregation of Dashboard usage statistics
+"""
+type DashboardUsageAggregation {
+    """
+    The time window start time
+    """
+    bucket: Long
+
+    """
+    The time window span
+    """
+    duration: WindowDuration
+
+    """
+    The resource urn associated with the usage information, eg a Dashboard urn
+    """
+    resource: String
+
+    """
+    The rolled up usage metrics
+    """
+    metrics: DashboardUsageAggregationMetrics
+}
+
+"""
+Rolled up metrics about Dashboard usage over time
+"""
+type DashboardUsageAggregationMetrics {
+    """
+    The unique number of dashboard users within the time range
+    """
+    uniqueUserCount: Int
+
+    """
+    The total number of dashboard views within the time range
+    """
+    viewsCount: Int
+
+    """
+    The total number of dashboard executions within the time range
+    """
+    executionsCount: Int
+
+}
+
 """
 The duration of a fixed window of time
 """
@@ -5273,7 +5431,7 @@ enum WindowDuration {
 }
 
 """
-A time range used in fetching Dataset Usage statistics
+A time range used in fetching Usage statistics
 """
 enum TimeRange {
     """

datahub-web-react/build.gradle

@@ -74,7 +74,7 @@ clean {
   delete 'build'
   delete 'just'
   delete 'src/types.generated.ts'
-  delete 'src/graphql/*.generated.ts'
+  delete fileTree(dir: 'src/graphql', include: '*.generated.ts')
 }
 
 configurations {

datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx

@@ -6,6 +6,8 @@ import { getSourceConfigs, jsonToYaml, yamlToJson } from '../utils';
 import { YamlEditor } from './YamlEditor';
 import { ANTD_GRAY } from '../../../entity/shared/constants';
 import { IngestionSourceBuilderStep } from './steps';
+import RecipeBuilder from './RecipeBuilder';
+import { CONNECTORS_WITH_FORM } from './RecipeForm/utils';
 
 const LOOKML_DOC_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/looker#module-lookml';
 
@@ -37,17 +39,19 @@ const ControlsContainer = styled.div`
 export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) => {
     const existingRecipeJson = state.config?.recipe;
     const existingRecipeYaml = existingRecipeJson && jsonToYaml(existingRecipeJson);
+    const { type } = state;
+    const sourceConfigs = getSourceConfigs(type as string);
 
-    const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || '');
+    const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || sourceConfigs.placeholderRecipe);
 
     useEffect(() => {
-        setStagedRecipeYml(existingRecipeYaml || '');
+        if (existingRecipeYaml) {
+            setStagedRecipeYml(existingRecipeYaml);
+        }
     }, [existingRecipeYaml]);
 
     const [stepComplete, setStepComplete] = useState(false);
 
-    const { type } = state;
-    const sourceConfigs = getSourceConfigs(type as string);
     const isEditing: boolean = prev === undefined;
     const displayRecipe = stagedRecipeYml || sourceConfigs.placeholderRecipe;
     const sourceDisplayName = sourceConfigs.displayName;
@@ -85,6 +89,19 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps)
         goTo(IngestionSourceBuilderStep.CREATE_SCHEDULE);
     };
 
+    if (type && CONNECTORS_WITH_FORM.has(type)) {
+        return (
+            <RecipeBuilder
+                type={type}
+                isEditing={isEditing}
+                displayRecipe={displayRecipe}
+                setStagedRecipe={setStagedRecipeYml}
+                onClickNext={onClickNext}
+                goToPrevious={prev}
+            />
+        );
+    }
+
     return (
         <>
             <Section>