Add query and chart on Dashboard (amundsen-io#224)

* Add query and chart on Dashboard

* Update

* Increment version

* Update test case

databuilder/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py

@@ -0,0 +1,98 @@
+import logging
+
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+from typing import Any  # noqa: F401
+
+from databuilder import Scoped
+from databuilder.extractor.base_extractor import Extractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.base_transformer import ChainedTransformer
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.template_variable_substitution_transformer import \
+    TemplateVariableSubstitutionTransformer, FIELD_NAME, TEMPLATE
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardChartsExtractor(Extractor):
+    """
+    A Extractor that extracts Dashboard charts
+
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+        self._conf = conf
+
+        restapi_query = self._build_restapi_query()
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
+            restapi_query=restapi_query,
+            conf=self._conf
+        )
+
+        # Constructing URL using resource path via TemplateVariableSubstitutionTransformer
+        transformers = []
+        chart_url_transformer = TemplateVariableSubstitutionTransformer()
+        chart_url_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, chart_url_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({FIELD_NAME: 'chart_url',
+                                         TEMPLATE: 'https://app.mode.com{chart_url}'})))
+
+        transformers.append(chart_url_transformer)
+
+        dict_to_model_transformer = DictToModel()
+        dict_to_model_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict(
+                    {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_chart.DashboardChart'})))
+        transformers.append(dict_to_model_transformer)
+
+        self._transformer = ChainedTransformer(transformers=transformers)
+
+    def extract(self):
+        # type: () -> Any
+
+        record = self._extractor.extract()
+        if not record:
+            return None
+
+        return self._transformer.transform(record=record)
+
+    def get_scope(self):
+        # type: () -> str
+        return 'extractor.mode_dashboard_chart'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
+        API, and run API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard execution (run)
+        """
+        # type: () -> RestApiQuery
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+        json_path = '(_embedded.reports[*].token)'
+        field_names = ['dashboard_id']
+        reports_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
+                                     json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
+        json_path = '_embedded.queries[*].[token,name]'
+        field_names = ['query_id', 'query_name']
+        query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
+                                         json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        charts_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries/{query_id}/charts'
+        json_path = '(_embedded.charts[*].token) | (_embedded.charts[*]._links.report_viz_web.href)'
+        field_names = ['chart_id', 'chart_url']
+        chart_names_query = RestApiQuery(query_to_join=query_names_query, url=charts_url_template, params=params,
+                                         json_path=json_path, field_names=field_names, skip_no_result=True,
+                                         json_path_contains_or=True)
+
+        return chart_names_query

databuilder/databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py

@@ -0,0 +1,93 @@
+import logging
+
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+from typing import Any  # noqa: F401
+
+from databuilder import Scoped
+from databuilder.extractor.base_extractor import Extractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.base_transformer import ChainedTransformer
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.template_variable_substitution_transformer import \
+    TemplateVariableSubstitutionTransformer, TEMPLATE, FIELD_NAME
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardQueriesExtractor(Extractor):
+    """
+    A Extractor that extracts run (execution) status and timestamp.
+
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+        self._conf = conf
+
+        restapi_query = self._build_restapi_query()
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
+            restapi_query=restapi_query,
+            conf=self._conf
+        )
+
+        # Constructing URL using several ID via TemplateVariableSubstitutionTransformer
+        transformers = []
+        variable_substitution_transformer = TemplateVariableSubstitutionTransformer()
+        variable_substitution_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf,
+                                        variable_substitution_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({FIELD_NAME: 'url',
+                                         TEMPLATE: 'https://app.mode.com/{organization}'
+                                                   '/reports/{dashboard_id}/queries/{query_id}'})))
+
+        transformers.append(variable_substitution_transformer)
+
+        dict_to_model_transformer = DictToModel()
+        dict_to_model_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict(
+                    {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_query.DashboardQuery'})))
+        transformers.append(dict_to_model_transformer)
+
+        self._transformer = ChainedTransformer(transformers=transformers)
+
+    def extract(self):
+        # type: () -> Any
+
+        record = self._extractor.extract()
+        if not record:
+            return None
+
+        return self._transformer.transform(record=record)
+
+    def get_scope(self):
+        # type: () -> str
+        return 'extractor.mode_dashboard_query'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
+        API, and queries API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard execution (run)
+        """
+        # type: () -> RestApiQuery
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+        json_path = '(_embedded.reports[*].token)'
+        field_names = ['dashboard_id']
+        reports_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                     json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
+        json_path = '_embedded.queries[*].[token,name]'
+        field_names = ['query_id', 'query_name']
+        query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
+                                         json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        return query_names_query

databuilder/databuilder/models/dashboard/dashboard_chart.py

@@ -0,0 +1,118 @@
+import logging
+
+from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
+
+from databuilder.models.dashboard.dashboard_query import DashboardQuery
+from databuilder.models.neo4j_csv_serde import (
+    Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
+
+LOGGER = logging.getLogger(__name__)
+
+
+class DashboardChart(Neo4jCsvSerializable):
+    """
+    A model that encapsulate Dashboard's charts
+    """
+    DASHBOARD_CHART_LABEL = 'Chart'
+    DASHBOARD_CHART_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group_id}/' \
+                                 '{dashboard_id}/query/{query_id}/chart/{chart_id}'
+    CHART_RELATION_TYPE = 'HAS_CHART'
+    CHART_REVERSE_RELATION_TYPE = 'CHART_OF'
+
+    def __init__(self,
+                 dashboard_group_id,  # type: Optional[str]
+                 dashboard_id,  # type: Optional[str]
+                 query_id,  # type: str
+                 chart_id,  # type: str
+                 chart_name=None,  # type: Optional[str]
+                 chart_type=None,  # type: Optional[str]
+                 chart_url=None,  # type: Optional[str]
+                 product='',  # type: Optional[str]
+                 cluster='gold',  # type: str
+                 **kwargs
+                 ):
+        self._dashboard_group_id = dashboard_group_id
+        self._dashboard_id = dashboard_id
+        self._query_id = query_id
+        self._chart_id = chart_id if chart_id else chart_name
+        self._chart_name = chart_name
+        self._chart_type = chart_type
+        self._chart_url = chart_url
+        self._product = product
+        self._cluster = cluster
+        self._node_iterator = self._create_node_iterator()
+        self._relation_iterator = self._create_relation_iterator()
+
+    def create_next_node(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._node_iterator)
+        except StopIteration:
+            return None
+
+    def _create_node_iterator(self):  # noqa: C901
+        # type: () -> Iterator[[Dict[str, Any]]]
+        node = {
+            NODE_LABEL: DashboardChart.DASHBOARD_CHART_LABEL,
+            NODE_KEY: self._get_chart_node_key(),
+            'id': self._chart_id
+        }
+
+        if self._chart_name:
+            node['name'] = self._chart_name
+
+        if self._chart_type:
+            node['type'] = self._chart_type
+
+        if self._chart_url:
+            node['url'] = self._chart_url
+
+        yield node
+
+    def create_next_relation(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._relation_iterator)
+        except StopIteration:
+            return None
+
+    def _create_relation_iterator(self):
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            RELATION_START_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL,
+            RELATION_END_LABEL: DashboardChart.DASHBOARD_CHART_LABEL,
+            RELATION_START_KEY: DashboardQuery.DASHBOARD_QUERY_KEY_FORMAT.format(
+                product=self._product,
+                cluster=self._cluster,
+                dashboard_group_id=self._dashboard_group_id,
+                dashboard_id=self._dashboard_id,
+                query_id=self._query_id
+            ),
+            RELATION_END_KEY: self._get_chart_node_key(),
+            RELATION_TYPE: DashboardChart.CHART_RELATION_TYPE,
+            RELATION_REVERSE_TYPE: DashboardChart.CHART_REVERSE_RELATION_TYPE
+        }
+
+    def _get_chart_node_key(self):
+        return DashboardChart.DASHBOARD_CHART_KEY_FORMAT.format(
+            product=self._product,
+            cluster=self._cluster,
+            dashboard_group_id=self._dashboard_group_id,
+            dashboard_id=self._dashboard_id,
+            query_id=self._query_id,
+            chart_id=self._chart_id
+        )
+
+    def __repr__(self):
+        return 'DashboardChart({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format(
+            self._dashboard_group_id,
+            self._dashboard_id,
+            self._query_id,
+            self._chart_id,
+            self._chart_name,
+            self._chart_type,
+            self._chart_url,
+            self._product,
+            self._cluster
+        )