-
Notifications
You must be signed in to change notification settings - Fork 208
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Mode Dashboard extractor with Generic REST API Query (#194)
* Initial check in on REST API Query * Working version * docstring * Update * Update * Make unit test happy * Update docstring * Update * Update * Update * Adding unit tests * Updated README.md * jsonpath_rw to extra_requires
- Loading branch information
1 parent
9556b18
commit 01a0f96
Showing
16 changed files
with
696 additions
and
63 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
98 changes: 98 additions & 0 deletions
98
databuilder/extractor/dashboard/mode_dashboard_extractor.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| import logging | ||
|
|
||
| from pyhocon import ConfigTree, ConfigFactory # noqa: F401 | ||
| from requests.auth import HTTPBasicAuth | ||
| from typing import Any # noqa: F401 | ||
|
|
||
| from databuilder import Scoped | ||
| from databuilder.extractor.base_extractor import Extractor | ||
| from databuilder.extractor.restapi.rest_api_extractor import RestAPIExtractor, REST_API_QUERY, MODEL_CLASS, \ | ||
| STATIC_RECORD_DICT | ||
| from databuilder.rest_api.base_rest_api_query import RestApiQuerySeed | ||
| from databuilder.rest_api.rest_api_query import RestApiQuery | ||
|
|
||
| # CONFIG KEYS | ||
| ORGANIZATION = 'organization' | ||
| MODE_ACCESS_TOKEN = 'mode_user_token' | ||
| MODE_PASSWORD_TOKEN = 'mode_password_token' | ||
|
|
||
| LOGGER = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class ModeDashboardExtractor(Extractor): | ||
| """ | ||
| A Extractor that extracts core metadata on Mode dashboard. https://app.mode.com/ | ||
| It extracts list of reports that consists of: | ||
| Dashboard group name (Space name) | ||
| Dashboard group id (Space token) | ||
| Dashboard group description (Space description) | ||
| Dashboard name (Report name) | ||
| Dashboard id (Report token) | ||
| Dashboard description (Report description) | ||
| Other information such as report run, owner, chart name, query name is in separate extractor. | ||
| """ | ||
|
|
||
| def init(self, conf): | ||
| # type: (ConfigTree) -> None | ||
|
|
||
| self._conf = conf | ||
|
|
||
| restapi_query = self._build_restapi_query() | ||
| self._extractor = RestAPIExtractor() | ||
| rest_api_extractor_conf = Scoped.get_scoped_conf(conf, self._extractor.get_scope()).with_fallback( | ||
| ConfigFactory.from_dict( | ||
| { | ||
| REST_API_QUERY: restapi_query, | ||
| MODEL_CLASS: 'databuilder.models.dashboard_metadata.DashboardMetadata', | ||
| STATIC_RECORD_DICT: {'product': 'mode'} | ||
| } | ||
| ) | ||
| ) | ||
|
|
||
| self._extractor.init(conf=rest_api_extractor_conf) | ||
|
|
||
| def extract(self): | ||
| # type: () -> Any | ||
|
|
||
| return self._extractor.extract() | ||
|
|
||
| def get_scope(self): | ||
| # type: () -> str | ||
|
|
||
| return 'extractor.mode_dashboard' | ||
|
|
||
| def _build_restapi_query(self): | ||
| """ | ||
| Build REST API Query. To get Mode Dashboard metadata, it needs to call two APIs (spaces API and reports | ||
| API) joining together. | ||
| :return: A RestApiQuery that provides Mode Dashboard metadata | ||
| """ | ||
| # type: () -> RestApiQuery | ||
|
|
||
| spaces_url_template = 'https://app.mode.com/api/{organization}/spaces?filter=all' | ||
| reports_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports' | ||
|
|
||
| # Seed query record for next query api to join with | ||
| seed_record = [{'organization': self._conf.get_string(ORGANIZATION)}] | ||
| seed_query = RestApiQuerySeed(seed_record=seed_record) | ||
|
|
||
| params = {'auth': HTTPBasicAuth(self._conf.get_string(MODE_ACCESS_TOKEN), | ||
| self._conf.get_string(MODE_PASSWORD_TOKEN))} | ||
|
|
||
| # Spaces | ||
| # JSONPATH expression. it goes into array which is located in _embedded.spaces and then extracts token, name, | ||
| # and description | ||
| json_path = '_embedded.spaces[*].[token,name,description]' | ||
| field_names = ['dashboard_group_id', 'dashboard_group', 'dashboard_group_description'] | ||
| spaces_query = RestApiQuery(query_to_join=seed_query, url=spaces_url_template, params=params, | ||
| json_path=json_path, field_names=field_names) | ||
|
|
||
| # Reports | ||
| # JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token, name, | ||
| # and description | ||
| json_path = '_embedded.reports[*].[token,name,description]' | ||
| field_names = ['dashboard_id', 'dashboard_name', 'description'] | ||
| reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params, | ||
| json_path=json_path, field_names=field_names, skip_no_result=True) | ||
| return reports_query |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| import logging | ||
| import importlib | ||
| from typing import Iterator, Any # noqa: F401 | ||
|
|
||
| from pyhocon import ConfigTree # noqa: F401 | ||
|
|
||
| from databuilder.extractor.base_extractor import Extractor | ||
| from databuilder.rest_api.base_rest_api_query import BaseRestApiQuery # noqa: F401 | ||
|
|
||
|
|
||
| REST_API_QUERY = 'restapi_query' | ||
| MODEL_CLASS = 'model_class' | ||
|
|
||
| # Static record that will be added into extracted record | ||
| # For example, DashboardMetadata requires product name (static name) of Dashboard and REST api does not provide | ||
| # it. and you can add {'product': 'mode'} so that it will be included in the record. | ||
| STATIC_RECORD_DICT = 'static_record_dict' | ||
|
|
||
| LOGGER = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class RestAPIExtractor(Extractor): | ||
| """ | ||
| An Extractor that calls one or more REST API to extract the data. | ||
| This extractor almost entirely depends on RestApiQuery. | ||
| """ | ||
|
|
||
| def init(self, conf): | ||
| # type: (ConfigTree) -> None | ||
|
|
||
| self._restapi_query = conf.get(REST_API_QUERY) # type: BaseRestApiQuery | ||
| self._iterator = None # type: Iterator[Dict[str, Any]] | ||
| self._static_dict = conf.get(STATIC_RECORD_DICT, dict()) | ||
| LOGGER.info('static record: {}'.format(self._static_dict)) | ||
|
|
||
| model_class = conf.get(MODEL_CLASS, None) | ||
| if model_class: | ||
| module_name, class_name = model_class.rsplit(".", 1) | ||
| mod = importlib.import_module(module_name) | ||
| self.model_class = getattr(mod, class_name) | ||
|
|
||
| def extract(self): | ||
| # type: () -> Any | ||
|
|
||
| """ | ||
| Fetch one result row from RestApiQuery, convert to {model_class} if specified before | ||
| returning. | ||
| :return: | ||
| """ | ||
|
|
||
| if not self._iterator: | ||
| self._iterator = self._restapi_query.execute() | ||
|
|
||
| try: | ||
| record = next(self._iterator) | ||
| except StopIteration: | ||
| return None | ||
|
|
||
| if self._static_dict: | ||
| record.update(self._static_dict) | ||
|
|
||
| if hasattr(self, 'model_class'): | ||
| return self.model_class(**record) | ||
|
|
||
| return record | ||
|
|
||
| def get_scope(self): | ||
| # type: () -> str | ||
|
|
||
| return 'extractor.restapi' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
Oops, something went wrong.