diff --git a/amundsen_application/config.py b/amundsen_application/config.py index 972bb665f..93240aded 100644 --- a/amundsen_application/config.py +++ b/amundsen_application/config.py @@ -64,6 +64,7 @@ class Config: # Settings for Issue tracker integration ISSUE_LABELS = [] # type: List[str] + ISSUE_TRACKER_API_TOKEN = None # type: str ISSUE_TRACKER_URL = None # type: str ISSUE_TRACKER_USER = None # type: str ISSUE_TRACKER_PASSWORD = None # type: str diff --git a/amundsen_application/models/data_issue.py b/amundsen_application/models/data_issue.py index fa25266d2..9f8197fff 100644 --- a/amundsen_application/models/data_issue.py +++ b/amundsen_application/models/data_issue.py @@ -1,13 +1,36 @@ # Copyright Contributors to the Amundsen project. # SPDX-License-Identifier: Apache-2.0 -# JIRA SDK does not return priority beyond the name -PRIORITY_MAP = { - 'Blocker': 'P0', - 'Critical': 'P1', - 'Major': 'P2', - 'Minor': 'P3' -} +from enum import Enum +from typing import Optional + + +class Priority(Enum): + P0 = ('P0', 'Blocker') + P1 = ('P1', 'Critical') + P2 = ('P2', 'Major') + P3 = ('P3', 'Minor') + + def __init__(self, level: str, jira_severity: str): + self.level = level + self.jira_severity = jira_severity + + # JIRA SDK does not return priority beyond the name + @staticmethod + def from_jira_severity(jira_severity: str) -> 'Optional[Priority]': + jira_severity_to_priority = { + p.jira_severity: p for p in Priority + } + + return jira_severity_to_priority.get(jira_severity) + + @staticmethod + def from_level(level: str) -> 'Optional[Priority]': + level_to_priority = { + p.level: p for p in Priority + } + + return level_to_priority.get(level) class DataIssue: @@ -16,22 +39,17 @@ def __init__(self, title: str, url: str, status: str, - priority: str) -> None: + priority: Optional[Priority]) -> None: self.issue_key = issue_key self.title = title self.url = url self.status = status - if priority in PRIORITY_MAP: - self.priority_display_name = PRIORITY_MAP[priority] - self.priority_name = priority.lower() - else: - self.priority_display_name = None # type: ignore - self.priority_name = None # type: ignore + self.priority = priority def serialize(self) -> dict: return {'issue_key': self.issue_key, 'title': self.title, 'url': self.url, 'status': self.status, - 'priority_name': self.priority_name, - 'priority_display_name': self.priority_display_name} + 'priority_name': self.priority.jira_severity.lower() if self.priority else None, + 'priority_display_name': self.priority.level if self.priority else None} diff --git a/amundsen_application/proxy/issue_tracker_clients/asana_client.py b/amundsen_application/proxy/issue_tracker_clients/asana_client.py new file mode 100644 index 000000000..00f1e640a --- /dev/null +++ b/amundsen_application/proxy/issue_tracker_clients/asana_client.py @@ -0,0 +1,183 @@ +# Copyright Contributors to the Amundsen project. +# SPDX-License-Identifier: Apache-2.0 + +import asana +import logging +from typing import Dict, List + +from amundsen_application.base.base_issue_tracker_client import BaseIssueTrackerClient +from amundsen_application.models.data_issue import DataIssue, Priority +from amundsen_application.models.issue_results import IssueResults + + +class AsanaClient(BaseIssueTrackerClient): + + def __init__(self, issue_labels: List[str], + issue_tracker_url: str, + issue_tracker_user: str, + issue_tracker_password: str, + issue_tracker_project_id: int, + issue_tracker_max_results: int) -> None: + self.issue_labels = issue_labels + self.asana_url = issue_tracker_url + self.asana_user = issue_tracker_user + self.asana_password = issue_tracker_password + self.asana_max_results = issue_tracker_max_results + + self.asana_project_gid = issue_tracker_project_id + self.asana_client = asana.Client.access_token(issue_tracker_password) + + asana_project = self.asana_client.projects.get_project(self.asana_project_gid) + self.asana_workspace_gid = asana_project['workspace']['gid'] + + self._setup_custom_fields() + + def get_issues(self, table_uri: str) -> IssueResults: + """ + :param table_uri: Table Uri ie databasetype://database/table + :return: Metadata of matching issues + """ + + table_parent_task_gid = self._get_parent_task_gid_for_table_uri(table_uri) + + tasks = list(self.asana_client.tasks.get_subtasks_for_task( + table_parent_task_gid, + { + 'opt_fields': [ + 'name', 'completed', 'notes', 'custom_fields', + ] + } + )) + + return IssueResults( + issues=[ + self._asana_task_to_amundsen_data_issue(task) for task in tasks + ], + total=len(tasks), + all_issues_url=self._task_url(table_parent_task_gid), + ) + + def create_issue(self, table_uri: str, title: str, description: str) -> DataIssue: + """ + Creates an issue in Jira + :param description: Description of the Jira issue + :param table_uri: Table Uri ie databasetype://database/table + :param title: Title of the Jira ticket + :return: Metadata about the newly created issue + """ + + table_parent_task_gid = self._get_parent_task_gid_for_table_uri(table_uri) + + return self._asana_task_to_amundsen_data_issue( + self.asana_client.tasks.create_subtask_for_task( + table_parent_task_gid, + { + 'name': title, + 'notes': description, + } + ) + ) + + def _setup_custom_fields(self) -> None: + TABLE_URI_FIELD_NAME = 'Table URI (Amundsen)' + PRIORITY_FIELD_NAME = 'Priority (Amundsen)' + + custom_fields = \ + self.asana_client.custom_field_settings.get_custom_field_settings_for_project( + self.asana_project_gid + ) + + custom_fields = {f['custom_field']['name']: f['custom_field'] for f in custom_fields} + + if TABLE_URI_FIELD_NAME in custom_fields: + table_uri_field = custom_fields[TABLE_URI_FIELD_NAME] + else: + table_uri_field = self.asana_client.custom_fields.create_custom_field({ + 'workspace': self.asana_workspace_gid, + 'name': TABLE_URI_FIELD_NAME, + 'format': 'custom', + 'resource_subtype': 'text', + }) + + self.asana_client.projects.add_custom_field_setting_for_project( + self.asana_project_gid, + { + 'custom_field': table_uri_field['gid'], + 'is_important': True, + } + ) + + if PRIORITY_FIELD_NAME in custom_fields: + priority_field = custom_fields[PRIORITY_FIELD_NAME] + else: + priority_field = self.asana_client.custom_fields.create_custom_field({ + 'workspace': self.asana_workspace_gid, + 'name': PRIORITY_FIELD_NAME, + 'format': 'custom', + 'resource_subtype': 'enum', + 'enum_options': [ + { + 'name': p.level + } for p in Priority + ] + }) + + self.asana_client.projects.add_custom_field_setting_for_project( + self.asana_project_gid, + { + 'custom_field': priority_field['gid'], + 'is_important': True, + } + ) + + self.table_uri_field_gid = table_uri_field['gid'] + self.priority_field_gid = priority_field['gid'] + + def _get_parent_task_gid_for_table_uri(self, table_uri: str) -> str: + table_parent_tasks = list(self.asana_client.tasks.search_tasks_for_workspace( + self.asana_workspace_gid, + { + 'projects.any': [self.asana_project_gid], + 'custom_fields.{}.value'.format(self.table_uri_field_gid): table_uri, + } + )) + + # Create the parent task if it doesn't exist. + if len(table_parent_tasks) == 0: + table_parent_task = self.asana_client.tasks.create_task({ + 'name': table_uri, + 'custom_fields': { + self.table_uri_field_gid: table_uri, + }, + 'projects': [self.asana_project_gid], + }) + + return table_parent_task['gid'] + else: + if len(table_parent_tasks) > 1: + logging.warn('There are currently two tasks with the name "{}"'.format(table_uri)) + + return table_parent_tasks[0]['gid'] + + def _task_url(self, task_gid: str) -> str: + return 'https://app.asana.com/0/{project_gid}/{task_gid}'.format( + project_gid=self.asana_project_gid, task_gid=task_gid + ) + + def _asana_task_to_amundsen_data_issue(self, task: Dict) -> DataIssue: + custom_fields = {f['gid']: f for f in task['custom_fields']} + priority_field = custom_fields[self.priority_field_gid] + + priority = None + if priority_field.get('enum_value'): + priority = Priority.from_level(priority_field['enum_value']['name']) + else: + priority = Priority.P3 + + return DataIssue( + issue_key=task['gid'], + title=task['name'], + url=self._task_url(task['gid']), + status='closed' if task['completed'] else 'open', + priority=priority, + ) diff --git a/amundsen_application/proxy/issue_tracker_clients/jira_client.py b/amundsen_application/proxy/issue_tracker_clients/jira_client.py index fa7dbf8df..fd5bbe9ad 100644 --- a/amundsen_application/proxy/issue_tracker_clients/jira_client.py +++ b/amundsen_application/proxy/issue_tracker_clients/jira_client.py @@ -8,7 +8,7 @@ from amundsen_application.base.base_issue_tracker_client import BaseIssueTrackerClient from amundsen_application.proxy.issue_tracker_clients.issue_exceptions import IssueConfigurationException -from amundsen_application.models.data_issue import DataIssue +from amundsen_application.models.data_issue import DataIssue, Priority from amundsen_application.models.issue_results import IssueResults import urllib.parse @@ -132,7 +132,7 @@ def _get_issue_properties(issue: Issue) -> DataIssue: title=issue.fields.summary, url=issue.permalink(), status=issue.fields.status.name, - priority=issue.fields.priority.name) + priority=Priority.from_jira_severity(issue.fields.priority.name)) def _generate_all_issues_url(self, table_uri: str, issues: List[DataIssue]) -> str: """ diff --git a/requirements.txt b/requirements.txt index c84eb9008..515720e1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -79,5 +79,8 @@ flask-restful==0.3.8 # SDK for JIRA jira==2.0.0 +# SDK for Asana +asana==0.10.0 + # Retrying library retrying>=1.3.3,<2.0 diff --git a/tests/unit/api/issue/test_issue.py b/tests/unit/api/issue/test_issue.py index 28d0e6bd0..3b598fd28 100644 --- a/tests/unit/api/issue/test_issue.py +++ b/tests/unit/api/issue/test_issue.py @@ -6,7 +6,7 @@ from http import HTTPStatus from amundsen_application import create_app from amundsen_application.proxy.issue_tracker_clients.issue_exceptions import IssueConfigurationException -from amundsen_application.models.data_issue import DataIssue +from amundsen_application.models.data_issue import DataIssue, Priority from amundsen_application.models.issue_results import IssueResults local_app = create_app('amundsen_application.config.TestConfig', 'tests/templates') @@ -31,7 +31,7 @@ def setUp(self) -> None: title='title', url='http://somewhere', status='open', - priority='Major') + priority=Priority.P2) self.expected_issues = IssueResults(issues=[self.mock_data_issue], total=0, all_issues_url="http://moredata") diff --git a/tests/unit/issue_tracker_clients/test_asana_client.py b/tests/unit/issue_tracker_clients/test_asana_client.py new file mode 100644 index 000000000..87b9618aa --- /dev/null +++ b/tests/unit/issue_tracker_clients/test_asana_client.py @@ -0,0 +1,41 @@ +# Copyright Contributors to the Amundsen project. +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import Mock + +import flask +import unittest +from amundsen_application.proxy.issue_tracker_clients.issue_exceptions import IssueConfigurationException +from amundsen_application.proxy.issue_tracker_clients.asana_client import AsanaClient +from amundsen_application.models.data_issue import DataIssue, Priority + +app = flask.Flask(__name__) +app.config.from_object('amundsen_application.config.TestConfig') + + +class AsanaClientTest(unittest.TestCase): + + def setUp(self) -> None: + self.mock_issue_instance = DataIssue(issue_key='key', + title='some title', + url='http://somewhere', + status='open', + priority=Priority.P2) + + @unittest.mock.patch('amundsen_application.proxy.issue_tracker_clients.asana_client.asana.Client') + def test_create_AsanaClient_validates_config(self, mock_asana_client: Mock) -> None: + with app.test_request_context(): + try: + AsanaClient( + issue_labels=[], + issue_tracker_url='', + issue_tracker_user='', + issue_tracker_password='', + issue_tracker_project_id=-1, + issue_tracker_max_results=-1) + + except IssueConfigurationException as e: + self.assertTrue(type(e), type(IssueConfigurationException)) + self.assertTrue(e, 'The following config settings must be set for Asana: ' + 'ISSUE_TRACKER_URL, ISSUE_TRACKER_USER, ISSUE_TRACKER_PASSWORD, ' + 'ISSUE_TRACKER_PROJECT_ID') diff --git a/tests/unit/issue_tracker_clients/test_jira_client.py b/tests/unit/issue_tracker_clients/test_jira_client.py index 44a23871e..b7df2cd7f 100644 --- a/tests/unit/issue_tracker_clients/test_jira_client.py +++ b/tests/unit/issue_tracker_clients/test_jira_client.py @@ -7,7 +7,7 @@ import unittest from amundsen_application.proxy.issue_tracker_clients.issue_exceptions import IssueConfigurationException from amundsen_application.proxy.issue_tracker_clients.jira_client import JiraClient, SEARCH_STUB_ALL_ISSUES -from amundsen_application.models.data_issue import DataIssue +from amundsen_application.models.data_issue import DataIssue, Priority from jira import JIRAError from typing import Dict, List @@ -41,7 +41,7 @@ def setUp(self) -> None: title='some title', url='http://somewhere', status='open', - priority='Major') + priority=Priority.P2) @unittest.mock.patch('amundsen_application.proxy.issue_tracker_clients.jira_client.JIRA') def test_create_JiraClient_validates_config(self, mock_JIRA_client: Mock) -> None: @@ -121,7 +121,7 @@ def test__generate_all_issues_url(self, mock_url_lib: Mock, mock_JIRA_client: Mo issue_tracker_password=app.config['ISSUE_TRACKER_PASSWORD'], issue_tracker_project_id=app.config['ISSUE_TRACKER_PROJECT_ID'], issue_tracker_max_results=app.config['ISSUE_TRACKER_MAX_RESULTS']) - issues = [DataIssue(issue_key='key', title='title', url='url', status='open', priority='Major')] + issues = [DataIssue(issue_key='key', title='title', url='url', status='open', priority=Priority.P2)] url = jira_client._generate_all_issues_url(table_uri="table", issues=issues) self.assertEqual(url, 'test_url/issues/?jql=test') diff --git a/tests/unit/models/test_data_issue.py b/tests/unit/models/test_data_issue.py index 5847048e8..6c66049d2 100644 --- a/tests/unit/models/test_data_issue.py +++ b/tests/unit/models/test_data_issue.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import unittest -from amundsen_application.models.data_issue import DataIssue +from amundsen_application.models.data_issue import DataIssue, Priority class DataIssueTest(unittest.TestCase): @@ -12,7 +12,7 @@ def setUp(self) -> None: self.title = 'title' self.url = 'https://place' self.status = 'open' - self.priority = 'Major' + self.priority = Priority.P2 self.maxDiff = None def test_mapping_priority(self) -> None: @@ -22,13 +22,13 @@ def test_mapping_priority(self) -> None: title=self.title, url=self.url, status=self.status, - priority=self.priority) - self.assertEqual(data_issue.priority_display_name, expected_priority_display_name) - self.assertEqual(data_issue.priority_name, expected_priority_name) - self.assertEqual(data_issue.issue_key, self.issue_key) - self.assertEqual(data_issue.title, self.title) - self.assertEqual(data_issue.url, self.url) - self.assertEqual(data_issue.status, self.status) + priority=self.priority).serialize() + self.assertEqual(data_issue['priority_display_name'], expected_priority_display_name) + self.assertEqual(data_issue['priority_name'], expected_priority_name) + self.assertEqual(data_issue['issue_key'], self.issue_key) + self.assertEqual(data_issue['title'], self.title) + self.assertEqual(data_issue['url'], self.url) + self.assertEqual(data_issue['status'], self.status) def test_mapping_priorty_missing(self) -> None: expected_priority_name = None # type: ignore @@ -37,10 +37,10 @@ def test_mapping_priorty_missing(self) -> None: title=self.title, url=self.url, status=self.status, - priority='missing') - self.assertEqual(data_issue.priority_display_name, expected_priority_display_name) - self.assertEqual(data_issue.priority_name, expected_priority_name) - self.assertEqual(data_issue.issue_key, self.issue_key) - self.assertEqual(data_issue.title, self.title) - self.assertEqual(data_issue.url, self.url) - self.assertEqual(data_issue.status, self.status) + priority=None).serialize() + self.assertEqual(data_issue['priority_display_name'], expected_priority_display_name) + self.assertEqual(data_issue['priority_name'], expected_priority_name) + self.assertEqual(data_issue['issue_key'], self.issue_key) + self.assertEqual(data_issue['title'], self.title) + self.assertEqual(data_issue['url'], self.url) + self.assertEqual(data_issue['status'], self.status)