From 2aae4971693000700fbd66d42aca86abfbc62154 Mon Sep 17 00:00:00 2001 From: sseifert Date: Sat, 7 Jan 2017 00:41:42 +0100 Subject: [PATCH 1/4] include custom fields in JQL query results convert list values to concatenated strings support definition of fieldMapping property in query which allows to rename output columns or pick specific members from dict values --- redash/query_runner/jql.py | 51 +++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index 50f110adfd..533f7ac562 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -26,15 +26,33 @@ def to_json(self): return json.dumps({'rows': self.rows, 'columns': self.columns.values()}) -def parse_issue(issue): +def parse_issue(issue, fieldMapping): result = OrderedDict() result['key'] = issue['key'] for k, v in issue['fields'].iteritems(): - if k.startswith('customfield_'): - continue - if isinstance(v, dict): + # if field mapping is defined optionally change output key and parsing rules for value + if k in fieldMapping: + mapping = fieldMapping[k] + output_key = k + if 'name' in mapping: + output_key = mapping['name'] + put_value(result, output_key, v, mapping) + + else: + put_value(result, k, v, {}) + + return result + + +def put_value(result, k, v, mapping): + if isinstance(v, dict): + if 'member' in mapping: + result[k] = v[mapping['member']] + + else: + # these special mapping rules are kept for backwards compatibility if 'key' in v: result['{}_key'.format(k)] = v['key'] if 'name' in v: @@ -45,19 +63,27 @@ def parse_issue(issue): if 'watchCount' in v: result[k] = v['watchCount'] - # elif isinstance(v, list): - # pass - else: - result[k] = v + + elif isinstance(v, list): + listValues = [] + for listItem in v: + if isinstance(listItem, dict): + if 'member' in mapping: + listValues.append(listItem[mapping['member']]) + else: + listValues.append(listItem) - return result + result[k] = ','.join(listValues) + + else: + result[k] = v -def parse_issues(data): +def parse_issues(data, fieldMapping): results = ResultSet() for issue in data['issues']: - results.add_row(parse_issue(issue)) + results.add_row(parse_issue(issue, fieldMapping)) return results @@ -109,6 +135,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') + fieldMapping = query.pop('fieldMapping', {}) if query_type == 'count': query['maxResults'] = 1 @@ -127,7 +154,7 @@ def run_query(self, query, user): if query_type == 'count': results = parse_count(data) else: - results = parse_issues(data) + results = parse_issues(data, fieldMapping) return results.to_json(), None except KeyboardInterrupt: From 689a1aac4d27f4e5baedb8ba4f4888e07a96c71b Mon Sep 17 00:00:00 2001 From: sseifert Date: Sat, 7 Jan 2017 12:15:30 +0100 Subject: [PATCH 2/4] should set nothing instead of empty string as value when no valid list items are found --- redash/query_runner/jql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index 533f7ac562..dea1426c69 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -72,8 +72,8 @@ def put_value(result, k, v, mapping): listValues.append(listItem[mapping['member']]) else: listValues.append(listItem) - - result[k] = ','.join(listValues) + if len(listValues) > 0: + result[k] = ','.join(listValues) else: result[k] = v From 9126874c879183bfe685961f65f47219ecd1fd8a Mon Sep 17 00:00:00 2001 From: sseifert Date: Mon, 9 Jan 2017 12:27:50 +0100 Subject: [PATCH 3/4] apply snake_case naming conventions --- redash/query_runner/jql.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index dea1426c69..be17a306ae 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -26,15 +26,15 @@ def to_json(self): return json.dumps({'rows': self.rows, 'columns': self.columns.values()}) -def parse_issue(issue, fieldMapping): +def parse_issue(issue, field_mapping): result = OrderedDict() result['key'] = issue['key'] for k, v in issue['fields'].iteritems(): # if field mapping is defined optionally change output key and parsing rules for value - if k in fieldMapping: - mapping = fieldMapping[k] + if k in field_mapping: + mapping = field_mapping[k] output_key = k if 'name' in mapping: output_key = mapping['name'] @@ -79,11 +79,11 @@ def put_value(result, k, v, mapping): result[k] = v -def parse_issues(data, fieldMapping): +def parse_issues(data, field_mapping): results = ResultSet() for issue in data['issues']: - results.add_row(parse_issue(issue, fieldMapping)) + results.add_row(parse_issue(issue, field_mapping)) return results @@ -135,7 +135,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') - fieldMapping = query.pop('fieldMapping', {}) + field_mapping = query.pop('fieldMapping', {}) if query_type == 'count': query['maxResults'] = 1 @@ -154,7 +154,7 @@ def run_query(self, query, user): if query_type == 'count': results = parse_count(data) else: - results = parse_issues(data, fieldMapping) + results = parse_issues(data, field_mapping) return results.to_json(), None except KeyboardInterrupt: From 187b557eeeea767e3cede203d3173ad17e1fa967 Mon Sep 17 00:00:00 2001 From: sseifert Date: Mon, 9 Jan 2017 14:37:18 +0100 Subject: [PATCH 4/4] adapt to new field mapping syntax and add unit tests --- redash/query_runner/jql.py | 127 ++++++++++++++++++++++----------- tests/query_runner/test_jql.py | 104 +++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 42 deletions(-) create mode 100644 tests/query_runner/test_jql.py diff --git a/redash/query_runner/jql.py b/redash/query_runner/jql.py index be17a306ae..2b37339c48 100644 --- a/redash/query_runner/jql.py +++ b/redash/query_runner/jql.py @@ -1,5 +1,6 @@ import json import requests +import re from collections import OrderedDict @@ -30,53 +31,55 @@ def parse_issue(issue, field_mapping): result = OrderedDict() result['key'] = issue['key'] - for k, v in issue['fields'].iteritems(): + for k, v in issue['fields'].iteritems():# + output_name = field_mapping.get_output_field_name(k) + member_names = field_mapping.get_dict_members(k) - # if field mapping is defined optionally change output key and parsing rules for value - if k in field_mapping: - mapping = field_mapping[k] - output_key = k - if 'name' in mapping: - output_key = mapping['name'] - put_value(result, output_key, v, mapping) - - else: - put_value(result, k, v, {}) - - return result + if isinstance(v, dict): + if len(member_names) > 0: + # if field mapping with dict member mappings defined get value of each member + for member_name in member_names: + if member_name in v: + result[field_mapping.get_dict_output_field_name(k,member_name)] = v[member_name] + else: + # these special mapping rules are kept for backwards compatibility + if 'key' in v: + result['{}_key'.format(output_name)] = v['key'] + if 'name' in v: + result['{}_name'.format(output_name)] = v['name'] + + if k in v: + result[output_name] = v[k] + + if 'watchCount' in v: + result[output_name] = v['watchCount'] + + elif isinstance(v, list): + if len(member_names) > 0: + # if field mapping with dict member mappings defined get value of each member + for member_name in member_names: + listValues = [] + for listItem in v: + if isinstance(listItem, dict): + if member_name in listItem: + listValues.append(listItem[member_name]) + if len(listValues) > 0: + result[field_mapping.get_dict_output_field_name(k,member_name)] = ','.join(listValues) -def put_value(result, k, v, mapping): - if isinstance(v, dict): - if 'member' in mapping: - result[k] = v[mapping['member']] + else: + # otherwise support list values only for non-dict items + listValues = [] + for listItem in v: + if not isinstance(listItem, dict): + listValues.append(listItem) + if len(listValues) > 0: + result[output_name] = ','.join(listValues) else: - # these special mapping rules are kept for backwards compatibility - if 'key' in v: - result['{}_key'.format(k)] = v['key'] - if 'name' in v: - result['{}_name'.format(k)] = v['name'] - - if k in v: - result[k] = v[k] - - if 'watchCount' in v: - result[k] = v['watchCount'] - - elif isinstance(v, list): - listValues = [] - for listItem in v: - if isinstance(listItem, dict): - if 'member' in mapping: - listValues.append(listItem[mapping['member']]) - else: - listValues.append(listItem) - if len(listValues) > 0: - result[k] = ','.join(listValues) + result[output_name] = v - else: - result[k] = v + return result def parse_issues(data, field_mapping): @@ -94,6 +97,46 @@ def parse_count(data): return results +class FieldMapping: + + def __init__(cls, query_field_mapping): + cls.mapping = [] + for k, v in query_field_mapping.iteritems(): + field_name = k + member_name = None + + # check for member name contained in field name + member_parser = re.search('(\w+)\.(\w+)', k) + if (member_parser): + field_name = member_parser.group(1) + member_name = member_parser.group(2) + + cls.mapping.append({ + 'field_name': field_name, + 'member_name': member_name, + 'output_field_name': v + }) + + def get_output_field_name(cls,field_name): + for item in cls.mapping: + if item['field_name'] == field_name and not item['member_name']: + return item['output_field_name'] + return field_name + + def get_dict_members(cls,field_name): + member_names = [] + for item in cls.mapping: + if item['field_name'] == field_name and item['member_name']: + member_names.append(item['member_name']) + return member_names + + def get_dict_output_field_name(cls,field_name, member_name): + for item in cls.mapping: + if item['field_name'] == field_name and item['member_name'] == member_name: + return item['output_field_name'] + return None + + class JiraJQL(BaseQueryRunner): noop_query = '{"queryType": "count"}' @@ -135,7 +178,7 @@ def run_query(self, query, user): try: query = json.loads(query) query_type = query.pop('queryType', 'select') - field_mapping = query.pop('fieldMapping', {}) + field_mapping = FieldMapping(query.pop('fieldMapping', {})) if query_type == 'count': query['maxResults'] = 1 diff --git a/tests/query_runner/test_jql.py b/tests/query_runner/test_jql.py new file mode 100644 index 0000000000..18428d1ab2 --- /dev/null +++ b/tests/query_runner/test_jql.py @@ -0,0 +1,104 @@ +from unittest import TestCase +from redash.query_runner.jql import FieldMapping, parse_issue + + +class TestFieldMapping(TestCase): + + def test_empty(self): + field_mapping = FieldMapping({}) + + self.assertEqual(field_mapping.get_output_field_name('field1'), 'field1') + self.assertEqual(field_mapping.get_dict_output_field_name('field1','member1'), None) + self.assertEqual(field_mapping.get_dict_members('field1'), []) + + def test_with_mappings(self): + field_mapping = FieldMapping({ + 'field1': 'output_name_1', + 'field2.member1': 'output_name_2', + 'field2.member2': 'output_name_3' + }) + + self.assertEqual(field_mapping.get_output_field_name('field1'), 'output_name_1') + self.assertEqual(field_mapping.get_dict_output_field_name('field1','member1'), None) + self.assertEqual(field_mapping.get_dict_members('field1'), []) + + self.assertEqual(field_mapping.get_output_field_name('field2'), 'field2') + self.assertEqual(field_mapping.get_dict_output_field_name('field2','member1'), 'output_name_2') + self.assertEqual(field_mapping.get_dict_output_field_name('field2','member2'), 'output_name_3') + self.assertEqual(field_mapping.get_dict_output_field_name('field2','member3'), None) + self.assertEqual(field_mapping.get_dict_members('field2'), ['member1','member2']) + + +class TestParseIssue(TestCase): + issue = { + 'key': 'KEY-1', + 'fields': { + 'string_field': 'value1', + 'int_field': 123, + 'string_list_field': ['value1','value2'], + 'dict_field': {'member1':'value1','member2': 'value2'}, + 'dict_list_field': [ + {'member1':'value1a','member2': 'value2a'}, + {'member1':'value1b','member2': 'value2b'} + ], + 'dict_legacy': {'key':'legacyKey','name':'legacyName','dict_legacy':'legacyValue'}, + 'watchers': {'watchCount':10} + } + } + + def test_no_mapping(self): + result = parse_issue(self.issue, FieldMapping({})) + + self.assertEqual(result['key'], 'KEY-1') + self.assertEqual(result['string_field'], 'value1') + self.assertEqual(result['int_field'], 123) + self.assertEqual(result['string_list_field'], 'value1,value2') + self.assertEqual('dict_field' in result, False) + self.assertEqual('dict_list_field' in result, False) + self.assertEqual(result['dict_legacy'], 'legacyValue') + self.assertEqual(result['dict_legacy_key'], 'legacyKey') + self.assertEqual(result['dict_legacy_name'], 'legacyName') + self.assertEqual(result['watchers'], 10) + + def test_mapping(self): + result = parse_issue(self.issue, FieldMapping({ + 'string_field': 'string_output_field', + 'string_list_field': 'string_output_list_field', + 'dict_field.member1': 'dict_field_1', + 'dict_field.member2': 'dict_field_2', + 'dict_list_field.member1': 'dict_list_field_1', + 'dict_legacy.key': 'dict_legacy', + 'watchers.watchCount': 'watchCount', + })) + + self.assertEqual(result['key'], 'KEY-1') + self.assertEqual(result['string_output_field'], 'value1') + self.assertEqual(result['int_field'], 123) + self.assertEqual(result['string_output_list_field'], 'value1,value2') + self.assertEqual(result['dict_field_1'], 'value1') + self.assertEqual(result['dict_field_2'], 'value2') + self.assertEqual(result['dict_list_field_1'], 'value1a,value1b') + self.assertEqual(result['dict_legacy'], 'legacyKey') + self.assertEqual('dict_legacy_key' in result, False) + self.assertEqual('dict_legacy_name' in result, False) + self.assertEqual('watchers' in result, False) + self.assertEqual(result['watchCount'], 10) + + + def test_mapping_nonexisting_field(self): + result = parse_issue(self.issue, FieldMapping({ + 'non_existing_field': 'output_name1', + 'dict_field.non_existing_member': 'output_name2', + 'dict_list_field.non_existing_member': 'output_name3' + })) + + self.assertEqual(result['key'], 'KEY-1') + self.assertEqual(result['string_field'], 'value1') + self.assertEqual(result['int_field'], 123) + self.assertEqual(result['string_list_field'], 'value1,value2') + self.assertEqual('dict_field' in result, False) + self.assertEqual('dict_list_field' in result, False) + self.assertEqual(result['dict_legacy'], 'legacyValue') + self.assertEqual(result['dict_legacy_key'], 'legacyKey') + self.assertEqual(result['dict_legacy_name'], 'legacyName') + self.assertEqual(result['watchers'], 10)