From 669c9bf033d1da9421bc1ad531108b8e1c46899f Mon Sep 17 00:00:00 2001 From: sshuster Date: Tue, 28 Jan 2020 07:33:48 -0800 Subject: [PATCH 1/7] Added programmatic description fields Added an abstract method for getting programmatic descriptions that is currently not being used. Modified _exec_table_query to return prog_descriptions as part of the TableData call Increased amundsen-common to latest version Adding a couple of todo statements --- metadata_service/api/table.py | 9 +++++- metadata_service/proxy/base_proxy.py | 6 ++++ metadata_service/proxy/neo4j_proxy.py | 40 +++++++++++++++++++++++---- requirements.txt | 2 +- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/metadata_service/api/table.py b/metadata_service/api/table.py index a714a182..29b745b4 100644 --- a/metadata_service/api/table.py +++ b/metadata_service/api/table.py @@ -63,6 +63,12 @@ 'source': fields.String } +programmatic_description_fields = { + 'source': fields.String, + 'text': fields.String, + 'is_editable': fields.Boolean +} + table_detail_fields = { 'database': fields.String, 'cluster': fields.String, @@ -82,7 +88,8 @@ 'table_writer': fields.Nested(table_writer_fields), # Optional 'last_updated_timestamp': fields.Integer, # Optional 'source': fields.Nested(source_fields), # Optional - 'is_view': fields.Boolean # Optional + 'is_view': fields.Boolean, # Optional + 'programmatic_descriptions': fields.List(fields.Nested(programmatic_description_fields)) } diff --git a/metadata_service/proxy/base_proxy.py b/metadata_service/proxy/base_proxy.py index 378293dc..7850e6e1 100644 --- a/metadata_service/proxy/base_proxy.py +++ b/metadata_service/proxy/base_proxy.py @@ -38,6 +38,12 @@ def get_table_description(self, *, table_uri: str) -> Union[str, None]: pass + #TODO determine if this is needed. If so, needs to be implemented + @abstractmethod + def get_programmatic_descriptions(self, *, + table_uri: str) -> Union[List[str], None]: + pass + @abstractmethod def put_table_description(self, *, table_uri: str, diff --git a/metadata_service/proxy/neo4j_proxy.py b/metadata_service/proxy/neo4j_proxy.py index 8348edea..f1a20a19 100644 --- a/metadata_service/proxy/neo4j_proxy.py +++ b/metadata_service/proxy/neo4j_proxy.py @@ -7,7 +7,7 @@ from amundsen_common.models.table import (Application, Column, Reader, Source, Statistics, Table, Tag, User, - Watermark) + Watermark, ProgrammaticDescription) from amundsen_common.models.user import User as UserEntity from beaker.cache import CacheManager from beaker.util import parse_cache_config_options @@ -67,7 +67,7 @@ def get_table(self, *, table_uri: str) -> Table: readers = self._exec_usage_query(table_uri) - wmk_results, table_writer, timestamp_value, owners, tags, source, badges = self._exec_table_query(table_uri) + wmk_results, table_writer, timestamp_value, owners, tags, source, badges, prog_descs = self._exec_table_query(table_uri) table = Table(database=last_neo4j_record['db']['name'], cluster=last_neo4j_record['clstr']['name'], @@ -83,7 +83,9 @@ def get_table(self, *, table_uri: str) -> Table: table_writer=table_writer, last_updated_timestamp=timestamp_value, source=source, - is_view=self._safe_get(last_neo4j_record, 'tbl', 'is_view')) + is_view=self._safe_get(last_neo4j_record, 'tbl', 'is_view'), + programmatic_descriptions = prog_descs + ) return table @@ -168,13 +170,15 @@ def _exec_table_query(self, table_uri: str) -> Tuple: OPTIONAL MATCH (tbl)-[:TAGGED_BY]->(tag:Tag{tag_type: $tag_normal_type}) OPTIONAL MATCH (tbl)-[:TAGGED_BY]->(badge:Tag{tag_type: $tag_badge_type}) OPTIONAL MATCH (tbl)-[:SOURCE]->(src:Source) + OPTIONAL MATCH (tbl)-[:DESCRIPTION]->(prog_descriptions:Programmatic_Description) RETURN collect(distinct wmk) as wmk_records, application, t.last_updated_timestamp as last_updated_timestamp, collect(distinct owner) as owner_records, collect(distinct tag) as tag_records, collect(distinct badge) as badge_records, - src + src, + collect(distinct prog_descriptions) as prog_descriptions """) table_records = self._execute_cypher_query(statement=table_level_query, @@ -236,7 +240,15 @@ def _exec_table_query(self, table_uri: str) -> Tuple: src = Source(source_type=table_records['src']['source_type'], source=table_records['src']['source']) - return wmk_results, table_writer, timestamp_value, owner_record, tags, src, badges + # TODO consider ordering prog descriptions here. + prog_descriptions = [] + for prog_description in table_records['prog_descriptions']: + LOGGER.info(prog_description) + prog_descriptions.append(ProgrammaticDescription(source = prog_description['description_source'], + text = prog_description['description'], + is_editable = prog_description['description_editable'])) + + return wmk_results, table_writer, timestamp_value, owner_record, tags, src, badges, prog_descriptions @no_type_check def _safe_get(self, dct, *keys): @@ -293,6 +305,24 @@ def get_table_description(self, *, return table_description + @timer_with_counter + def get_programmatic_descriptions(self, *, + table_uri: str) -> Union[List[str], None]: + table_description_query = textwrap.dedent(""" + MATCH (tbl:Table {key: $tbl_key})-[:DESCRIPTION]->(d:Programmatic_Description) + RETURN d.description AS description; + """) + + result = self._execute_cypher_query(statement=table_description_query, + param_dict={'tbl_key': table_uri}) + + #TODO handle multiple programmatics + table_descrpt = result.single() + + table_description = table_descrpt['description'] if table_descrpt else None + + return table_description + @timer_with_counter def put_table_description(self, *, table_uri: str, diff --git a/requirements.txt b/requirements.txt index 1acc07ee..31cf019b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ pytest-mock==1.1 typing==3.6.4 -amundsen-common==0.1.3 +amundsen-common==0.1.5 flasgger==0.9.3 Flask-RESTful==0.3.6 Flask==1.0.2 From 57fad2eb3f0f027073d723f442c7f1135e9d17ba Mon Sep 17 00:00:00 2001 From: sshuster Date: Thu, 30 Jan 2020 17:30:24 -0800 Subject: [PATCH 2/7] updating to source_id adding a couple of tests sorting alphabetically by default --- metadata_service/api/table.py | 2 +- metadata_service/proxy/neo4j_proxy.py | 11 ++++++----- tests/unit/proxy/test_neo4j_proxy.py | 28 +++++++++++++++++++++++---- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/metadata_service/api/table.py b/metadata_service/api/table.py index 29b745b4..230214d8 100644 --- a/metadata_service/api/table.py +++ b/metadata_service/api/table.py @@ -64,7 +64,7 @@ } programmatic_description_fields = { - 'source': fields.String, + 'source_id': fields.String, 'text': fields.String, 'is_editable': fields.Boolean } diff --git a/metadata_service/proxy/neo4j_proxy.py b/metadata_service/proxy/neo4j_proxy.py index f1a20a19..b9d4d0ec 100644 --- a/metadata_service/proxy/neo4j_proxy.py +++ b/metadata_service/proxy/neo4j_proxy.py @@ -242,12 +242,13 @@ def _exec_table_query(self, table_uri: str) -> Tuple: # TODO consider ordering prog descriptions here. prog_descriptions = [] - for prog_description in table_records['prog_descriptions']: + for prog_description in table_records.get('prog_descriptions', []): LOGGER.info(prog_description) - prog_descriptions.append(ProgrammaticDescription(source = prog_description['description_source'], - text = prog_description['description'], - is_editable = prog_description['description_editable'])) - + source_id = prog_description['description_source'] + prog_descriptions.append(ProgrammaticDescription(source_id = source_id, + text = prog_description['description'], + is_editable = prog_description['description_editable'])) + prog_descriptions.sort(key = lambda x: x.source_id) return wmk_results, table_writer, timestamp_value, owner_record, tags, src, badges, prog_descriptions @no_type_check diff --git a/tests/unit/proxy/test_neo4j_proxy.py b/tests/unit/proxy/test_neo4j_proxy.py index 6372cda2..63f533d2 100644 --- a/tests/unit/proxy/test_neo4j_proxy.py +++ b/tests/unit/proxy/test_neo4j_proxy.py @@ -5,7 +5,7 @@ from amundsen_common.models.table import (Application, Column, Source, Statistics, Table, Tag, User, - Watermark) + Watermark, ProgrammaticDescription) from amundsen_common.models.user import UserSchema from mock import MagicMock, patch @@ -96,7 +96,19 @@ def setUp(self) -> None: 'source': '/source_file_loc', 'key': 'some key', 'source_type': 'github' - } + }, + 'prog_descriptions': [ + { + 'description_source': 's3_crawler', + 'description': 'Test Test Test', + 'description_editable': 'false' + }, + { + 'description_source': 'quality_report', + 'description': 'Test Test', + 'description_editable': 'true' + } + ] } table_writer = { @@ -158,7 +170,11 @@ def test_get_table(self) -> None: last_updated_timestamp=1, source=Source(source='/source_file_loc', source_type='github'), - is_view=False) + is_view=False, + programmatic_descriptions= [ + ProgrammaticDescription(source_id='quality_report', text = 'Test Test', is_editable = 'true'), + ProgrammaticDescription(source_id='s3_crawler', text = 'Test Test Test', is_editable = 'false') + ]) self.assertEqual(str(expected), str(table)) @@ -203,7 +219,11 @@ def test_get_table_view_only(self) -> None: last_updated_timestamp=1, source=Source(source='/source_file_loc', source_type='github'), - is_view=True) + is_view=True, + programmatic_descriptions= [ + ProgrammaticDescription(source_id='quality_report', text = 'Test Test', is_editable = 'true'), + ProgrammaticDescription(source_id='s3_crawler', text = 'Test Test Test', is_editable = 'false') + ]) self.assertEqual(str(expected), str(table)) From ee76f1a53be2378844d5c210a97876f342d7852c Mon Sep 17 00:00:00 2001 From: sshuster Date: Mon, 3 Feb 2020 07:56:33 -0800 Subject: [PATCH 3/7] Fixing flake8 style errors --- metadata_service/proxy/base_proxy.py | 6 ----- metadata_service/proxy/neo4j_proxy.py | 39 +++++++++------------------ requirements.txt | 2 +- tests/unit/proxy/test_neo4j_proxy.py | 16 ++++++----- 4 files changed, 24 insertions(+), 39 deletions(-) diff --git a/metadata_service/proxy/base_proxy.py b/metadata_service/proxy/base_proxy.py index 7850e6e1..378293dc 100644 --- a/metadata_service/proxy/base_proxy.py +++ b/metadata_service/proxy/base_proxy.py @@ -38,12 +38,6 @@ def get_table_description(self, *, table_uri: str) -> Union[str, None]: pass - #TODO determine if this is needed. If so, needs to be implemented - @abstractmethod - def get_programmatic_descriptions(self, *, - table_uri: str) -> Union[List[str], None]: - pass - @abstractmethod def put_table_description(self, *, table_uri: str, diff --git a/metadata_service/proxy/neo4j_proxy.py b/metadata_service/proxy/neo4j_proxy.py index b9d4d0ec..218b072d 100644 --- a/metadata_service/proxy/neo4j_proxy.py +++ b/metadata_service/proxy/neo4j_proxy.py @@ -67,7 +67,8 @@ def get_table(self, *, table_uri: str) -> Table: readers = self._exec_usage_query(table_uri) - wmk_results, table_writer, timestamp_value, owners, tags, source, badges, prog_descs = self._exec_table_query(table_uri) + wmk_results, table_writer, timestamp_value, owners, tags, source, badges, prog_descs = \ + self._exec_table_query(table_uri) table = Table(database=last_neo4j_record['db']['name'], cluster=last_neo4j_record['clstr']['name'], @@ -84,7 +85,7 @@ def get_table(self, *, table_uri: str) -> Table: last_updated_timestamp=timestamp_value, source=source, is_view=self._safe_get(last_neo4j_record, 'tbl', 'is_view'), - programmatic_descriptions = prog_descs + programmatic_descriptions=prog_descs ) return table @@ -240,16 +241,20 @@ def _exec_table_query(self, table_uri: str) -> Tuple: src = Source(source_type=table_records['src']['source_type'], source=table_records['src']['source']) - # TODO consider ordering prog descriptions here. + prog_descriptions = self._extract_programmatic_descriptions_from_query(table_records) + + return wmk_results, table_writer, timestamp_value, owner_record, tags, src, badges, prog_descriptions + + def _extract_programmatic_descriptions_from_query(self, table_records: dict) -> list: prog_descriptions = [] for prog_description in table_records.get('prog_descriptions', []): LOGGER.info(prog_description) source_id = prog_description['description_source'] - prog_descriptions.append(ProgrammaticDescription(source_id = source_id, - text = prog_description['description'], - is_editable = prog_description['description_editable'])) - prog_descriptions.sort(key = lambda x: x.source_id) - return wmk_results, table_writer, timestamp_value, owner_record, tags, src, badges, prog_descriptions + prog_descriptions.append(ProgrammaticDescription(source_id=source_id, + text=prog_description['description'], + is_editable=prog_description['description_editable'])) + prog_descriptions.sort(key=lambda x: x.source_id) + return prog_descriptions @no_type_check def _safe_get(self, dct, *keys): @@ -306,24 +311,6 @@ def get_table_description(self, *, return table_description - @timer_with_counter - def get_programmatic_descriptions(self, *, - table_uri: str) -> Union[List[str], None]: - table_description_query = textwrap.dedent(""" - MATCH (tbl:Table {key: $tbl_key})-[:DESCRIPTION]->(d:Programmatic_Description) - RETURN d.description AS description; - """) - - result = self._execute_cypher_query(statement=table_description_query, - param_dict={'tbl_key': table_uri}) - - #TODO handle multiple programmatics - table_descrpt = result.single() - - table_description = table_descrpt['description'] if table_descrpt else None - - return table_description - @timer_with_counter def put_table_description(self, *, table_uri: str, diff --git a/requirements.txt b/requirements.txt index 31cf019b..e8cf8cd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ pytest-mock==1.1 typing==3.6.4 -amundsen-common==0.1.5 +amundsen-common==0.1.6 flasgger==0.9.3 Flask-RESTful==0.3.6 Flask==1.0.2 diff --git a/tests/unit/proxy/test_neo4j_proxy.py b/tests/unit/proxy/test_neo4j_proxy.py index 63f533d2..420b9a5e 100644 --- a/tests/unit/proxy/test_neo4j_proxy.py +++ b/tests/unit/proxy/test_neo4j_proxy.py @@ -171,9 +171,11 @@ def test_get_table(self) -> None: source=Source(source='/source_file_loc', source_type='github'), is_view=False, - programmatic_descriptions= [ - ProgrammaticDescription(source_id='quality_report', text = 'Test Test', is_editable = 'true'), - ProgrammaticDescription(source_id='s3_crawler', text = 'Test Test Test', is_editable = 'false') + programmatic_descriptions=[ + ProgrammaticDescription(source_id='quality_report', + text='Test Test', is_editable='true'), + ProgrammaticDescription(source_id='s3_crawler', + text='Test Test Test', is_editable='false') ]) self.assertEqual(str(expected), str(table)) @@ -220,9 +222,11 @@ def test_get_table_view_only(self) -> None: source=Source(source='/source_file_loc', source_type='github'), is_view=True, - programmatic_descriptions= [ - ProgrammaticDescription(source_id='quality_report', text = 'Test Test', is_editable = 'true'), - ProgrammaticDescription(source_id='s3_crawler', text = 'Test Test Test', is_editable = 'false') + programmatic_descriptions=[ + ProgrammaticDescription(source_id='quality_report', + text='Test Test', is_editable='true'), + ProgrammaticDescription(source_id='s3_crawler', + text='Test Test Test', is_editable='false') ]) self.assertEqual(str(expected), str(table)) From 5bb3b6974fb994db93578d18a7245a58f8e0869e Mon Sep 17 00:00:00 2001 From: sshuster Date: Mon, 3 Feb 2020 17:53:53 -0800 Subject: [PATCH 4/7] No longer supporting is_editable property --- metadata_service/proxy/neo4j_proxy.py | 3 +-- tests/unit/api/table/test_table_detail_api.py | 8 +++++--- tests/unit/proxy/test_neo4j_proxy.py | 14 ++++++-------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/metadata_service/proxy/neo4j_proxy.py b/metadata_service/proxy/neo4j_proxy.py index 218b072d..e9ff9c7f 100644 --- a/metadata_service/proxy/neo4j_proxy.py +++ b/metadata_service/proxy/neo4j_proxy.py @@ -251,8 +251,7 @@ def _extract_programmatic_descriptions_from_query(self, table_records: dict) -> LOGGER.info(prog_description) source_id = prog_description['description_source'] prog_descriptions.append(ProgrammaticDescription(source_id=source_id, - text=prog_description['description'], - is_editable=prog_description['description_editable'])) + text=prog_description['description'])) prog_descriptions.sort(key=lambda x: x.source_id) return prog_descriptions diff --git a/tests/unit/api/table/test_table_detail_api.py b/tests/unit/api/table/test_table_detail_api.py index 5e9818d4..4b1bd8eb 100644 --- a/tests/unit/api/table/test_table_detail_api.py +++ b/tests/unit/api/table/test_table_detail_api.py @@ -37,7 +37,8 @@ 'col_type': 'String', 'sort_order': 0, 'stats': STATS - }] + }], + 'programmatic_descriptions': [] } API_RESPONSE = { @@ -54,7 +55,8 @@ 'type': 'String', 'sort_order': 0, 'stats': STATS - }] + }], + 'programmatic_descriptions': [] } @@ -63,7 +65,7 @@ def test_should_get_column_details(self) -> None: self.mock_proxy.get_table.return_value = QUERY_RESPONSE response = self.app.test_client().get(f'/table/{TABLE_URI}') - + print(response.json) self.assertEqual(response.json, API_RESPONSE) self.assertEqual(response.status_code, HTTPStatus.OK) self.mock_proxy.get_table.assert_called_with(table_uri=TABLE_URI) diff --git a/tests/unit/proxy/test_neo4j_proxy.py b/tests/unit/proxy/test_neo4j_proxy.py index 420b9a5e..ae984e86 100644 --- a/tests/unit/proxy/test_neo4j_proxy.py +++ b/tests/unit/proxy/test_neo4j_proxy.py @@ -100,13 +100,11 @@ def setUp(self) -> None: 'prog_descriptions': [ { 'description_source': 's3_crawler', - 'description': 'Test Test Test', - 'description_editable': 'false' + 'description': 'Test Test Test' }, { 'description_source': 'quality_report', - 'description': 'Test Test', - 'description_editable': 'true' + 'description': 'Test Test' } ] } @@ -173,9 +171,9 @@ def test_get_table(self) -> None: is_view=False, programmatic_descriptions=[ ProgrammaticDescription(source_id='quality_report', - text='Test Test', is_editable='true'), + text='Test Test'), ProgrammaticDescription(source_id='s3_crawler', - text='Test Test Test', is_editable='false') + text='Test Test Test') ]) self.assertEqual(str(expected), str(table)) @@ -224,9 +222,9 @@ def test_get_table_view_only(self) -> None: is_view=True, programmatic_descriptions=[ ProgrammaticDescription(source_id='quality_report', - text='Test Test', is_editable='true'), + text='Test Test'), ProgrammaticDescription(source_id='s3_crawler', - text='Test Test Test', is_editable='false') + text='Test Test Test') ]) self.assertEqual(str(expected), str(table)) From 237e2e2eca93e794ab28509e9ef4160a7baada26 Mon Sep 17 00:00:00 2001 From: sshuster Date: Wed, 12 Feb 2020 11:50:55 -0800 Subject: [PATCH 5/7] modifying source_id to source increasing amundsen-common dependency to 0.2.1 --- metadata_service/api/table.py | 5 ++--- metadata_service/proxy/neo4j_proxy.py | 6 +++--- requirements.txt | 2 +- tests/unit/proxy/test_neo4j_proxy.py | 8 ++++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/metadata_service/api/table.py b/metadata_service/api/table.py index bf848ed0..ce29dc7e 100644 --- a/metadata_service/api/table.py +++ b/metadata_service/api/table.py @@ -64,9 +64,8 @@ } programmatic_description_fields = { - 'source_id': fields.String, - 'text': fields.String, - 'is_editable': fields.Boolean + 'source': fields.String, + 'text': fields.String } table_detail_fields = { diff --git a/metadata_service/proxy/neo4j_proxy.py b/metadata_service/proxy/neo4j_proxy.py index 9efc0278..759724b5 100644 --- a/metadata_service/proxy/neo4j_proxy.py +++ b/metadata_service/proxy/neo4j_proxy.py @@ -249,10 +249,10 @@ def _extract_programmatic_descriptions_from_query(self, table_records: dict) -> prog_descriptions = [] for prog_description in table_records.get('prog_descriptions', []): LOGGER.info(prog_description) - source_id = prog_description['description_source'] - prog_descriptions.append(ProgrammaticDescription(source_id=source_id, + source = prog_description['description_source'] + prog_descriptions.append(ProgrammaticDescription(source=source, text=prog_description['description'])) - prog_descriptions.sort(key=lambda x: x.source_id) + prog_descriptions.sort(key=lambda x: x.source) return prog_descriptions @no_type_check diff --git a/requirements.txt b/requirements.txt index e8cf8cd0..35af9653 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ pytest-mock==1.1 typing==3.6.4 -amundsen-common==0.1.6 +amundsen-common==0.2.1 flasgger==0.9.3 Flask-RESTful==0.3.6 Flask==1.0.2 diff --git a/tests/unit/proxy/test_neo4j_proxy.py b/tests/unit/proxy/test_neo4j_proxy.py index 602666ec..c625aef0 100644 --- a/tests/unit/proxy/test_neo4j_proxy.py +++ b/tests/unit/proxy/test_neo4j_proxy.py @@ -170,9 +170,9 @@ def test_get_table(self) -> None: source_type='github'), is_view=False, programmatic_descriptions=[ - ProgrammaticDescription(source_id='quality_report', + ProgrammaticDescription(source='quality_report', text='Test Test'), - ProgrammaticDescription(source_id='s3_crawler', + ProgrammaticDescription(source='s3_crawler', text='Test Test Test') ]) @@ -221,9 +221,9 @@ def test_get_table_view_only(self) -> None: source_type='github'), is_view=True, programmatic_descriptions=[ - ProgrammaticDescription(source_id='quality_report', + ProgrammaticDescription(source='quality_report', text='Test Test'), - ProgrammaticDescription(source_id='s3_crawler', + ProgrammaticDescription(source='s3_crawler', text='Test Test Test') ]) From bc27e714cf944febbab218a68952c969cf2df2e3 Mon Sep 17 00:00:00 2001 From: sshuster Date: Wed, 12 Feb 2020 17:11:07 -0800 Subject: [PATCH 6/7] Fixing comments per code review. Small refactor, removing unneeded logging statements. --- metadata_service/proxy/neo4j_proxy.py | 15 +++++++++------ tests/unit/api/table/test_table_detail_api.py | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/metadata_service/proxy/neo4j_proxy.py b/metadata_service/proxy/neo4j_proxy.py index 759724b5..10dd47c8 100644 --- a/metadata_service/proxy/neo4j_proxy.py +++ b/metadata_service/proxy/neo4j_proxy.py @@ -241,17 +241,20 @@ def _exec_table_query(self, table_uri: str) -> Tuple: src = Source(source_type=table_records['src']['source_type'], source=table_records['src']['source']) - prog_descriptions = self._extract_programmatic_descriptions_from_query(table_records) + prog_descriptions = self._extract_programmatic_descriptions_from_query( + table_records.get('prog_descriptions', []) + ) return wmk_results, table_writer, timestamp_value, owner_record, tags, src, badges, prog_descriptions - def _extract_programmatic_descriptions_from_query(self, table_records: dict) -> list: + def _extract_programmatic_descriptions_from_query(self, raw_prog_descriptions: dict) -> list: prog_descriptions = [] - for prog_description in table_records.get('prog_descriptions', []): - LOGGER.info(prog_description) + for prog_description in raw_prog_descriptions: source = prog_description['description_source'] - prog_descriptions.append(ProgrammaticDescription(source=source, - text=prog_description['description'])) + if source is None: + LOGGER.error("A programmatic description with no source was found... skipping.") + else: + prog_descriptions.append(ProgrammaticDescription(source=source, text=prog_description['description'])) prog_descriptions.sort(key=lambda x: x.source) return prog_descriptions diff --git a/tests/unit/api/table/test_table_detail_api.py b/tests/unit/api/table/test_table_detail_api.py index 4b1bd8eb..ac1a989c 100644 --- a/tests/unit/api/table/test_table_detail_api.py +++ b/tests/unit/api/table/test_table_detail_api.py @@ -65,7 +65,6 @@ def test_should_get_column_details(self) -> None: self.mock_proxy.get_table.return_value = QUERY_RESPONSE response = self.app.test_client().get(f'/table/{TABLE_URI}') - print(response.json) self.assertEqual(response.json, API_RESPONSE) self.assertEqual(response.status_code, HTTPStatus.OK) self.mock_proxy.get_table.assert_called_with(table_uri=TABLE_URI) From 3e3f42bb06199890e35020e8aecb2f1728507701 Mon Sep 17 00:00:00 2001 From: sshuster Date: Wed, 12 Feb 2020 17:23:37 -0800 Subject: [PATCH 7/7] increasing minor version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b886ccd2..69363f40 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -__version__ = '1.1.9' +__version__ = '1.2.1' requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')