diff --git a/home/forms/search.py b/home/forms/search.py index 51001651..299bc6ff 100644 --- a/home/forms/search.py +++ b/home/forms/search.py @@ -1,8 +1,8 @@ from copy import deepcopy from urllib.parse import urlencode +from data_platform_catalogue.entities import FindMoJdataEntityType from data_platform_catalogue.search_types import DomainOption -from data_platform_catalogue.entities import EntityTypes from django import forms from django.utils.translation import gettext as _ @@ -38,7 +38,7 @@ def get_entity_types(): return sorted( [ (entity.name, entity.value) - for entity in EntityTypes + for entity in FindMoJdataEntityType if entity.name != "GLOSSARY_TERM" ] ) diff --git a/home/service/details.py b/home/service/details.py index 0eda852c..25023664 100644 --- a/home/service/details.py +++ b/home/service/details.py @@ -1,7 +1,10 @@ import os from urllib.parse import urlsplit -from data_platform_catalogue.entities import EntityRef, EntityTypes, RelationshipType +from data_platform_catalogue.entities import ( + EntityRef, RelationshipType, DatabaseEntityMapping, + DashboardEntityMapping, PublicationCollectionEntityMapper, PublicationDatasetEntityMapping +) from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.core.validators import URLValidator from django.utils.translation import gettext as _ @@ -114,7 +117,7 @@ def _get_context(self): "entity": self.table_metadata, "entity_type": "Table", "parent_entity": self.parent_entity, - "parent_type": EntityTypes.DATABASE.name.lower(), + "parent_type": DatabaseEntityMapping.url_formatted, "h1_value": self.table_metadata.name, "has_lineage": self.has_lineage(), "lineage_url": f"{split_datahub_url.scheme}://{split_datahub_url.netloc}/dataset/{self.table_metadata.urn}/Lineage?is_lineage_mode=true&", # noqa: E501 @@ -161,7 +164,7 @@ def _get_context(self): self.chart_metadata.platform.display_name ), "parent_entity": self.parent_entity, - "parent_type": EntityTypes.DASHBOARD.name.lower(), + "parent_type": DashboardEntityMapping.url_formatted, "h1_value": self.chart_metadata.name, "is_access_requirements_a_url": is_access_requirements_a_url( self.chart_metadata.custom_properties.access_information.dc_access_requirements @@ -219,7 +222,7 @@ def __init__(self, urn: str): def _get_context(self): context = { "entity": self.publication_collection_metadata, - "entity_type": EntityTypes.PUBLICATION_COLLECTION.value, + "entity_type": PublicationCollectionEntityMapper.find_moj_data_type.value, "platform_name": friendly_platform_name( self.publication_collection_metadata.platform.display_name ), @@ -258,12 +261,12 @@ def _get_context(self): return { "entity": self.publication_dataset_metadata, - "entity_type": EntityTypes.PUBLICATION_DATASET.value, + "entity_type": PublicationDatasetEntityMapping.find_moj_data_type.value, "parent_entity": self.parent_entity, "platform_name": friendly_platform_name( self.publication_dataset_metadata.platform.display_name ), - "parent_type": EntityTypes.PUBLICATION_COLLECTION.name.lower(), + "parent_type": DatabaseEntityMapping.url_formatted, "h1_value": self.publication_dataset_metadata.name, # noqa: E501 "is_access_requirements_a_url": is_access_requirements_a_url( diff --git a/home/service/search.py b/home/service/search.py index 0d109efe..a29f3165 100644 --- a/home/service/search.py +++ b/home/service/search.py @@ -2,7 +2,7 @@ from copy import deepcopy from typing import Any -from data_platform_catalogue.entities import EntityTypes +from data_platform_catalogue.entities import FindMoJdataEntityMapper, Mappers from data_platform_catalogue.search_types import ( DomainOption, MultiSelectFilter, @@ -45,19 +45,16 @@ def _build_custom_property_filter( ) -> list[str]: return [f"{filter_param}{filter_value}" for filter_value in filter_value_list] - def _build_entity_types(self, entity_types: list[str]) -> tuple[EntityTypes, ...]: + def _build_entity_types(self, entity_types: list[str]) -> tuple[FindMoJdataEntityMapper, ...]: default_entities = tuple( - entity - for entity in EntityTypes - if entity.name != "GLOSSARY_TERM" + Mapper + for Mapper in Mappers + if Mapper.datahub_type.value != "GLOSSARY_TERM" ) - chosen_entities = ( - tuple( - EntityTypes[entity] - for entity in entity_types - ) - if entity_types - else None + chosen_entities = tuple( + Mapper + for Mapper in Mappers + if Mapper.find_moj_data_type.name in entity_types ) return chosen_entities if chosen_entities else default_entities diff --git a/home/views.py b/home/views.py index bf7024ba..5c56ac01 100644 --- a/home/views.py +++ b/home/views.py @@ -3,7 +3,14 @@ from urllib.parse import urlparse from data_platform_catalogue.client.exceptions import EntityDoesNotExist -from data_platform_catalogue.entities import EntityTypes +from data_platform_catalogue.entities import ( + ChartEntityMapping, + DashboardEntityMapping, + DatabaseEntityMapping, + PublicationCollectionEntityMapper, + PublicationDatasetEntityMapping, + TableEntityMapping, +) from data_platform_catalogue.search_types import DomainOption from django.conf import settings from django.http import Http404, HttpResponse, HttpResponseBadRequest @@ -31,12 +38,12 @@ from home.service.search import SearchService type_details_map = { - EntityTypes.TABLE.url_formatted: DatasetDetailsService, - EntityTypes.DATABASE.url_formatted: DatabaseDetailsService, - EntityTypes.CHART.url_formatted: ChartDetailsService, - EntityTypes.DASHBOARD.url_formatted: DashboardDetailsService, - EntityTypes.PUBLICATION_COLLECTION.url_formatted: PublicationCollectionDetailsService, - EntityTypes.PUBLICATION_DATASET.url_formatted: PublicationDatasetDetailsService, + TableEntityMapping.url_formatted: DatasetDetailsService, + DatabaseEntityMapping.url_formatted: DatabaseDetailsService, + ChartEntityMapping.url_formatted: ChartDetailsService, + DashboardEntityMapping.url_formatted: DashboardDetailsService, + PublicationCollectionEntityMapper.url_formatted: PublicationCollectionDetailsService, + PublicationDatasetEntityMapping.url_formatted: PublicationDatasetDetailsService } @@ -67,11 +74,11 @@ def details_view(request, result_type, urn): @cache_control(max_age=300, private=True) def details_view_csv(request, result_type, urn) -> HttpResponse: match result_type: - case EntityTypes.TABLE.url_formatted: + case TableEntityMapping.url_formatted: csv_formatter = DatasetDetailsCsvFormatter(DatasetDetailsService(urn)) - case EntityTypes.DATABASE.url_formatted: + case DatabaseEntityMapping.url_formatted: csv_formatter = DatabaseDetailsCsvFormatter(DatabaseDetailsService(urn)) - case EntityTypes.DASHBOARD.url_formatted: + case DashboardEntityMapping.url_formatted: csv_formatter = DashboardDetailsCsvFormatter(DashboardDetailsService(urn)) case _: logging.error("Invalid result type for csv details view %s", result_type) diff --git a/lib/datahub-client/data_platform_catalogue/client/datahub_client.py b/lib/datahub-client/data_platform_catalogue/client/datahub_client.py index 28c92e2c..7c83ecd8 100644 --- a/lib/datahub-client/data_platform_catalogue/client/datahub_client.py +++ b/lib/datahub-client/data_platform_catalogue/client/datahub_client.py @@ -63,7 +63,10 @@ Database, EntityRef, EntitySummary, - EntityTypes, + FindMoJdataEntityMapper, + TableEntityMapping, + ChartEntityMapping, + DatabaseEntityMapping, Governance, PublicationCollection, PublicationDataset, @@ -203,10 +206,10 @@ def search( query: str = "*", count: int = 20, page: str | None = None, - result_types: Sequence[EntityTypes] = ( - EntityTypes.TABLE, - EntityTypes.CHART, - EntityTypes.DATABASE, + result_types: Sequence[FindMoJdataEntityMapper] = ( + TableEntityMapping, + ChartEntityMapping, + DatabaseEntityMapping, ), filters: Sequence[MultiSelectFilter] | None = None, sort: SortOption | None = None, @@ -395,7 +398,7 @@ def get_publication_collection_details(self, urn: str) -> PublicationCollection: child_relations = parse_relations( relationship_type=RelationshipType.CHILD, relations_list=[response["relationships"]], - entity_type_of_relations=EntityTypes.PUBLICATION_DATASET.url_formatted, + entity_type_of_relations=PublicationDatasetEntityMapping.url_formatted, ) relations_to_display = self.list_relations_to_display(child_relations) diff --git a/lib/datahub-client/data_platform_catalogue/client/search/search_client.py b/lib/datahub-client/data_platform_catalogue/client/search/search_client.py index 00223a34..448cec5f 100644 --- a/lib/datahub-client/data_platform_catalogue/client/search/search_client.py +++ b/lib/datahub-client/data_platform_catalogue/client/search/search_client.py @@ -21,8 +21,15 @@ from data_platform_catalogue.entities import ( DatahubEntityType, DatahubSubtype, - EntityRef, - EntityTypes, + FindMoJdataEntityMapper, + TableEntityMapping, + ChartEntityMapping, + DatabaseEntityMapping, + DashboardEntityMapping, + PublicationDatasetEntityMapping, + PublicationCollectionEntityMapper, + GlossaryTermEntityMapping, + EntityRef ) from data_platform_catalogue.search_types import ( DomainOption, @@ -37,9 +44,6 @@ logger = logging.getLogger(__name__) -EntityTypeParsingFuncMap = namedtuple("EntityTypes", ["result_type", "parse_function"]) - - class SearchClient: def __init__(self, graph: DataHubGraph): self.graph = graph @@ -48,79 +52,59 @@ def __init__(self, graph: DataHubGraph): self.list_domains_query = get_graphql_query("listDomains") self.get_glossary_terms_query = get_graphql_query("getGlossaryTerms") self.get_tags_query = get_graphql_query("getTags") - self.fmd_type_to_datahub_types_mapping = { - EntityTypes.TABLE.value: ( - DatahubEntityType.DATASET.value, - ["Model", "Table", "Seed", "Source"], - ), - EntityTypes.CHART.value: (DatahubEntityType.CHART.value, []), - EntityTypes.DATABASE.value: ( - DatahubEntityType.CONTAINER.value, - ["Database"], - ), - EntityTypes.DASHBOARD.value: (DatahubEntityType.DASHBOARD.value, []), - EntityTypes.PUBLICATION_DATASET.value: ( - DatahubEntityType.DATASET.value, - ["Publication dataset"], - ), - EntityTypes.PUBLICATION_COLLECTION.value: ( - DatahubEntityType.CONTAINER.value, - ["Publication collection"], - ), - } self.datahub_types_to_fmd_type_and_parser_mapping = { ( DatahubEntityType.DATASET.value, DatahubSubtype.PUBLICATION_DATASET.value, ): ( self._parse_dataset, - EntityTypes.PUBLICATION_DATASET, + PublicationDatasetEntityMapping, ), (DatahubEntityType.DATASET.value, DatahubSubtype.METRIC.value): ( self._parse_dataset, - EntityTypes.TABLE, + TableEntityMapping, ), (DatahubEntityType.DATASET.value, DatahubSubtype.TABLE.value): ( self._parse_dataset, - EntityTypes.TABLE, + TableEntityMapping, ), (DatahubEntityType.DATASET.value, DatahubSubtype.MODEL.value): ( self._parse_dataset, - EntityTypes.TABLE, + TableEntityMapping, ), (DatahubEntityType.DATASET.value, DatahubSubtype.SEED.value): ( self._parse_dataset, - EntityTypes.TABLE, + TableEntityMapping, ), (DatahubEntityType.DATASET.value, DatahubSubtype.SOURCE.value): ( self._parse_dataset, - EntityTypes.TABLE, + TableEntityMapping, ), (DatahubEntityType.CONTAINER.value, DatahubSubtype.DATABASE.value): ( self._parse_container, - EntityTypes.DATABASE, + DatabaseEntityMapping, ), ( DatahubEntityType.CONTAINER.value, DatahubSubtype.PUBLICATION_COLLECTION.value, ): ( self._parse_container, - EntityTypes.PUBLICATION_COLLECTION, + PublicationCollectionEntityMapper, ), ( DatahubEntityType.DATASET.value, DatahubSubtype.PUBLICATION_DATASET.value, ): ( self._parse_container, - EntityTypes.PUBLICATION_DATASET, + PublicationDatasetEntityMapping, ), (DatahubEntityType.CHART.value, None): ( self._parse_dataset, - EntityTypes.CHART, + ChartEntityMapping, ), (DatahubEntityType.DASHBOARD.value, None): ( self._parse_container, - EntityTypes.DASHBOARD, + DashboardEntityMapping, ), } @@ -129,10 +113,10 @@ def search( query: str = "*", count: int = 20, page: str | None = None, - result_types: Sequence[EntityTypes] = ( - EntityTypes.TABLE, - EntityTypes.CHART, - EntityTypes.DATABASE, + result_types: Sequence[FindMoJdataEntityMapper] = ( + TableEntityMapping, + ChartEntityMapping, + DatabaseEntityMapping, ), filters: Sequence[MultiSelectFilter] | None = None, sort: SortOption | None = None, @@ -146,18 +130,12 @@ def search( start = 0 if page is None else int(page) * count - fmd_entity_types = [result_type.value for result_type in result_types] entity_type_filters = [ ( - MultiSelectFilter( - "_entityType", - self.fmd_type_to_datahub_types_mapping[entity_type][0], - ), - MultiSelectFilter( - "typeNames", self.fmd_type_to_datahub_types_mapping[entity_type][1] - ), + MultiSelectFilter("_entityType", result.datahub_type.value), + MultiSelectFilter("typeNames", result.datahub_subtypes), ) - for entity_type in fmd_entity_types + for result in result_types ] formatted_filters = map_filters(filters, entity_type_filters) @@ -275,7 +253,7 @@ def _get_data_collection_page_results(self, response, key_for_results: str): matched_fields: dict = {} if entity_type == "DATASET": page_results.append( - self._parse_dataset(entity, matched_fields, EntityTypes.TABLE) + self._parse_dataset(entity, matched_fields, TableEntityMapping) ) else: raise ValueError(f"Unexpected entity type: {entity_type}") @@ -283,7 +261,7 @@ def _get_data_collection_page_results(self, response, key_for_results: str): def _map_result_types( self, - result_types: Sequence[EntityTypes], + result_types: Sequence[FindMoJdataEntityMapper], ) -> list[str]: """ Map result types to Datahub EntityTypes @@ -310,7 +288,7 @@ def _parse_list_domains( return list_domain_options def _parse_dataset( - self, entity: dict[str, Any], matches, result_type: EntityTypes + self, entity: dict[str, Any], matches, result_type: FindMoJdataEntityMapper ) -> SearchResult: """ Map a dataset entity to a SearchResult @@ -333,7 +311,7 @@ def _parse_dataset( "total_parents": entity.get("relationships", {}).get("total", 0), "domain_name": domain.display_name, "domain_id": domain.urn, - "entity_types": self._parse_types_and_sub_types(entity, result_type.value), + "entity_types": self._parse_types_and_sub_types(entity, result_type.find_moj_data_type.value), } logger.debug(f"{metadata=}") @@ -392,7 +370,7 @@ def _parse_glossary_term(self, entity) -> SearchResult: return SearchResult( urn=entity["urn"], - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, matches={}, name=name, display_name=display_name, @@ -455,7 +433,7 @@ def _parse_global_tags(self, tag_query_results) -> list[tuple[str, str]]: return tags_list def _parse_container( - self, entity: dict[str, Any], matches, subtype: EntityTypes + self, entity: dict[str, Any], matches, subtype: FindMoJdataEntityMapper ) -> SearchResult: """ Map a Container entity to a SearchResult @@ -473,7 +451,7 @@ def _parse_container( "owner_email": owner.email, "domain_name": domain.display_name, "domain_id": domain.urn, - "entity_types": self._parse_types_and_sub_types(entity, subtype.value), + "entity_types": self._parse_types_and_sub_types(entity, subtype.find_moj_data_type.value), } metadata.update(custom_properties) diff --git a/lib/datahub-client/data_platform_catalogue/entities.py b/lib/datahub-client/data_platform_catalogue/entities.py index 4cd3e1fa..917e34fb 100644 --- a/lib/datahub-client/data_platform_catalogue/entities.py +++ b/lib/datahub-client/data_platform_catalogue/entities.py @@ -34,100 +34,87 @@ class DatahubSubtype(Enum): DATABASE = "Database" +class FindMoJdataEntityType(Enum): + TABLE = "Table" + GLOSSARY_TERM = "Glossary term" + CHART = "Chart" + DATABASE = "Database" + DASHBOARD = "Dashboard" + PUBLICATION_DATASET = "Publication dataset" + PUBLICATION_COLLECTION = "Publication collection" + + @dataclass -class FindMoJDataEntityMapper: - find_moj_data_type: str - datahub_type: str +class FindMoJdataEntityMapper: + find_moj_data_type: FindMoJdataEntityType + datahub_type: DatahubEntityType datahub_subtypes: list[str] url_formatted: str -class TableEntityMapper(FindMoJDataEntityMapper): - def __init__(self): - super().__init__( - "Table", - DatahubEntityType.DATASET.value, - ["Model", "Table", "Seed", "Source"], - "table", - ) - - -class ChartEntityMapper(FindMoJDataEntityMapper): - def __init__(self): - super().__init__("Chart", DatahubEntityType.CHART.value, [], "chart") - - -class DatabaseEntityMapper(FindMoJDataEntityMapper): - def __init__(self): - super().__init__( - "Database", DatahubEntityType.CONTAINER.value, ["Database"], "database" - ) - - -class DashboardEntityMapper(FindMoJDataEntityMapper): - def __init__(self): - super().__init__( - "Dashboard", DatahubEntityType.DASHBOARD.value, [], "dashboard" - ) - - -class PublicationDatasetEntityMapper(FindMoJDataEntityMapper): - def __init__(self): - super().__init__( - "Publication dataset", - DatahubEntityType.DATASET.value, - ["Publication dataset"], - "publication_dataset", - ) - - -class PublicationCollectionEntityMapper(FindMoJDataEntityMapper): - def __init__(self): - super().__init__( - "Publication collection", - DatahubEntityType.CONTAINER.value, - ["Publication collection"], - "publication_collection", - ) - - -class EntityTypes(Enum): - """Maps between Find MoJ data's entity type, Datahub entity types and the url_formatted representation - - Each entity has 3 properties: - - value: Human-Readable description of the entity - - datahub_entity_type: Datahub's description of the entity - - url_formatted: URL formatted representation of the entity - - ex: EntityTypes.TABLE.value returns `Table` - EntityTypes.GLOSSARY_TERM.datahub_entity_type returns `GLOSSARY_TERM`""" - - TABLE = ("Table", DatahubEntityType.DATASET.value, "table") - GLOSSARY_TERM = ( - "Glossary term", - DatahubEntityType.GLOSSARY_TERM.value, - "glossary_term", - ) - CHART = ("Chart", DatahubEntityType.CHART.value, "chart") - DATABASE = ("Database", DatahubEntityType.CONTAINER.value, "database") - DASHBOARD = ("Dashboard", DatahubEntityType.DASHBOARD.value, "dashboard") - PUBLICATION_DATASET = ( - "Publication dataset", - DatahubEntityType.DATASET.value, - "publication_dataset", - ) - PUBLICATION_COLLECTION = ( - "Publication collection", - DatahubEntityType.CONTAINER.value, - "publication_collection", - ) - - def __new__(cls, value, datahub_entity_type, url_formatted): - obj = object.__new__(cls) - obj._value_ = value - obj.datahub_entity_type = datahub_entity_type - obj.url_formatted = url_formatted - return obj +TableEntityMapping = FindMoJdataEntityMapper( + FindMoJdataEntityType.TABLE, + DatahubEntityType.DATASET, + [ + DatahubSubtype.MODEL.value, + DatahubSubtype.TABLE.value, + DatahubSubtype.SEED.value, + DatahubSubtype.SOURCE.value + ], + "table" +) + +ChartEntityMapping = FindMoJdataEntityMapper( + FindMoJdataEntityType.CHART, + DatahubEntityType.CHART, + [], + "chart" +) + +GlossaryTermEntityMapping = FindMoJdataEntityMapper( + FindMoJdataEntityType.GLOSSARY_TERM, + DatahubEntityType.GLOSSARY_TERM, + [], + "glossary_term" +) + +DatabaseEntityMapping = FindMoJdataEntityMapper( + FindMoJdataEntityType.DATABASE, + DatahubEntityType.CONTAINER, + [DatahubSubtype.DATABASE.value], + "database" +) + +DashboardEntityMapping = FindMoJdataEntityMapper( + FindMoJdataEntityType.DASHBOARD, + DatahubEntityType.DASHBOARD, + [], + "dashboard" +) + +PublicationDatasetEntityMapping = FindMoJdataEntityMapper( + FindMoJdataEntityType.PUBLICATION_DATASET, + DatahubEntityType.DATASET, + [DatahubSubtype.PUBLICATION_DATASET.value], + "publication_dataset" +) + +PublicationCollectionEntityMapper = FindMoJdataEntityMapper( + FindMoJdataEntityType.PUBLICATION_COLLECTION, + DatahubEntityType.CONTAINER, + [DatahubSubtype.PUBLICATION_COLLECTION.value], + "publication_collection" +) + +Mappers = [ + TableEntityMapping, + ChartEntityMapping, + GlossaryTermEntityMapping, + DatabaseEntityMapping, + DashboardEntityMapping, + PublicationDatasetEntityMapping, + PublicationCollectionEntityMapper +] class Audience(Enum): @@ -610,25 +597,20 @@ class Database(Entity): ) # tables: list = Field(description="list of tables in the database") - class PublicationCollection(Entity): - """Collections of datasets that are periodically published to GOV.UK""" - + """For source system publication collections""" urn: str | None = Field( description="Unique identifier for the entity. Relates to Datahub's urn", examples=["urn:li:container:criminal_justice_stats"], ) external_url: str = Field( description="URL to view the collection", - examples=[ - "https://www.gov.uk/government/collections/civil-justice-statistics-quarterly" - ], + examples=["https://data.justice.gov.uk/prisons/criminal-jsutice/publications"], ) class PublicationDataset(Entity): - """A dataset published to GOV.UK""" - + """For source system publication collections""" urn: str | None = Field( description="Unique identifier for the entity. Relates to Datahub's urn", examples=["urn:li:dataset:(urn:li:dataPlatform:gov.uk,statistics2011,DEV)"], diff --git a/lib/datahub-client/data_platform_catalogue/search_types.py b/lib/datahub-client/data_platform_catalogue/search_types.py index 2f8f6ce0..fd459d43 100644 --- a/lib/datahub-client/data_platform_catalogue/search_types.py +++ b/lib/datahub-client/data_platform_catalogue/search_types.py @@ -6,8 +6,8 @@ from data_platform_catalogue.entities import ( EntityRef, - EntityTypes, GlossaryTermRef, + FindMoJdataEntityMapper, TagRef, ) @@ -64,7 +64,7 @@ class DomainOption: @dataclass class SearchResult: urn: str - result_type: EntityTypes + result_type: FindMoJdataEntityMapper name: str display_name: str = "" fully_qualified_name: str = "" diff --git a/lib/datahub-client/tests/client/search/test_search_client.py b/lib/datahub-client/tests/client/search/test_search_client.py index 6a7740b6..4e5f6a60 100644 --- a/lib/datahub-client/tests/client/search/test_search_client.py +++ b/lib/datahub-client/tests/client/search/test_search_client.py @@ -7,7 +7,10 @@ AccessInformation, DataSummary, EntityRef, - EntityTypes, + TableEntityMapping, + DatabaseEntityMapping, + ChartEntityMapping, + GlossaryTermEntityMapping, FurtherInformation, TagRef, UsageRestrictions, @@ -122,7 +125,7 @@ def test_one_search_result(mock_graph, searcher): page_results=[ SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="customers", display_name="customers", fully_qualified_name="jaffle_shop.customers", @@ -213,7 +216,7 @@ def test_dataset_result(mock_graph, searcher): page_results=[ SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="customers", display_name="customers", fully_qualified_name="jaffle_shop.customers", @@ -363,7 +366,7 @@ def test_2_dataset_results_with_one_malformed_result(mock_graph, searcher): page_results=[ SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="customers", display_name="customers", fully_qualified_name="jaffle_shop.customers", @@ -460,7 +463,7 @@ def test_full_page(mock_graph, searcher): page_results=[ SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="customers", fully_qualified_name="jaffle_shop.customers", display_name="customers", @@ -491,7 +494,7 @@ def test_full_page(mock_graph, searcher): ), SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers2,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="customers2", fully_qualified_name="calm-pagoda-323403.jaffle_shop.customers2", display_name="customers2", @@ -522,7 +525,7 @@ def test_full_page(mock_graph, searcher): ), SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers3,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="customers3", fully_qualified_name="calm-pagoda-323403.jaffle_shop.customers3", display_name="customers3", @@ -600,7 +603,7 @@ def test_query_match(mock_graph, searcher): page_results=[ SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="customers", display_name="customers", fully_qualified_name="calm-pagoda-323403.jaffle_shop.customers", @@ -685,7 +688,7 @@ def test_result_with_owner(mock_graph, searcher): page_results=[ SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:bigquery,calm-pagoda-323403.jaffle_shop.customers,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="customers", display_name="customers", fully_qualified_name="calm-pagoda-323403.jaffle_shop.customers", @@ -913,7 +916,7 @@ def test_get_glossary_terms(mock_graph, searcher): } ] }, - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, ), SearchResult( urn="urn:li:glossaryTerm:0eb7af28-62b4-4149-a6fa-72a8f1fea1e6", @@ -922,7 +925,7 @@ def test_get_glossary_terms(mock_graph, searcher): fully_qualified_name="Security classification", description="Only data that is 'official'", metadata={"parentNodes": []}, - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, ), ], ) @@ -975,7 +978,7 @@ def test_search_for_charts(mock_graph, searcher): page_results=[ SearchResult( urn="urn:li:chart:(justice-data,absconds)", - result_type=EntityTypes.CHART, + result_type=ChartEntityMapping, name="Absconds", display_name="Absconds", fully_qualified_name="Absconds", @@ -1097,7 +1100,7 @@ def test_search_for_container(mock_graph, searcher): page_results=[ SearchResult( urn="urn:li:container:test_db", - result_type=EntityTypes.DATABASE, + result_type=DatabaseEntityMapping, name="test_db", display_name="test_db", fully_qualified_name="test_db", @@ -1151,7 +1154,7 @@ def test_search_for_container(mock_graph, searcher): def test_tag_to_display(tags, result): test_search_result = SearchResult( urn="urn:li:dataset:(urn:li:dataPlatform:athena,test_db.test_table,PROD)", - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, name="test_table", display_name="test_table", fully_qualified_name="test_db.test_table", diff --git a/lib/datahub-client/tests/test_integration_with_datahub_server.py b/lib/datahub-client/tests/test_integration_with_datahub_server.py index 89cbcd3c..b2f7648a 100644 --- a/lib/datahub-client/tests/test_integration_with_datahub_server.py +++ b/lib/datahub-client/tests/test_integration_with_datahub_server.py @@ -9,25 +9,20 @@ import os import time -from datetime import datetime, timezone +from datetime import datetime import pytest from data_platform_catalogue.client.datahub_client import DataHubCatalogueClient from data_platform_catalogue.entities import ( AccessInformation, - Column, - ColumnRef, CustomEntityProperties, Database, - DataSummary, DomainRef, EntityRef, - FurtherInformation, + TableEntityMapping, Governance, OwnerRef, - RelationshipType, - Table, TagRef, UsageRestrictions, ) @@ -35,7 +30,6 @@ DomainOption, MultiSelectFilter, ) -from data_platform_catalogue.entities import EntityTypes jwt_token = os.environ.get("CATALOGUE_TOKEN") api_url = os.environ.get("CATALOGUE_URL", "") @@ -66,7 +60,7 @@ def test_search_by_domain(): response = client.search( filters=[MultiSelectFilter("domains", ["does-not-exist"])], - result_types=(EntityTypes.TABLE,), + result_types=(TableEntityMapping), ) assert response.total_results == 0 diff --git a/templates/partial/search_result.html b/templates/partial/search_result.html index c73872d4..89e9b686 100644 --- a/templates/partial/search_result.html +++ b/templates/partial/search_result.html @@ -8,11 +8,11 @@

- {% with result_type=result.result_type.name|lower %} + {% with result_type=result.result_type.url_formatted %} {{result.name}} {% endwith %} - {{ result.result_type.value }} + {{ result.result_type.find_moj_data_type.value }}

{% if result.description %} diff --git a/tests/conftest.py b/tests/conftest.py index a7982375..5044a2b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,3 @@ -from datetime import datetime, timezone -from pathlib import Path from random import choice from typing import Any, Generator from unittest.mock import MagicMock, patch @@ -15,7 +13,6 @@ from home.service.search import SearchService from home.service.search_tag_fetcher import SearchTagFetcher from notifications_python_client.notifications import NotificationsAPIClient -from pytest import CollectReport, StashKey from selenium.webdriver import ChromeOptions from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By @@ -33,7 +30,10 @@ Database, DomainRef, EntityRef, - EntityTypes, + FindMoJdataEntityMapper, + DatabaseEntityMapping, + TableEntityMapping, + GlossaryTermEntityMapping, EntitySummary, GlossaryTermRef, Governance, @@ -44,8 +44,6 @@ ) from data_platform_catalogue.search_types import ( DomainOption, - FacetOption, - SearchFacets, SearchResponse, SearchResult, ) @@ -306,7 +304,7 @@ def page_titles(): def generate_search_result( - result_type: EntityTypes | None = None, urn=None, metadata=None + result_type: FindMoJdataEntityMapper | None = None, urn=None, metadata=None ) -> SearchResult: """ Generate a random search result @@ -316,7 +314,7 @@ def generate_search_result( return SearchResult( urn=urn or fake.unique.name(), result_type=( - choice((EntityTypes.DATABASE, EntityTypes.TABLE)) + choice((DatabaseEntityMapping, TableEntityMapping)) if result_type is None else result_type ), @@ -330,7 +328,7 @@ def generate_search_result( def search_result_from_database(database: Database): return SearchResult( urn=database.urn or "", - result_type=EntityTypes.DATABASE, + result_type=DatabaseEntityMapping, name=database.name, fully_qualified_name=database.fully_qualified_name or "", description=database.description, @@ -574,7 +572,7 @@ def example_table(name="example_table"): return generate_table_metadata(name=name) -def generate_page(page_size=20, result_type: EntityTypes | None = None): +def generate_page(page_size=20, result_type: FindMoJdataEntityMapper | None = None): """ Generate a fake search page """ @@ -712,7 +710,7 @@ def mock_get_glossary_terms_response(mock_catalogue): } ] }, - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, ), SearchResult( urn="urn:li:glossaryTerm:022b9b68-c211-47ae-aef0-2db13acfeca8", @@ -728,14 +726,14 @@ def mock_get_glossary_terms_response(mock_catalogue): } ] }, - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, ), SearchResult( urn="urn:li:glossaryTerm:0eb7af28-62b4-4149-a6fa-72a8f1fea1e6", name="Security classification", description="Only data that is 'official'", metadata={"parentNodes": []}, - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, ), ], ) @@ -792,7 +790,7 @@ def search_context(search_service): def detail_database_context(mock_catalogue): mock_catalogue.search.return_value = SearchResponse( total_results=1, - page_results=generate_page(page_size=1, result_type=EntityTypes.DATABASE), + page_results=generate_page(page_size=1, result_type=DatabaseEntityMapping), ) details_service = DatabaseDetailsService(urn="urn:li:container:test") @@ -814,7 +812,7 @@ def dataset_with_parent(mock_catalogue) -> dict[str, Any]: total_results=1, page_results=[ generate_search_result( - result_type=EntityTypes.TABLE, + result_type=TableEntityMapping, urn="table-abc", metadata={}, ) diff --git a/tests/end_to_end/test_datahub_to_fmd.py b/tests/end_to_end/test_datahub_to_fmd.py index e31025f8..a5d784d4 100644 --- a/tests/end_to_end/test_datahub_to_fmd.py +++ b/tests/end_to_end/test_datahub_to_fmd.py @@ -6,7 +6,7 @@ @pytest.mark.slow @pytest.mark.datahub -class TestDatahubToFindMoJData: +class TestDatahubToFindMoJdata: """ Test that Find MoJ data works with a real Datahub backend. The datahub mark is used to bypass the `mock_catalogue` fixture. diff --git a/tests/home/service/test_glossary.py b/tests/home/service/test_glossary.py index bc743843..2a551fde 100644 --- a/tests/home/service/test_glossary.py +++ b/tests/home/service/test_glossary.py @@ -1,5 +1,5 @@ +from data_platform_catalogue.entities import GlossaryTermEntityMapping from data_platform_catalogue.search_types import SearchResult -from data_platform_catalogue.entities import EntityTypes from home.service.glossary import GlossaryService @@ -15,7 +15,7 @@ def test_get_context(self): "members": [ SearchResult( urn="urn:li:glossaryTerm:022b9b68-c211-47ae-aef0-2db13acfeca8", - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, name="IAO", description="Information asset owner.\n", matches={}, @@ -34,7 +34,7 @@ def test_get_context(self): ), SearchResult( urn="urn:li:glossaryTerm:022b9b68-c211-47ae-aef0-2db13acfeca8", - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, name="Other term", description="Term description to test groupings work", matches={}, @@ -59,7 +59,7 @@ def test_get_context(self): "members": [ SearchResult( urn="urn:li:glossaryTerm:0eb7af28-62b4-4149-a6fa-72a8f1fea1e6", - result_type=EntityTypes.GLOSSARY_TERM, + result_type=GlossaryTermEntityMapping, name="Security classification", description="Only data that is 'official'", matches={}, diff --git a/tests/integration/test_interact_with_search_results.py b/tests/integration/test_interact_with_search_results.py index ec4cbd98..05963dd1 100644 --- a/tests/integration/test_interact_with_search_results.py +++ b/tests/integration/test_interact_with_search_results.py @@ -1,7 +1,6 @@ import pytest -from data_platform_catalogue.search_types import SearchResult -from data_platform_catalogue.entities import EntityTypes +from data_platform_catalogue.entities import TableEntityMapping from tests.conftest import ( generate_page, generate_table_metadata, @@ -46,7 +45,7 @@ def test_table_search_to_details(self, mock_catalogue): """ mock_search_response( mock_catalogue=mock_catalogue, - page_results=generate_page(result_type=EntityTypes.TABLE), + page_results=generate_page(result_type=TableEntityMapping), total_results=100, ) self.start_on_the_search_page() @@ -63,7 +62,7 @@ def test_table_search_to_details_accessibility(self, mock_catalogue): """ mock_search_response( mock_catalogue=mock_catalogue, - page_results=generate_page(result_type=EntityTypes.TABLE), + page_results=generate_page(result_type=TableEntityMapping), total_results=100, ) table_no_column_description = generate_table_metadata(column_description="") diff --git a/tests/integration/test_search_result_metadata.py b/tests/integration/test_search_result_metadata.py index 54db7c83..3d080f49 100644 --- a/tests/integration/test_search_result_metadata.py +++ b/tests/integration/test_search_result_metadata.py @@ -1,6 +1,6 @@ import pytest +from data_platform_catalogue.entities import DatabaseEntityMapping from data_platform_catalogue.search_types import SearchResult -from data_platform_catalogue.entities import EntityTypes from tests.conftest import mock_search_response @@ -21,7 +21,7 @@ def setup(self, live_server, selenium): def test_matched_fields_hidden(self, mock_catalogue): result = SearchResult( urn="fake-urn", - result_type=EntityTypes.DATABASE, + result_type=DatabaseEntityMapping, name="abc", fully_qualified_name="abc", description="bla bla bla", @@ -36,7 +36,7 @@ def test_matched_fields_hidden(self, mock_catalogue): def test_matched_fields_shown(self, mock_catalogue): result = SearchResult( urn="fake-urn", - result_type=EntityTypes.DATABASE, + result_type=DatabaseEntityMapping, name="abc", fully_qualified_name="abc", description="bla bla bla",