diff --git a/ted_sws/core/model/notice.py b/ted_sws/core/model/notice.py index b3fda199f..8d5f64287 100644 --- a/ted_sws/core/model/notice.py +++ b/ted_sws/core/model/notice.py @@ -39,7 +39,9 @@ class NoticeStatus(IntEnum): NORMALISED_METADATA = 20 INELIGIBLE_FOR_TRANSFORMATION = 23 # backlog status ELIGIBLE_FOR_TRANSFORMATION = 27 # forward status + PREPROCESSED_FOR_TRANSFORMATION = 29 TRANSFORMED = 30 + DISTILLED = 35 VALIDATED = 40 INELIGIBLE_FOR_PACKAGING = 43 # backlog status ELIGIBLE_FOR_PACKAGING = 47 # forward status @@ -67,8 +69,11 @@ def __gt__(self, other): NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION], NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION: [ NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION], - NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION: [NoticeStatus.TRANSFORMED], - NoticeStatus.TRANSFORMED: [NoticeStatus.VALIDATED], + NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION: [ + NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION], + NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION: [NoticeStatus.TRANSFORMED], + NoticeStatus.TRANSFORMED: [NoticeStatus.DISTILLED], + NoticeStatus.DISTILLED: [NoticeStatus.VALIDATED], NoticeStatus.VALIDATED: [NoticeStatus.INELIGIBLE_FOR_PACKAGING, NoticeStatus.ELIGIBLE_FOR_PACKAGING], NoticeStatus.INELIGIBLE_FOR_PACKAGING: [NoticeStatus.ELIGIBLE_FOR_PACKAGING], @@ -148,9 +153,19 @@ class Notice(WorkExpression): _normalised_metadata: Optional[NormalisedMetadata] = None xml_manifestation: XMLManifestation = Field(..., allow_mutation=False) + _preprocessed_xml_manifestation: Optional[XMLManifestation] = None + _distilled_rdf_manifestation: Optional[RDFManifestation] = None _rdf_manifestation: Optional[RDFManifestation] = None _mets_manifestation: Optional[METSManifestation] = None + @property + def preprocessed_xml_manifestation(self) -> XMLManifestation: + return self._preprocessed_xml_manifestation + + @property + def distilled_rdf_manifestation(self) -> RDFManifestation: + return self._distilled_rdf_manifestation + @property def normalised_metadata(self) -> NormalisedMetadata: return self._normalised_metadata @@ -170,6 +185,28 @@ def rdf_validation(self) -> RDFValidationManifestation: return self.rdf_manifestation.validation + def set_preprocessed_xml_manifestation(self, preprocessed_xml_manifestation: XMLManifestation): + """ + Set preprocessed XML manifestation to the notice. + :param preprocessed_xml_manifestation: + :return: + """ + if self.preprocessed_xml_manifestation == preprocessed_xml_manifestation: + return + self._preprocessed_xml_manifestation = preprocessed_xml_manifestation + self.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION) + + def set_distilled_rdf_manifestation(self, distilled_rdf_manifestation: RDFManifestation): + """ + Set distilled RDF manifestation to the notice. + :param distilled_rdf_manifestation: + :return: + """ + if self.distilled_rdf_manifestation == distilled_rdf_manifestation: + return + self._distilled_rdf_manifestation = distilled_rdf_manifestation + self.update_status_to(NoticeStatus.DISTILLED) + def set_normalised_metadata(self, normalised_metadata: NormalisedMetadata): """ Add normalised metadata to the notice. @@ -316,7 +353,9 @@ def update_status_to(self, new_status: NoticeStatus): self._status = new_status if new_status < NoticeStatus.NORMALISED_METADATA: self._normalised_metadata = None + self._preprocessed_xml_manifestation = None if new_status < NoticeStatus.TRANSFORMED: self._rdf_manifestation = None + self._distilled_rdf_manifestation = None if new_status < NoticeStatus.PACKAGED: self._mets_manifestation = None diff --git a/ted_sws/notice_transformer/services/notice_transformer.py b/ted_sws/notice_transformer/services/notice_transformer.py index eae2294c1..95e46afe4 100644 --- a/ted_sws/notice_transformer/services/notice_transformer.py +++ b/ted_sws/notice_transformer/services/notice_transformer.py @@ -83,7 +83,7 @@ def transform_test_data(self, output_path: pathlib.Path): file_resources = [] for data in transformation_test_data.test_data: notice = Notice(ted_id="tmp_notice",xml_manifestation=XMLManifestation(object_data=data.file_content)) - notice._status = NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION + notice._status = NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION notice_result = self.transform_notice(notice=notice) file_resources.append( FileResource(file_name=data.file_name, file_content=notice_result.rdf_manifestation.object_data)) diff --git a/tests/features/model/conftest.py b/tests/features/model/conftest.py index ab1f63a28..33388398c 100644 --- a/tests/features/model/conftest.py +++ b/tests/features/model/conftest.py @@ -32,6 +32,8 @@ def publicly_available_notice(fetched_notice_data, normalised_metadata_dict) -> notice._rdf_manifestation = RDFManifestation(object_data="RDF manifestation content", validation=validation) notice._mets_manifestation = METSManifestation(object_data="METS manifestation content") notice._normalised_metadata = NormalisedMetadata(**normalised_metadata_dict) + notice._distilled_rdf_manifestation = RDFManifestation(object_data="RDF manifestation content", validation=validation) + notice._preprocessed_xml_manifestation = xml_manifestation notice._status = NoticeStatus.PUBLICLY_AVAILABLE return notice @@ -47,4 +49,5 @@ def raw_notice(fetched_notice_data) -> Notice: def transformation_eligible_notice(raw_notice, normalised_metadata_dict) -> Notice: raw_notice.set_normalised_metadata(normalised_metadata=NormalisedMetadata(**normalised_metadata_dict)) raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION) + raw_notice.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION) return raw_notice diff --git a/tests/features/model/test_notice_operations.py b/tests/features/model/test_notice_operations.py index 8f3eb0d41..7be2b9097 100644 --- a/tests/features/model/test_notice_operations.py +++ b/tests/features/model/test_notice_operations.py @@ -307,11 +307,12 @@ def step_impl(a_notice, availability): @given("a notice eligible for transformation", ) def step_impl(transformation_eligible_notice): - assert transformation_eligible_notice.status is NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION + assert transformation_eligible_notice.status is NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION @when("RDF validation report is added") def step_impl(transformation_eligible_notice, rdf_validation): + transformation_eligible_notice.update_status_to(NoticeStatus.DISTILLED) transformation_eligible_notice.set_rdf_validation(rdf_validation) diff --git a/tests/unit/core/model/conftest.py b/tests/unit/core/model/conftest.py index 1a66bb2cf..9f49be166 100644 --- a/tests/unit/core/model/conftest.py +++ b/tests/unit/core/model/conftest.py @@ -32,6 +32,8 @@ def publicly_available_notice(fetched_notice_data, normalised_metadata_dict) -> notice._rdf_manifestation = RDFManifestation(object_data="RDF manifestation content", validation=validation) notice._mets_manifestation = METSManifestation(object_data="METS manifestation content") notice._normalised_metadata = NormalisedMetadata(**normalised_metadata_dict) + notice._distilled_rdf_manifestation = RDFManifestation(object_data="RDF manifestation content", validation=validation) + notice._preprocessed_xml_manifestation = xml_manifestation notice._status = NoticeStatus.PUBLICLY_AVAILABLE return notice diff --git a/tests/unit/core/model/test_notice_status_transition.py b/tests/unit/core/model/test_notice_status_transition.py index 4314677eb..39a2650d0 100644 --- a/tests/unit/core/model/test_notice_status_transition.py +++ b/tests/unit/core/model/test_notice_status_transition.py @@ -45,7 +45,9 @@ def test_notice_status_upstream_transition(raw_notice): raw_notice.update_status_to(NoticeStatus.NORMALISED_METADATA) raw_notice.update_status_to(NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION) raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION) + raw_notice.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION) raw_notice.update_status_to(NoticeStatus.TRANSFORMED) + raw_notice.update_status_to(NoticeStatus.DISTILLED) raw_notice.update_status_to(NoticeStatus.VALIDATED) raw_notice.update_status_to(NoticeStatus.INELIGIBLE_FOR_PACKAGING) raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_PACKAGING) @@ -69,6 +71,8 @@ def test_notice_status_transition_below_packaged(publicly_available_notice): assert publicly_available_notice.normalised_metadata is not None assert publicly_available_notice.xml_manifestation is not None assert publicly_available_notice.original_metadata is not None + assert publicly_available_notice.distilled_rdf_manifestation is not None + assert publicly_available_notice.preprocessed_xml_manifestation is not None def test_notice_status_transition_below_transformed(publicly_available_notice): @@ -96,3 +100,25 @@ def test_notice_status_transition_below_normalised_metadata(publicly_available_n assert publicly_available_notice.normalised_metadata is None assert publicly_available_notice.xml_manifestation is not None assert publicly_available_notice.original_metadata is not None + + +def test_notice_status_transition_check_preprocessed_and_distilled_state(raw_notice, publicly_available_notice): + """ + + :param raw_notice: + :param publicly_available_notice: + :return: + """ + assert raw_notice.status == NoticeStatus.RAW + raw_notice._status = NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION + raw_notice.set_preprocessed_xml_manifestation( + preprocessed_xml_manifestation=publicly_available_notice.preprocessed_xml_manifestation) + assert raw_notice.status == NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION + raw_notice.set_preprocessed_xml_manifestation( + preprocessed_xml_manifestation=publicly_available_notice.preprocessed_xml_manifestation) + raw_notice.update_status_to(NoticeStatus.TRANSFORMED) + raw_notice.set_distilled_rdf_manifestation( + distilled_rdf_manifestation=publicly_available_notice.distilled_rdf_manifestation) + assert raw_notice.status == NoticeStatus.DISTILLED + raw_notice.set_distilled_rdf_manifestation( + distilled_rdf_manifestation=publicly_available_notice.distilled_rdf_manifestation) diff --git a/tests/unit/core/model/test_update_notice_state.py b/tests/unit/core/model/test_update_notice_state.py index e9f604671..c180e339a 100644 --- a/tests/unit/core/model/test_update_notice_state.py +++ b/tests/unit/core/model/test_update_notice_state.py @@ -64,7 +64,9 @@ def test_setting_mets_manifestation_downstream(raw_notice): raw_notice.update_status_to(NoticeStatus.NORMALISED_METADATA) raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION) + raw_notice.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION) raw_notice.set_rdf_manifestation(RDFManifestation(object_data="rdf data")) + raw_notice.update_status_to(NoticeStatus.DISTILLED) raw_notice.update_status_to(NoticeStatus.VALIDATED) raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_PACKAGING) raw_notice.set_mets_manifestation(METSManifestation(object_data="mets data")) diff --git a/tests/unit/notice_transformer/test_notice_transformer.py b/tests/unit/notice_transformer/test_notice_transformer.py index 752c9634f..5ab37bab6 100644 --- a/tests/unit/notice_transformer/test_notice_transformer.py +++ b/tests/unit/notice_transformer/test_notice_transformer.py @@ -9,13 +9,13 @@ def test_notice_transformer(fake_rml_mapper, fake_mapping_suite, notice_2018): notice_transformer = NoticeTransformer(mapping_suite=fake_mapping_suite, rml_mapper=fake_rml_mapper) - notice_2018._status = NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION + notice_2018._status = NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION notice_transformer.transform_notice(notice=notice_2018) assert notice_2018.status == NoticeStatus.TRANSFORMED def test_notice_transformer_function(fake_rml_mapper, fake_mapping_suite, notice_2018): - notice_2018._status = NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION + notice_2018._status = NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION result_notice = transform_notice(notice_2018, fake_mapping_suite, fake_rml_mapper) assert result_notice.status == NoticeStatus.TRANSFORMED