Feature/ted 152

ted_sws/core/model/notice.py

@@ -39,7 +39,9 @@ class NoticeStatus(IntEnum):
     NORMALISED_METADATA = 20
     INELIGIBLE_FOR_TRANSFORMATION = 23  # backlog status
     ELIGIBLE_FOR_TRANSFORMATION = 27  # forward status
+    PREPROCESSED_FOR_TRANSFORMATION = 29
     TRANSFORMED = 30
+    DISTILLED = 35
     VALIDATED = 40
     INELIGIBLE_FOR_PACKAGING = 43  # backlog status
     ELIGIBLE_FOR_PACKAGING = 47  # forward status
@@ -67,8 +69,11 @@ def __gt__(self, other):
                                                                           NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION],
                                        NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION: [
                                            NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION],
-                                       NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION: [NoticeStatus.TRANSFORMED],
-                                       NoticeStatus.TRANSFORMED: [NoticeStatus.VALIDATED],
+                                       NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION: [
+                                           NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION],
+                                       NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION: [NoticeStatus.TRANSFORMED],
+                                       NoticeStatus.TRANSFORMED: [NoticeStatus.DISTILLED],
+                                       NoticeStatus.DISTILLED: [NoticeStatus.VALIDATED],
                                        NoticeStatus.VALIDATED: [NoticeStatus.INELIGIBLE_FOR_PACKAGING,
                                                                 NoticeStatus.ELIGIBLE_FOR_PACKAGING],
                                        NoticeStatus.INELIGIBLE_FOR_PACKAGING: [NoticeStatus.ELIGIBLE_FOR_PACKAGING],
@@ -148,9 +153,19 @@ class Notice(WorkExpression):
     _normalised_metadata: Optional[NormalisedMetadata] = None
 
     xml_manifestation: XMLManifestation = Field(..., allow_mutation=False)
+    _preprocessed_xml_manifestation: Optional[XMLManifestation] = None
+    _distilled_rdf_manifestation: Optional[RDFManifestation] = None
     _rdf_manifestation: Optional[RDFManifestation] = None
     _mets_manifestation: Optional[METSManifestation] = None
 
+    @property
+    def preprocessed_xml_manifestation(self) -> XMLManifestation:
+        return self._preprocessed_xml_manifestation
+
+    @property
+    def distilled_rdf_manifestation(self) -> RDFManifestation:
+        return self._distilled_rdf_manifestation
+
     @property
     def normalised_metadata(self) -> NormalisedMetadata:
         return self._normalised_metadata
@@ -170,6 +185,28 @@ def rdf_validation(self) -> RDFValidationManifestation:
 
         return self.rdf_manifestation.validation
 
+    def set_preprocessed_xml_manifestation(self, preprocessed_xml_manifestation: XMLManifestation):
+        """
+            Set preprocessed XML manifestation to the notice.
+        :param preprocessed_xml_manifestation:
+        :return:
+        """
+        if self.preprocessed_xml_manifestation == preprocessed_xml_manifestation:
+            return
+        self._preprocessed_xml_manifestation = preprocessed_xml_manifestation
+        self.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION)
+
+    def set_distilled_rdf_manifestation(self, distilled_rdf_manifestation: RDFManifestation):
+        """
+            Set distilled RDF manifestation to the notice.
+        :param distilled_rdf_manifestation:
+        :return:
+        """
+        if self.distilled_rdf_manifestation == distilled_rdf_manifestation:
+            return
+        self._distilled_rdf_manifestation = distilled_rdf_manifestation
+        self.update_status_to(NoticeStatus.DISTILLED)
+
     def set_normalised_metadata(self, normalised_metadata: NormalisedMetadata):
         """
             Add normalised metadata to the notice.
@@ -316,7 +353,9 @@ def update_status_to(self, new_status: NoticeStatus):
             self._status = new_status
             if new_status < NoticeStatus.NORMALISED_METADATA:
                 self._normalised_metadata = None
+                self._preprocessed_xml_manifestation = None
             if new_status < NoticeStatus.TRANSFORMED:
                 self._rdf_manifestation = None
+                self._distilled_rdf_manifestation = None
             if new_status < NoticeStatus.PACKAGED:
                 self._mets_manifestation = None

ted_sws/notice_transformer/services/notice_transformer.py

@@ -83,7 +83,7 @@ def transform_test_data(self, output_path: pathlib.Path):
         file_resources = []
         for data in transformation_test_data.test_data:
             notice = Notice(ted_id="tmp_notice",xml_manifestation=XMLManifestation(object_data=data.file_content))
-            notice._status = NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION
+            notice._status = NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION
             notice_result = self.transform_notice(notice=notice)
             file_resources.append(
                 FileResource(file_name=data.file_name, file_content=notice_result.rdf_manifestation.object_data))

tests/features/model/conftest.py

@@ -32,6 +32,8 @@ def publicly_available_notice(fetched_notice_data, normalised_metadata_dict) ->
     notice._rdf_manifestation = RDFManifestation(object_data="RDF manifestation content", validation=validation)
     notice._mets_manifestation = METSManifestation(object_data="METS manifestation content")
     notice._normalised_metadata = NormalisedMetadata(**normalised_metadata_dict)
+    notice._distilled_rdf_manifestation = RDFManifestation(object_data="RDF manifestation content", validation=validation)
+    notice._preprocessed_xml_manifestation = xml_manifestation
     notice._status = NoticeStatus.PUBLICLY_AVAILABLE
     return notice
 
@@ -47,4 +49,5 @@ def raw_notice(fetched_notice_data) -> Notice:
 def transformation_eligible_notice(raw_notice, normalised_metadata_dict) -> Notice:
     raw_notice.set_normalised_metadata(normalised_metadata=NormalisedMetadata(**normalised_metadata_dict))
     raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION)
+    raw_notice.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION)
     return raw_notice

tests/features/model/test_notice_operations.py

@@ -307,11 +307,12 @@ def step_impl(a_notice, availability):
 
 @given("a notice eligible for transformation", )
 def step_impl(transformation_eligible_notice):
-    assert transformation_eligible_notice.status is NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION
+    assert transformation_eligible_notice.status is NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION
 
 
 @when("RDF validation report is added")
 def step_impl(transformation_eligible_notice, rdf_validation):
+    transformation_eligible_notice.update_status_to(NoticeStatus.DISTILLED)
     transformation_eligible_notice.set_rdf_validation(rdf_validation)
 
 

tests/unit/core/model/conftest.py

@@ -32,6 +32,8 @@ def publicly_available_notice(fetched_notice_data, normalised_metadata_dict) ->
     notice._rdf_manifestation = RDFManifestation(object_data="RDF manifestation content", validation=validation)
     notice._mets_manifestation = METSManifestation(object_data="METS manifestation content")
     notice._normalised_metadata = NormalisedMetadata(**normalised_metadata_dict)
+    notice._distilled_rdf_manifestation = RDFManifestation(object_data="RDF manifestation content", validation=validation)
+    notice._preprocessed_xml_manifestation = xml_manifestation
     notice._status = NoticeStatus.PUBLICLY_AVAILABLE
     return notice
 

tests/unit/core/model/test_notice_status_transition.py

@@ -45,7 +45,9 @@ def test_notice_status_upstream_transition(raw_notice):
     raw_notice.update_status_to(NoticeStatus.NORMALISED_METADATA)
     raw_notice.update_status_to(NoticeStatus.INELIGIBLE_FOR_TRANSFORMATION)
     raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION)
+    raw_notice.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION)
     raw_notice.update_status_to(NoticeStatus.TRANSFORMED)
+    raw_notice.update_status_to(NoticeStatus.DISTILLED)
     raw_notice.update_status_to(NoticeStatus.VALIDATED)
     raw_notice.update_status_to(NoticeStatus.INELIGIBLE_FOR_PACKAGING)
     raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_PACKAGING)
@@ -69,6 +71,8 @@ def test_notice_status_transition_below_packaged(publicly_available_notice):
     assert publicly_available_notice.normalised_metadata is not None
     assert publicly_available_notice.xml_manifestation is not None
     assert publicly_available_notice.original_metadata is not None
+    assert publicly_available_notice.distilled_rdf_manifestation is not None
+    assert publicly_available_notice.preprocessed_xml_manifestation is not None
 
 
 def test_notice_status_transition_below_transformed(publicly_available_notice):

tests/unit/core/model/test_update_notice_state.py

@@ -64,7 +64,9 @@ def test_setting_mets_manifestation_downstream(raw_notice):
 
     raw_notice.update_status_to(NoticeStatus.NORMALISED_METADATA)
     raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION)
+    raw_notice.update_status_to(NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION)
     raw_notice.set_rdf_manifestation(RDFManifestation(object_data="rdf data"))
+    raw_notice.update_status_to(NoticeStatus.DISTILLED)
     raw_notice.update_status_to(NoticeStatus.VALIDATED)
     raw_notice.update_status_to(NoticeStatus.ELIGIBLE_FOR_PACKAGING)
     raw_notice.set_mets_manifestation(METSManifestation(object_data="mets data"))

tests/unit/notice_transformer/test_notice_transformer.py

@@ -9,13 +9,13 @@
 
 def test_notice_transformer(fake_rml_mapper, fake_mapping_suite, notice_2018):
     notice_transformer = NoticeTransformer(mapping_suite=fake_mapping_suite, rml_mapper=fake_rml_mapper)
-    notice_2018._status = NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION
+    notice_2018._status = NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION
     notice_transformer.transform_notice(notice=notice_2018)
     assert notice_2018.status == NoticeStatus.TRANSFORMED
 
 
 def test_notice_transformer_function(fake_rml_mapper, fake_mapping_suite, notice_2018):
-    notice_2018._status = NoticeStatus.ELIGIBLE_FOR_TRANSFORMATION
+    notice_2018._status = NoticeStatus.PREPROCESSED_FOR_TRANSFORMATION
     result_notice = transform_notice(notice_2018, fake_mapping_suite, fake_rml_mapper)
     assert result_notice.status == NoticeStatus.TRANSFORMED