WIP

wagtail_localize/segments/extract.py

@@ -28,8 +28,7 @@ def handle_block(self, block_type, block_value):
             template, texts = extract_html_segments(block_value.source)
 
             return [TemplateValue("", "html", template, len(texts))] + [
-                SegmentValue.from_html("", text)
-                for text in texts
+                SegmentValue.from_html("", text) for text in texts
             ]
 
         elif isinstance(block_type, (ImageChooserBlock, SnippetChooserBlock)):
@@ -114,8 +113,7 @@ def extract_segments(instance):
             template, texts = extract_html_segments(field.value_from_object(instance))
 
             field_segments = [TemplateValue("", "html", template, len(texts))] + [
-                SegmentValue.from_html("", text)
-                for text in texts
+                SegmentValue.from_html("", text) for text in texts
             ]
 
             segments.extend(segment.wrap(field.name) for segment in field_segments)

wagtail_localize/segments/ingest.py

@@ -18,7 +18,11 @@ def organise_template_segments(segments):
     # The first segment is always the template, followed by the texts in order of their position
     segments.sort(key=lambda segment: segment.order)
     template = segments[0]
-    return template.format, template.template, [segment.html for segment in segments[1:]]
+    return (
+        template.format,
+        template.template,
+        [segment.html for segment in segments[1:]],
+    )
 
 
 def handle_related_object(related_object, src_locale, tgt_locale, segments):

wagtail_localize/segments/tests/test_segment_extraction.py

@@ -284,9 +284,7 @@ def test_listblock(self):
             segments,
             [
                 SegmentValue(f"test_streamfield.{block_id}", "Test content"),
-                SegmentValue(
-                    f"test_streamfield.{block_id}", "Some more test content"
-                ),
+                SegmentValue(f"test_streamfield.{block_id}", "Some more test content"),
             ],
         )
 

wagtail_localize/segments/tests/test_segment_ingestion.py

@@ -530,7 +530,9 @@ def test_listblock(self):
             self.src_locale,
             self.locale,
             [
-                SegmentValue(f"test_streamfield.{block_id}", "Tester le contenu", order=0),
+                SegmentValue(
+                    f"test_streamfield.{block_id}", "Tester le contenu", order=0
+                ),
                 SegmentValue(
                     f"test_streamfield.{block_id}", "Encore du contenu de test", order=1
                 ),

wagtail_localize/translation_engines/pontoon/importer.py

@@ -18,6 +18,7 @@
     Segment,
     SegmentLocation,
     TemplateLocation,
+    SegmentTranslationContext,
 )
 
 from .models import (
@@ -42,6 +43,11 @@ def import_resource(self, resource, language, old_po, new_po):
                 )
                 translation, created = segment.translations.get_or_create(
                     language=language,
+                    context=SegmentTranslationContext.get_from_string(
+                        changed_entry.msgctxt
+                    )
+                    if changed_entry.msgctxt
+                    else None,
                     defaults={
                         "text": changed_entry.msgstr,
                         "updated_at": timezone.now(),

wagtail_localize/translation_engines/pontoon/models.py

@@ -17,6 +17,7 @@
     TranslatableObject,
     TranslatableRevision,
     RelatedObjectLocation,
+    SegmentTranslation,
 )
 from wagtail_localize.translation_memory.utils import (
     insert_segments,
@@ -142,26 +143,22 @@ def get_segments(self):
         """
         Gets all segments that are in the latest submission to Pontoon.
         """
-        return Segment.objects.filter(locations__revision_id=self.current_revision_id)
+        return SegmentLocation.objects.filter(revision_id=self.current_revision_id)
 
-    def get_all_segments(self, annotate_obsolete=False):
+    def get_obsolete_translations(self, language):
         """
-        Gets all segments that have ever been submitted to Pontoon.
+        Gets all past translations for this resource that are not used in
+        the latest submission.
         """
-        segments = Segment.objects.filter(
-            locations__revision__pontoon_submission__resource_id=self.pk
-        )
-
-        if annotate_obsolete:
-            segments = segments.annotate(
-                is_obsolete=~Exists(
-                    SegmentLocation.objects.filter(
-                        segment=OuterRef("pk"), revision_id=self.current_revision_id,
-                    )
+        return SegmentTranslation.objects.annotate(
+            is_in_latest_submission=Exists(
+                SegmentLocation.objects.filter(
+                    revision_id=self.current_revision_id,
+                    segment_id=OuterRef("translation_of_id"),
+                    context_id=OuterRef("context_id"),
                 )
             )
-
-        return segments.distinct()
+        ).filter(language=language, is_in_latest_submission=False,)
 
     def find_translatable_submission(self, language):
         """

wagtail_localize/translation_engines/pontoon/pofile.py

@@ -13,8 +13,16 @@ def generate_source_pofile(resource):
         "Content-Type": "text/html; charset=utf-8",
     }
 
-    for segment in resource.get_segments().iterator():
-        po.append(polib.POEntry(msgid=segment.text, msgstr=""))
+    for segment in (
+        resource.get_segments().select_related("segment", "context").iterator()
+    ):
+        po.append(
+            polib.POEntry(
+                msgid=segment.segment.text,
+                msgstr="",
+                msgctxt=segment.context.as_string(),
+            )
+        )
 
     return str(po)
 
@@ -32,19 +40,36 @@ def generate_language_pofile(resource, language):
     }
 
     # Live segments
-    for segment in resource.get_segments().annotate_translation(language).iterator():
-        po.append(polib.POEntry(msgid=segment.text, msgstr=segment.translation or ""))
-
-    # Add any obsolete segments that have translations for future referene
     for segment in (
-        resource.get_all_segments(annotate_obsolete=True)
+        resource.get_segments()
+        .select_related("segment", "context")
         .annotate_translation(language)
-        .filter(is_obsolete=True, translation__isnull=False)
         .iterator()
     ):
         po.append(
             polib.POEntry(
-                msgid=segment.text, msgstr=segment.translation or "", obsolete=True
+                msgid=segment.segment.text,
+                msgstr=segment.translation or "",
+                msgctxt=segment.context.as_string(),
+            )
+        )
+
+    # Add any obsolete segments that have translations for future reference
+    # We find this by looking for obsolete contexts and annotate the latest
+    # translation for each one. Contexts that were never translated are
+    # excluded
+    for translation in (
+        resource.get_obsolete_translations(language)
+        .select_related("translation_of", "context")
+        .filter(context__isnull=False)
+        .iterator()
+    ):
+        po.append(
+            polib.POEntry(
+                msgid=translation.translation_of.text,
+                msgstr=translation.text or "",
+                msgctxt=translation.context.as_string(),
+                obsolete=True,
             )
         )
 

wagtail_localize/translation_engines/pontoon/tests/test_importer.py

@@ -29,7 +29,7 @@ def create_test_po(entries):
     }
 
     for entry in entries:
-        po.append(polib.POEntry(msgid=entry[0], msgstr=entry[1]))
+        po.append(polib.POEntry(msgctxt=entry[0], msgid=entry[1], msgstr=entry[2]))
 
     return str(po)
 
@@ -52,12 +52,24 @@ def test_importer(self):
         new_page_succeeded = False
 
         with self.subTest(stage="New page"):
-            po_v1 = create_test_po([("The test translatable field", "")]).encode(
-                "utf-8"
-            )
+            po_v1 = create_test_po(
+                [
+                    (
+                        f"{self.page.translation_key}:test_charfield",
+                        "The test translatable field",
+                        "",
+                    )
+                ]
+            ).encode("utf-8")
 
             po_v2 = create_test_po(
-                [("The test translatable field", "Le champ traduisible de test")]
+                [
+                    (
+                        f"{self.page.translation_key}:test_charfield",
+                        "The test translatable field",
+                        "Le champ traduisible de test",
+                    )
+                ]
             ).encode("utf-8")
 
             importer = Importer(Language.objects.default(), logging.getLogger("dummy"))
@@ -98,6 +110,7 @@ def test_importer(self):
             po_v3 = create_test_po(
                 [
                     (
+                        f"{self.page.translation_key}:test_charfield",
                         "The test translatable field",
                         "Le champ testable à traduire avec un contenu mis à jour",
                     )
@@ -146,12 +159,19 @@ def test_importer_doesnt_import_if_parent_not_translated(self):
         with self.subTest(stage="Create child page"):
             # Translate
             po_v1 = create_test_po(
-                [("The test child's translatable field", "")]
+                [
+                    (
+                        f"{child_page.translation_key}:test_charfield",
+                        "The test child's translatable field",
+                        "",
+                    )
+                ]
             ).encode("utf-8")
 
             po_v2 = create_test_po(
                 [
                     (
+                        f"{child_page.translation_key}:test_charfield",
                         "The test child's translatable field",
                         "Le champ traduisible de test",
                     )
@@ -187,12 +207,24 @@ def test_importer_doesnt_import_if_parent_not_translated(self):
             return
 
         with self.subTest(stage="Create parent page"):
-            po_v1 = create_test_po([("The test translatable field", "")]).encode(
-                "utf-8"
-            )
+            po_v1 = create_test_po(
+                [
+                    (
+                        f"{self.page.translation_key}:test_charfield",
+                        "The test translatable field",
+                        "",
+                    )
+                ]
+            ).encode("utf-8")
 
             po_v2 = create_test_po(
-                [("The test translatable field", "Le champ traduisible de test")]
+                [
+                    (
+                        f"{self.page.translation_key}:test_charfield",
+                        "The test translatable field",
+                        "Le champ traduisible de test",
+                    )
+                ]
             ).encode("utf-8")
 
             importer = Importer(Language.objects.default(), logging.getLogger("dummy"))
@@ -247,12 +279,24 @@ def test_importer_doesnt_import_if_dependency_not_translated(self):
         )
 
         with self.subTest(stage="New page"):
-            po_v1 = create_test_po([("The test translatable field", "")]).encode(
-                "utf-8"
-            )
+            po_v1 = create_test_po(
+                [
+                    (
+                        f"{self.page.translation_key}:test_charfield",
+                        "The test translatable field",
+                        "",
+                    )
+                ]
+            ).encode("utf-8")
 
             po_v2 = create_test_po(
-                [("The test translatable field", "Le champ traduisible de test")]
+                [
+                    (
+                        f"{self.page.translation_key}:test_charfield",
+                        "The test translatable field",
+                        "Le champ traduisible de test",
+                    )
+                ]
             ).encode("utf-8")
 
             importer = Importer(Language.objects.default(), logging.getLogger("dummy"))
@@ -284,11 +328,25 @@ def test_importer_doesnt_import_if_dependency_not_translated(self):
             return
 
         with self.subTest(stage="Create snippet"):
-            po_v1 = create_test_po([("Test content", "")]).encode("utf-8")
+            po_v1 = create_test_po(
+                [
+                    (
+                        f"{self.page.test_snippet.translation_key}:field",
+                        "Test content",
+                        "",
+                    )
+                ]
+            ).encode("utf-8")
 
-            po_v2 = create_test_po([("Test content", "Tester le contenu")]).encode(
-                "utf-8"
-            )
+            po_v2 = create_test_po(
+                [
+                    (
+                        f"{self.page.test_snippet.translation_key}:field",
+                        "Test content",
+                        "Tester le contenu",
+                    )
+                ]
+            ).encode("utf-8")
 
             importer = Importer(Language.objects.default(), logging.getLogger("dummy"))
             importer.start_import("0" * 39 + "1")
@@ -333,12 +391,19 @@ def setUp(self):
 
     def test_importer_rich_text(self):
         po_v1 = create_test_po(
-            [('<a id="a1">The <b>test</b> translatable field</a>.', "")]
+            [
+                (
+                    f"{self.page.translation_key}:test_richtextfield",
+                    '<a id="a1">The <b>test</b> translatable field</a>.',
+                    "",
+                )
+            ]
         ).encode("utf-8")
 
         po_v2 = create_test_po(
             [
                 (
+                    f"{self.page.translation_key}:test_richtextfield",
                     '<a id="a1">The <b>test</b> translatable field</a>.',
                     '<a id="a1">Le champ traduisible de <b>test</b></a>.',
                 )

wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py

@@ -9,6 +9,7 @@
 from wagtail_localize.test.models import TestPage
 from wagtail_localize.translation_memory.models import (
     Segment,
+    SegmentTranslationContext,
     SegmentTranslation,
     SegmentLocation,
 )
@@ -108,8 +109,14 @@ def test_generate_language_pofile_with_existing_obsolete_translation(self):
         segment.text_id = Segment.get_text_id(segment.text)
         segment.save()
 
+        context = SegmentTranslationContext.objects.get(
+            object_id=self.page.translation_key, path="test_charfield"
+        )
         SegmentTranslation.objects.create(
-            translation_of=segment, language=self.language, text="Du texte obsolète"
+            translation_of=segment,
+            context=context,
+            language=self.language,
+            text="Du texte obsolète",
         )
 
         # Create a new revision. This will create a new segment like how the current segment was before I changed it