From 2a12dfd8631ed229590489966cd01d2da53c173b Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 2 Jan 2020 12:53:19 +0000 Subject: [PATCH] WIP --- wagtail_localize/segments/extract.py | 6 +- wagtail_localize/segments/ingest.py | 6 +- .../segments/tests/test_segment_extraction.py | 4 +- .../segments/tests/test_segment_ingestion.py | 4 +- .../translation_engines/pontoon/importer.py | 6 + .../translation_engines/pontoon/models.py | 27 ++-- .../translation_engines/pontoon/pofile.py | 43 +++-- .../pontoon/tests/test_importer.py | 103 +++++++++--- .../pontoon/tests/test_pofile_generation.py | 9 +- .../0012_segmenttranslationcontext.py | 88 +++++++++++ ...0013_populate_segmenttranslationcontext.py | 76 +++++++++ .../migrations/0014_remove_location_paths.py | 19 +++ wagtail_localize/translation_memory/models.py | 147 ++++++++++++------ wagtail_localize/translation_memory/utils.py | 10 +- 14 files changed, 443 insertions(+), 105 deletions(-) create mode 100644 wagtail_localize/translation_memory/migrations/0012_segmenttranslationcontext.py create mode 100644 wagtail_localize/translation_memory/migrations/0013_populate_segmenttranslationcontext.py create mode 100644 wagtail_localize/translation_memory/migrations/0014_remove_location_paths.py diff --git a/wagtail_localize/segments/extract.py b/wagtail_localize/segments/extract.py index 205b62fb8..4a02f3555 100644 --- a/wagtail_localize/segments/extract.py +++ b/wagtail_localize/segments/extract.py @@ -28,8 +28,7 @@ def handle_block(self, block_type, block_value): template, texts = extract_html_segments(block_value.source) return [TemplateValue("", "html", template, len(texts))] + [ - SegmentValue.from_html("", text) - for text in texts + SegmentValue.from_html("", text) for text in texts ] elif isinstance(block_type, (ImageChooserBlock, SnippetChooserBlock)): @@ -114,8 +113,7 @@ def extract_segments(instance): template, texts = extract_html_segments(field.value_from_object(instance)) field_segments = [TemplateValue("", "html", template, len(texts))] + [ - SegmentValue.from_html("", text) - for text in texts + SegmentValue.from_html("", text) for text in texts ] segments.extend(segment.wrap(field.name) for segment in field_segments) diff --git a/wagtail_localize/segments/ingest.py b/wagtail_localize/segments/ingest.py index b28aec9ba..77a7b8171 100644 --- a/wagtail_localize/segments/ingest.py +++ b/wagtail_localize/segments/ingest.py @@ -18,7 +18,11 @@ def organise_template_segments(segments): # The first segment is always the template, followed by the texts in order of their position segments.sort(key=lambda segment: segment.order) template = segments[0] - return template.format, template.template, [segment.html for segment in segments[1:]] + return ( + template.format, + template.template, + [segment.html for segment in segments[1:]], + ) def handle_related_object(related_object, src_locale, tgt_locale, segments): diff --git a/wagtail_localize/segments/tests/test_segment_extraction.py b/wagtail_localize/segments/tests/test_segment_extraction.py index 3f37ad7d9..d19e5bccd 100644 --- a/wagtail_localize/segments/tests/test_segment_extraction.py +++ b/wagtail_localize/segments/tests/test_segment_extraction.py @@ -284,9 +284,7 @@ def test_listblock(self): segments, [ SegmentValue(f"test_streamfield.{block_id}", "Test content"), - SegmentValue( - f"test_streamfield.{block_id}", "Some more test content" - ), + SegmentValue(f"test_streamfield.{block_id}", "Some more test content"), ], ) diff --git a/wagtail_localize/segments/tests/test_segment_ingestion.py b/wagtail_localize/segments/tests/test_segment_ingestion.py index 61e6a50f7..8bd38fef9 100644 --- a/wagtail_localize/segments/tests/test_segment_ingestion.py +++ b/wagtail_localize/segments/tests/test_segment_ingestion.py @@ -530,7 +530,9 @@ def test_listblock(self): self.src_locale, self.locale, [ - SegmentValue(f"test_streamfield.{block_id}", "Tester le contenu", order=0), + SegmentValue( + f"test_streamfield.{block_id}", "Tester le contenu", order=0 + ), SegmentValue( f"test_streamfield.{block_id}", "Encore du contenu de test", order=1 ), diff --git a/wagtail_localize/translation_engines/pontoon/importer.py b/wagtail_localize/translation_engines/pontoon/importer.py index 9ad06f0b1..093e4aa89 100644 --- a/wagtail_localize/translation_engines/pontoon/importer.py +++ b/wagtail_localize/translation_engines/pontoon/importer.py @@ -18,6 +18,7 @@ Segment, SegmentLocation, TemplateLocation, + SegmentTranslationContext, ) from .models import ( @@ -42,6 +43,11 @@ def import_resource(self, resource, language, old_po, new_po): ) translation, created = segment.translations.get_or_create( language=language, + context=SegmentTranslationContext.get_from_string( + changed_entry.msgctxt + ) + if changed_entry.msgctxt + else None, defaults={ "text": changed_entry.msgstr, "updated_at": timezone.now(), diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index 5ebea840b..97c6ec8a5 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -17,6 +17,7 @@ TranslatableObject, TranslatableRevision, RelatedObjectLocation, + SegmentTranslation, ) from wagtail_localize.translation_memory.utils import ( insert_segments, @@ -142,26 +143,22 @@ def get_segments(self): """ Gets all segments that are in the latest submission to Pontoon. """ - return Segment.objects.filter(locations__revision_id=self.current_revision_id) + return SegmentLocation.objects.filter(revision_id=self.current_revision_id) - def get_all_segments(self, annotate_obsolete=False): + def get_obsolete_translations(self, language): """ - Gets all segments that have ever been submitted to Pontoon. + Gets all past translations for this resource that are not used in + the latest submission. """ - segments = Segment.objects.filter( - locations__revision__pontoon_submission__resource_id=self.pk - ) - - if annotate_obsolete: - segments = segments.annotate( - is_obsolete=~Exists( - SegmentLocation.objects.filter( - segment=OuterRef("pk"), revision_id=self.current_revision_id, - ) + return SegmentTranslation.objects.annotate( + is_in_latest_submission=Exists( + SegmentLocation.objects.filter( + revision_id=self.current_revision_id, + segment_id=OuterRef("translation_of_id"), + context_id=OuterRef("context_id"), ) ) - - return segments.distinct() + ).filter(language=language, is_in_latest_submission=False,) def find_translatable_submission(self, language): """ diff --git a/wagtail_localize/translation_engines/pontoon/pofile.py b/wagtail_localize/translation_engines/pontoon/pofile.py index 9ab49a6e9..fd0e15eae 100644 --- a/wagtail_localize/translation_engines/pontoon/pofile.py +++ b/wagtail_localize/translation_engines/pontoon/pofile.py @@ -13,8 +13,16 @@ def generate_source_pofile(resource): "Content-Type": "text/html; charset=utf-8", } - for segment in resource.get_segments().iterator(): - po.append(polib.POEntry(msgid=segment.text, msgstr="")) + for segment in ( + resource.get_segments().select_related("segment", "context").iterator() + ): + po.append( + polib.POEntry( + msgid=segment.segment.text, + msgstr="", + msgctxt=segment.context.as_string(), + ) + ) return str(po) @@ -32,19 +40,36 @@ def generate_language_pofile(resource, language): } # Live segments - for segment in resource.get_segments().annotate_translation(language).iterator(): - po.append(polib.POEntry(msgid=segment.text, msgstr=segment.translation or "")) - - # Add any obsolete segments that have translations for future referene for segment in ( - resource.get_all_segments(annotate_obsolete=True) + resource.get_segments() + .select_related("segment", "context") .annotate_translation(language) - .filter(is_obsolete=True, translation__isnull=False) .iterator() ): po.append( polib.POEntry( - msgid=segment.text, msgstr=segment.translation or "", obsolete=True + msgid=segment.segment.text, + msgstr=segment.translation or "", + msgctxt=segment.context.as_string(), + ) + ) + + # Add any obsolete segments that have translations for future reference + # We find this by looking for obsolete contexts and annotate the latest + # translation for each one. Contexts that were never translated are + # excluded + for translation in ( + resource.get_obsolete_translations(language) + .select_related("translation_of", "context") + .filter(context__isnull=False) + .iterator() + ): + po.append( + polib.POEntry( + msgid=translation.translation_of.text, + msgstr=translation.text or "", + msgctxt=translation.context.as_string(), + obsolete=True, ) ) diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py index 19fab1c09..7d14a80ee 100644 --- a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py +++ b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py @@ -29,7 +29,7 @@ def create_test_po(entries): } for entry in entries: - po.append(polib.POEntry(msgid=entry[0], msgstr=entry[1])) + po.append(polib.POEntry(msgctxt=entry[0], msgid=entry[1], msgstr=entry[2])) return str(po) @@ -52,12 +52,24 @@ def test_importer(self): new_page_succeeded = False with self.subTest(stage="New page"): - po_v1 = create_test_po([("The test translatable field", "")]).encode( - "utf-8" - ) + po_v1 = create_test_po( + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "", + ) + ] + ).encode("utf-8") po_v2 = create_test_po( - [("The test translatable field", "Le champ traduisible de test")] + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "Le champ traduisible de test", + ) + ] ).encode("utf-8") importer = Importer(Language.objects.default(), logging.getLogger("dummy")) @@ -98,6 +110,7 @@ def test_importer(self): po_v3 = create_test_po( [ ( + f"{self.page.translation_key}:test_charfield", "The test translatable field", "Le champ testable à traduire avec un contenu mis à jour", ) @@ -146,12 +159,19 @@ def test_importer_doesnt_import_if_parent_not_translated(self): with self.subTest(stage="Create child page"): # Translate po_v1 = create_test_po( - [("The test child's translatable field", "")] + [ + ( + f"{child_page.translation_key}:test_charfield", + "The test child's translatable field", + "", + ) + ] ).encode("utf-8") po_v2 = create_test_po( [ ( + f"{child_page.translation_key}:test_charfield", "The test child's translatable field", "Le champ traduisible de test", ) @@ -187,12 +207,24 @@ def test_importer_doesnt_import_if_parent_not_translated(self): return with self.subTest(stage="Create parent page"): - po_v1 = create_test_po([("The test translatable field", "")]).encode( - "utf-8" - ) + po_v1 = create_test_po( + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "", + ) + ] + ).encode("utf-8") po_v2 = create_test_po( - [("The test translatable field", "Le champ traduisible de test")] + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "Le champ traduisible de test", + ) + ] ).encode("utf-8") importer = Importer(Language.objects.default(), logging.getLogger("dummy")) @@ -247,12 +279,24 @@ def test_importer_doesnt_import_if_dependency_not_translated(self): ) with self.subTest(stage="New page"): - po_v1 = create_test_po([("The test translatable field", "")]).encode( - "utf-8" - ) + po_v1 = create_test_po( + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "", + ) + ] + ).encode("utf-8") po_v2 = create_test_po( - [("The test translatable field", "Le champ traduisible de test")] + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "Le champ traduisible de test", + ) + ] ).encode("utf-8") importer = Importer(Language.objects.default(), logging.getLogger("dummy")) @@ -284,11 +328,25 @@ def test_importer_doesnt_import_if_dependency_not_translated(self): return with self.subTest(stage="Create snippet"): - po_v1 = create_test_po([("Test content", "")]).encode("utf-8") + po_v1 = create_test_po( + [ + ( + f"{self.page.test_snippet.translation_key}:field", + "Test content", + "", + ) + ] + ).encode("utf-8") - po_v2 = create_test_po([("Test content", "Tester le contenu")]).encode( - "utf-8" - ) + po_v2 = create_test_po( + [ + ( + f"{self.page.test_snippet.translation_key}:field", + "Test content", + "Tester le contenu", + ) + ] + ).encode("utf-8") importer = Importer(Language.objects.default(), logging.getLogger("dummy")) importer.start_import("0" * 39 + "1") @@ -333,12 +391,19 @@ def setUp(self): def test_importer_rich_text(self): po_v1 = create_test_po( - [('The test translatable field.', "")] + [ + ( + f"{self.page.translation_key}:test_richtextfield", + 'The test translatable field.', + "", + ) + ] ).encode("utf-8") po_v2 = create_test_po( [ ( + f"{self.page.translation_key}:test_richtextfield", 'The test translatable field.', 'Le champ traduisible de test.', ) diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py b/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py index 50e0ab189..b07eef5b4 100644 --- a/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py +++ b/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py @@ -9,6 +9,7 @@ from wagtail_localize.test.models import TestPage from wagtail_localize.translation_memory.models import ( Segment, + SegmentTranslationContext, SegmentTranslation, SegmentLocation, ) @@ -108,8 +109,14 @@ def test_generate_language_pofile_with_existing_obsolete_translation(self): segment.text_id = Segment.get_text_id(segment.text) segment.save() + context = SegmentTranslationContext.objects.get( + object_id=self.page.translation_key, path="test_charfield" + ) SegmentTranslation.objects.create( - translation_of=segment, language=self.language, text="Du texte obsolète" + translation_of=segment, + context=context, + language=self.language, + text="Du texte obsolète", ) # Create a new revision. This will create a new segment like how the current segment was before I changed it diff --git a/wagtail_localize/translation_memory/migrations/0012_segmenttranslationcontext.py b/wagtail_localize/translation_memory/migrations/0012_segmenttranslationcontext.py new file mode 100644 index 000000000..55112aa5b --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0012_segmenttranslationcontext.py @@ -0,0 +1,88 @@ +# Generated by Django 2.2.9 on 2019-12-31 15:30 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize", "0002_initial_data"), + ("wagtail_localize_translation_memory", "0011_relatedobjectlocation"), + ] + + operations = [ + migrations.CreateModel( + name="SegmentTranslationContext", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("path_id", models.UUIDField()), + ("path", models.TextField()), + ( + "object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="wagtail_localize_translation_memory.TranslatableObject", + ), + ), + ], + options={"unique_together": {("object", "path_id")},}, + ), + migrations.AddField( + model_name="relatedobjectlocation", + name="context", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AddField( + model_name="segmentlocation", + name="context", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AddField( + model_name="segmenttranslation", + name="context", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="translations", + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AddField( + model_name="templatelocation", + name="context", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AlterUniqueTogether( + name="segmenttranslation", + unique_together={("language", "translation_of", "context")}, + ), + ] diff --git a/wagtail_localize/translation_memory/migrations/0013_populate_segmenttranslationcontext.py b/wagtail_localize/translation_memory/migrations/0013_populate_segmenttranslationcontext.py new file mode 100644 index 000000000..d55afc2f8 --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0013_populate_segmenttranslationcontext.py @@ -0,0 +1,76 @@ +# Generated by Django 2.2.9 on 2019-12-31 16:30 + +import uuid +from django.db import migrations + + +def get_path_id(path): + return uuid.uuid5(uuid.UUID("fcab004a-2b50-11ea-978f-2e728ce88125"), path) + + +def populate_segmenttranslationcontext(apps, schema_editor): + SegmentLocation = apps.get_model( + "wagtail_localize_translation_memory.SegmentLocation" + ) + TemplateLocation = apps.get_model( + "wagtail_localize_translation_memory.TemplateLocation" + ) + RelatedObjectLocation = apps.get_model( + "wagtail_localize_translation_memory.RelatedObjectLocation" + ) + SegmentTranslation = apps.get_model( + "wagtail_localize_translation_memory.SegmentTranslation" + ) + SegmentTranslationContext = apps.get_model( + "wagtail_localize_translation_memory.SegmentTranslationContext" + ) + + def update_location_model(model): + for location in model.objects.select_related("revision"): + context, created = SegmentTranslationContext.objects.get_or_create( + object_id=location.revision.object_id, + path_id=get_path_id(location.path), + defaults={"path": location.path,}, + ) + + location.context = context + location.save(update_fields=["context"]) + + update_location_model(SegmentLocation) + update_location_model(TemplateLocation) + update_location_model(RelatedObjectLocation) + + for translation in SegmentTranslation.objects.all(): + contexts = ( + SegmentLocation.objects.filter(segment_id=translation.segment_id) + .order_by("order_by") + .values_list("context_id", flat=True) + .distinct() + ) + + if len(contexts) > 0: + translation.context_id = contexts[0] + translation.save(update_fields=["context_id"]) + + # Create duplicate SegmentTranslations for remaining contexts + if len(contexts) > 1: + for context in contexts[1:]: + SegmentTranslation.objects.create( + translation_of_id=translation.translation_of_id, + language_id=translation.language_id, + context_id=context, + text=translation.text, + created_at=translation.created_at, + updated_at=translation.updated_at, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_translation_memory", "0012_segmenttranslationcontext"), + ] + + operations = [ + migrations.RunPython(populate_segmenttranslationcontext), + ] diff --git a/wagtail_localize/translation_memory/migrations/0014_remove_location_paths.py b/wagtail_localize/translation_memory/migrations/0014_remove_location_paths.py new file mode 100644 index 000000000..765e294b2 --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0014_remove_location_paths.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.7 on 2020-01-02 12:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "wagtail_localize_translation_memory", + "0013_populate_segmenttranslationcontext", + ), + ] + + operations = [ + migrations.RemoveField(model_name="relatedobjectlocation", name="path",), + migrations.RemoveField(model_name="segmentlocation", name="path",), + migrations.RemoveField(model_name="templatelocation", name="path",), + ] diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index 8b2b0007c..08d6054dc 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -118,12 +118,15 @@ def from_instance(cls, instance, force=False): if content_json == previous_revision.content_json: return previous_revision, False - return cls.objects.create( - object=object, - locale=instance.locale, - content_json=content_json, - created_at=timezone.now(), - ), True + return ( + cls.objects.create( + object=object, + locale=instance.locale, + content_json=content_json, + created_at=timezone.now(), + ), + True, + ) def as_instance(self): """ @@ -170,23 +173,29 @@ def create_or_update_translation(self, locale, translation=None): created = True # Fetch all translated segments - segment_locations = SegmentLocation.objects.filter( - revision=self - ).annotate_translation(locale.language) + segment_locations = ( + SegmentLocation.objects.filter(revision=self) + .annotate_translation(locale.language) + .select_related("context") + ) - template_locations = TemplateLocation.objects.filter( - revision=self - ).select_related("template") + template_locations = ( + TemplateLocation.objects.filter(revision=self) + .select_related("template") + .select_related("context") + ) - related_object_locations = RelatedObjectLocation.objects.filter( - revision=self - ).select_related("object") + related_object_locations = ( + RelatedObjectLocation.objects.filter(revision=self) + .select_related("object") + .select_related("context") + ) segments = [] for location in segment_locations: segment = SegmentValue.from_html( - location.path, location.translation + location.context.path, location.translation ).with_order(location.order) if location.html_attrs: segment.replace_html_attrs(json.loads(location.html_attrs)) @@ -196,7 +205,7 @@ def create_or_update_translation(self, locale, translation=None): for location in template_locations: template = location.template segment = TemplateValue( - location.path, + location.context.path, template.template_format, template.template, template.segment_count, @@ -206,7 +215,7 @@ def create_or_update_translation(self, locale, translation=None): for location in related_object_locations: segment = RelatedObjectValue( - location.path, + location.context.path, location.object.content_type, location.object.translation_key, order=location.order, @@ -232,22 +241,6 @@ def create_or_update_translation(self, locale, translation=None): return translation, created -class SegmentQuerySet(models.QuerySet): - def annotate_translation(self, language): - """ - Adds a 'translation' field to the segments containing the - text content of the segment translated into the specified - language. - """ - return self.annotate( - translation=Subquery( - SegmentTranslation.objects.filter( - translation_of_id=OuterRef("pk"), language_id=pk(language) - ).values("text") - ) - ) - - class Segment(models.Model): UUID_NAMESPACE = uuid.UUID("59ed7d1c-7eb5-45fa-9c8b-7a7057ed56d7") @@ -255,8 +248,6 @@ class Segment(models.Model): text_id = models.UUIDField() text = models.TextField() - objects = SegmentQuerySet.as_manager() - @classmethod def get_text_id(cls, text): return uuid.uuid5(cls.UUID_NAMESPACE, text) @@ -281,23 +272,69 @@ class Meta: unique_together = [("language", "text_id")] +class SegmentTranslationContext(models.Model): + object = models.ForeignKey( + TranslatableObject, on_delete=models.CASCADE, related_name="+" + ) + path_id = models.UUIDField() + path = models.TextField() + + class Meta: + unique_together = [ + ("object", "path_id"), + ] + + @classmethod + def get_path_id(cls, path): + return uuid.uuid5(uuid.UUID("fcab004a-2b50-11ea-978f-2e728ce88125"), path) + + def save(self, *args, **kwargs): + if self.path and self.path_id is None: + self.path_id = self.get_path_id(self.path) + + return super().save(*args, **kwargs) + + def as_string(self): + """ + Creates a string that can be used in the "msgctxt" field of PO files. + """ + return str(self.object_id) + ":" + self.path + + @classmethod + def get_from_string(cls, msgctxt): + """ + Looks for the SegmentTranslationContext that the given string represents. + """ + object_id, path = msgctxt.split(":") + path_id = cls.get_path_id(path) + return cls.objects.get(object_id=object_id, path_id=path_id) + + class SegmentTranslation(models.Model): translation_of = models.ForeignKey( Segment, on_delete=models.CASCADE, related_name="translations" ) language = models.ForeignKey("wagtail_localize.Language", on_delete=models.CASCADE) + context = models.ForeignKey( + SegmentTranslationContext, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="translations", + ) text = models.TextField() created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) class Meta: - unique_together = [("language", "translation_of")] + unique_together = [("language", "translation_of", "context")] @classmethod - def from_text(cls, translation_of, language, text): + def from_text(cls, translation_of, language, context, text): segment, created = cls.objects.get_or_create( translation_of=translation_of, language_id=pk(language), + context_id=pk(context), defaults={"text": text}, ) @@ -330,7 +367,13 @@ def from_template_value(cls, template_value): class BaseLocation(models.Model): revision = models.ForeignKey(TranslatableRevision, on_delete=models.CASCADE) - path = models.TextField() + context = models.ForeignKey( + SegmentTranslationContext, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="+", + ) order = models.PositiveIntegerField() class Meta: @@ -347,7 +390,9 @@ def annotate_translation(self, language): return self.annotate( translation=Subquery( SegmentTranslation.objects.filter( - translation_of_id=OuterRef("segment_id"), language_id=pk(language) + translation_of_id=OuterRef("segment_id"), + language_id=pk(language), + context_id=OuterRef("context_id"), ).values("text") ) ) @@ -380,10 +425,13 @@ class SegmentLocation(BaseLocation): @classmethod def from_segment_value(cls, revision, language, segment_value): segment = Segment.from_text(language, segment_value.html_with_ids) + context, context_created = SegmentTranslationContext.objects.get_or_create( + object_id=revision.object_id, path=segment_value.path, + ) segment_loc, created = cls.objects.get_or_create( - revision_id=pk(revision), - path=segment_value.path, + revision=revision, + context=context, order=segment_value.order, segment=segment, html_attrs=json.dumps(segment_value.get_html_attrs()), @@ -400,10 +448,13 @@ class TemplateLocation(BaseLocation): @classmethod def from_template_value(cls, revision, template_value): template = Template.from_template_value(template_value) + context, context_created = SegmentTranslationContext.objects.get_or_create( + object_id=revision.object_id, path=template_value.path, + ) template_loc, created = cls.objects.get_or_create( - revision_id=pk(revision), - path=template_value.path, + revision=revision, + context=context, order=template_value.order, template=template, ) @@ -418,9 +469,13 @@ class RelatedObjectLocation(BaseLocation): @classmethod def from_related_object_value(cls, revision, related_object_value): + context, context_created = SegmentTranslationContext.objects.get_or_create( + object_id=revision.object_id, path=related_object_value.path, + ) + related_object_loc, created = cls.objects.get_or_create( - revision_id=pk(revision), - path=related_object_value.path, + revision=revision, + context=context, order=related_object_value.order, object=TranslatableObject.objects.get_or_create( content_type=related_object_value.content_type, diff --git a/wagtail_localize/translation_memory/utils.py b/wagtail_localize/translation_memory/utils.py index e13fe3966..10e2a02a2 100644 --- a/wagtail_localize/translation_memory/utils.py +++ b/wagtail_localize/translation_memory/utils.py @@ -33,18 +33,16 @@ def get_translation_progress(revision_id, language): - The number of segments that have been translated into the language """ # Get QuerySet of Segments that need to be translated - required_segments = Segment.objects.filter( - id__in=SegmentLocation.objects.filter(revision_id=revision_id).values_list( - "segment_id" - ) - ) + required_segments = SegmentLocation.objects.filter(revision_id=revision_id) # Annotate each Segment with a flag that indicates whether the segment is translated # into the specified language required_segments = required_segments.annotate( is_translated=Exists( SegmentTranslation.objects.filter( - translation_of_id=OuterRef("pk"), language=language + translation_of_id=OuterRef("segment_id"), + context_id=OuterRef("context_id"), + language=language, ) ) )