From f4134bb5ae6a9293b7357fcf8d3a974d167b11d9 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Tue, 17 Dec 2019 16:32:56 +0000 Subject: [PATCH 01/21] Merge Pontoon integration --- docs/pontoon.md | 79 +++++ mkdocs.yml | 1 + setup.py | 6 + tox.ini | 2 +- wagtail_localize/test/settings.py | 1 + .../translation_engines/pontoon/__init__.py | 3 + .../translation_engines/pontoon/apps.py | 7 + .../translation_engines/pontoon/git.py | 149 ++++++++ .../translation_engines/pontoon/importer.py | 171 +++++++++ .../pontoon/management/__init__.py | 0 .../pontoon/management/commands/__init__.py | 0 .../commands/submit_whole_site_to_pontoon.py | 36 ++ .../management/commands/sync_pontoon.py | 19 + .../pontoon/migrations/0001_initial.py | 74 ++++ .../0002_pontoonresourcetranslation.py | 56 +++ .../pontoon/migrations/0003_synclog.py | 45 +++ .../migrations/0004_pontoonsynclogresource.py | 54 +++ .../migrations/0005_auto_20190923_1405.py | 16 + .../migrations/0006_auto_20191007_1700.py | 14 + .../pontoon/migrations/__init__.py | 0 .../translation_engines/pontoon/models.py | 330 ++++++++++++++++++ .../translation_engines/pontoon/pofile.py | 51 +++ .../translation_engines/pontoon/sync.py | 177 ++++++++++ .../wagtail_localize_pontoon/dashboard.html | 69 ++++ .../pontoon/tests/__init__.py | 0 .../pontoon/tests/test_importer.py | 260 ++++++++++++++ .../pontoon/tests/test_pofile_generation.py | 141 ++++++++ .../translation_engines/pontoon/views.py | 26 ++ .../pontoon/wagtail_hooks.py | 40 +++ 29 files changed, 1826 insertions(+), 1 deletion(-) create mode 100644 docs/pontoon.md create mode 100644 wagtail_localize/translation_engines/pontoon/__init__.py create mode 100644 wagtail_localize/translation_engines/pontoon/apps.py create mode 100644 wagtail_localize/translation_engines/pontoon/git.py create mode 100644 wagtail_localize/translation_engines/pontoon/importer.py create mode 100644 wagtail_localize/translation_engines/pontoon/management/__init__.py create mode 100644 wagtail_localize/translation_engines/pontoon/management/commands/__init__.py create mode 100644 wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py create mode 100644 wagtail_localize/translation_engines/pontoon/management/commands/sync_pontoon.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0001_initial.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0002_pontoonresourcetranslation.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0003_synclog.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0004_pontoonsynclogresource.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0005_auto_20190923_1405.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0006_auto_20191007_1700.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/__init__.py create mode 100644 wagtail_localize/translation_engines/pontoon/models.py create mode 100644 wagtail_localize/translation_engines/pontoon/pofile.py create mode 100644 wagtail_localize/translation_engines/pontoon/sync.py create mode 100644 wagtail_localize/translation_engines/pontoon/templates/wagtail_localize_pontoon/dashboard.html create mode 100644 wagtail_localize/translation_engines/pontoon/tests/__init__.py create mode 100644 wagtail_localize/translation_engines/pontoon/tests/test_importer.py create mode 100644 wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py create mode 100644 wagtail_localize/translation_engines/pontoon/views.py create mode 100644 wagtail_localize/translation_engines/pontoon/wagtail_hooks.py diff --git a/docs/pontoon.md b/docs/pontoon.md new file mode 100644 index 00000000..6277a216 --- /dev/null +++ b/docs/pontoon.md @@ -0,0 +1,79 @@ +# wagtail-localize-pontoon + +Use [Pontoon](https://pontoon.mozilla.org/) as a translation engine for [wagtail-localize](https://github.com/kaedroho/wagtail-localize). + +**Note: This project will be merged into `wagtail-localize` soon.** + +## Installation + +Install both `wagtail-localize` and `wagtail-localize-pontoon`, then add the following to your `INSTALLED_APPS`: + +```python +INSTALLED_APPS = [ + ... + 'wagtail_localize', + 'wagtail_localize.translation_memory', + 'wagtail_localize_pontoon', + ... +] +``` + +Then set the following settings: + +`WAGTAILLOCALIZE_PONTOON_GIT_URL` - This is a URL to an empty git repository where `wagtail-localize-pontoon` will push source strings and Pontoon will push back translations. +`WAGTAILLOCALIZE_PONTOON_GIT_CLONE_DIR` - The local directory where the git repository will be checked out. + +## Configuring page types + +Any page types that need to be translatable must inherit from `TranslatablePageMixin` and have `translated_fields` set to a list of field +names that are translatable. + +To migrate existing page types to be translatable, use the `BootstrapTranslatableMixin` clas sfrom `wagtail-localize` to help create the migrations. See docstring on that class for details. + +## Running initial sync + +When adding to an existing site, we firstly need to manually submit any existing content to Pontoon. + +Firstly run the `sync_languages` command. This creates `Language` objects for all the languages defined in your `LANGUAGES` setting. + +Then run the `submit_whole_site_to_pontoon` management command. This generates submissions for all live translatable pages on the site. + +Then finally run the `sync_pontoon` management command. This pushes the source strings to Pontoon. + +## How it works + +This relies heavily on `wagtail-localize`'s `translation_memory` module to track which source strings need to be translated in order to create/update a translated version of a page. + +### Creating submissions + +Pages are submitted to Pontoon when the English (US) version of any transltable page is published. Nothing is uploaded to git at this time (this would be done when the `sync_pontoon` management command is next run), but the following happens when the page is published: + + - All translatable segments are extracted from the page and saved into the `translation_memory.Segment` model. This model holds unique source strings, the locations where these strings appear on actual pages is stored in the `translation_memory.SegmentPageLocation` model. + - A `wagtail_localize_pontoon.PontoonResourceSubmission` is created to note which page revision needs to be submitted to Pontoon. + +Note: The `submit_whole_site_to_pontoon` command runs this process for all live translatable pages. + +### Pushing source strings to Pontoon + +The `sync_pontoon` command firstly fetches the git repo, then checks for and imports any translated strings. This is covered in the next section. + +After new translations are ingested (if there are any), it will rewrite all of the source and locale `.po` files based on the records in `PontoonResourceSubmission` and the segments/translations in translation memory. + +Note: All source strings are added into both the source `.pot` file and each locale-specific `.po` file, because Pontoon will not send back translations unless the source strings exist in both places. + +If a segment is no longer used on a page, it is removed from the source `.pot` file, but may be left in the locale-specific `.po` files if a translation existed for that string but will be flagged as obsolete. + +### Pulling translations from Pontoon + +At the beginning of the `sync_pontoon` command, the git repo is fetched and if there are any changes, a diff is performed between the new remote `HEAD` and the local `HEAD`. + +If any of the locale PO files have been modified, they will be parsed and any new/changed translations saved in the `translation_memory.SegmentTranslation` model. + +After a locale PO file is imported, the translation progress of the associated page is checked by making a query against the `translation_memory.{Segment,SegmentTranslation}` models. If the page is ready to be translated, it will create/update the translated version of the page and publish it. + +If a page is ready to be translated, but it's parent is not translated into the target language, the translation is delayed until the parent is translated. + +### Caveats + +- Any edits on translated pages will be overwritten if the original page is updated and translated again. +- If a page is translated but one of the strings is edited in pontoon, the new version of the string will not be pulled through automatically. The original page should be re-submitted to Pontoon again first. diff --git a/mkdocs.yml b/mkdocs.yml index e4cbed48..8935d79f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,3 +13,4 @@ nav: - Getting Started: - Setup: setup.md - Tutorial: tutorial.md + - How to Guides: pontoon.md diff --git a/setup.py b/setup.py index e25b3130..13bfac4b 100644 --- a/setup.py +++ b/setup.py @@ -42,6 +42,12 @@ 'testing': [ 'psycopg2>=2.6', ], + 'pontoon': [ + 'polib>=1.1,<2.0', + 'pygit2>=0.28,<0.29', + 'gitpython>=3.0,<4.0', + 'toml>=0.10,<0.11', + ], }, zip_safe=False, ) diff --git a/tox.ini b/tox.ini index d8e67dae..e02c335c 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ envlist = py{37}-dj{22,master}-wa{26,27}-{postgres} ignore = D100,D101,D102,D103,D105,D200,D202,D204,D205,D209,D400,D401,E303,E501,W503,N805,N806 [testenv] -install_command = pip install -e ".[testing]" -U {opts} {packages} +install_command = pip install -e ".[testing,pontoon]" -U {opts} {packages} commands = coverage run testmanage.py test basepython = diff --git a/wagtail_localize/test/settings.py b/wagtail_localize/test/settings.py index fa0063f4..e5139cf6 100644 --- a/wagtail_localize/test/settings.py +++ b/wagtail_localize/test/settings.py @@ -37,6 +37,7 @@ "wagtail_localize.admin.regions", "wagtail_localize.admin.workflow", "wagtail_localize.translation_memory", + "wagtail_localize.translation_engines.pontoon", "wagtail.contrib.search_promotions", "wagtail.contrib.forms", "wagtail.contrib.redirects", diff --git a/wagtail_localize/translation_engines/pontoon/__init__.py b/wagtail_localize/translation_engines/pontoon/__init__.py new file mode 100644 index 00000000..6a1b194a --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/__init__.py @@ -0,0 +1,3 @@ +default_app_config = ( + "wagtail_localize.translation_engines.pontoon.apps.WagtailLocalizePontoonAppConfig" +) diff --git a/wagtail_localize/translation_engines/pontoon/apps.py b/wagtail_localize/translation_engines/pontoon/apps.py new file mode 100644 index 00000000..a22e06d2 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class WagtailLocalizePontoonAppConfig(AppConfig): + label = "wagtail_localize_pontoon" + name = "wagtail_localize.translation_engines.pontoon" + verbose_name = "Wagtail Localize Pontoon translation engine" diff --git a/wagtail_localize/translation_engines/pontoon/git.py b/wagtail_localize/translation_engines/pontoon/git.py new file mode 100644 index 00000000..ae82bf29 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/git.py @@ -0,0 +1,149 @@ +import os.path + +import pygit2 +import toml +from git import Git, Repo + +from django.conf import settings + + +class Repository: + def __init__(self, pygit, gitpython): + self.pygit = pygit + self.gitpython = gitpython + + @classmethod + def open(cls): + git_clone_dir = settings.WAGTAILLOCALIZE_PONTOON_GIT_CLONE_DIR + + if not os.path.isdir(git_clone_dir): + git_url = settings.WAGTAILLOCALIZE_PONTOON_GIT_URL + Repo.clone_from(git_url, git_clone_dir, bare=True) + + return cls(pygit2.Repository(git_clone_dir), Repo(git_clone_dir)) + + def reader(self): + return RepositoryReader(self.pygit) + + def writer(self): + return RepositoryWriter(self.pygit) + + def pull(self): + self.gitpython.remotes.origin.fetch("+refs/heads/*:refs/remotes/origin/*") + new_head = self.pygit.lookup_reference("refs/remotes/origin/master") + self.pygit.head.set_target(new_head.target) + + def push(self): + self.gitpython.remotes.origin.push(["refs/heads/master"]) + + def get_changed_files(self, old_commit, new_commit): + """ + For each file that has changed, yields a three-tuple containing the filename, old content and new content + """ + if old_commit is not None and not self.pygit.descendant_of( + new_commit, old_commit + ): + raise ValueError("Second commit must be a descendant of first commit") + + old_index = pygit2.Index() + new_index = pygit2.Index() + if old_commit is not None: + old_tree = self.pygit.get(old_commit).tree + old_index.read_tree(old_tree) + else: + old_tree = self.pygit.get("4b825dc642cb6eb9a060e54bf8d69288fbee4904") + + new_tree = self.pygit.get(new_commit).tree + new_index.read_tree(new_tree) + + for patch in self.pygit.diff(old_tree, new_tree): + if patch.delta.status_char() != "M": + continue + + if not patch.delta.new_file.path.startswith("locales/"): + continue + + old_file_oid = old_index[patch.delta.old_file.path].oid + new_file_oid = new_index[patch.delta.new_file.path].oid + old_file = self.pygit.get(old_file_oid) + new_file = self.pygit.get(new_file_oid) + yield patch.delta.new_file.path, old_file.data, new_file.data + + def get_head_commit_id(self): + return self.pygit.head.target.hex + + +class RepositoryReader: + def __init__(self, repo): + self.repo = repo + self.index = pygit2.Index() + self.last_commit = self.repo.head.target + self.index.read_tree(self.repo.get(self.last_commit).tree) + + def read_file(self, filename): + oid = self.index[filename].oid + return self.repo.get(oid).data + + +class RepositoryWriter: + def __init__(self, repo): + self.repo = repo + self.index = pygit2.Index() + + def has_changes(self): + """ + Returns True if the contents of this writer is different to the repo's current HEAD + """ + tree = self.repo.get(self.index.write_tree(self.repo)) + diff = tree.diff_to_tree(self.repo.get(self.repo.head.target).tree) + return bool(diff) + + def write_file(self, filename, contents): + """ + Inserts a file into this writer + """ + blob = self.repo.create_blob(contents) + self.index.add(pygit2.IndexEntry(filename, blob, pygit2.GIT_FILEMODE_BLOB)) + + def write_config(self, languages, paths): + self.write_file( + "l10n.toml", + toml.dumps( + { + "locales": languages, + "paths": [ + {"reference": str(source_path), "l10n": str(locale_path)} + for source_path, locale_path in paths + ], + } + ), + ) + + def copy_unmanaged_files(self, reader): + """ + Copies any files we don't manage from the specified reader. + + This is everything excluding l10n.toml, templates and locales. + """ + for entry in reader.index: + if ( + entry.path == "l10n.toml" + or entry.path.startswith("templates/") + or entry.path.startswith("locales/") + ): + continue + + self.index.add(entry) + + def commit(self, message): + """ + Creates a new commit with the contents of this writer + """ + tree = self.index.write_tree(self.repo) + + sig = pygit2.Signature( + "Wagtail Localize", "wagtail_localize_pontoon@wagtail.io" + ) + self.repo.create_commit( + "refs/heads/master", sig, sig, message, tree, [self.repo.head.target] + ) diff --git a/wagtail_localize/translation_engines/pontoon/importer.py b/wagtail_localize/translation_engines/pontoon/importer.py new file mode 100644 index 00000000..7de88b8a --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/importer.py @@ -0,0 +1,171 @@ +import json + +from django.db import transaction +from django.utils import timezone +from django.utils.text import slugify +import polib + +from wagtail.core.models import Page +from wagtail_localize.models import Locale, Region, ParentNotTranslatedError +from wagtail_localize.segments import SegmentValue, TemplateValue +from wagtail_localize.segments.ingest import ingest_segments +from wagtail_localize.translation_memory.models import ( + Segment, + SegmentPageLocation, + TemplatePageLocation, +) + +from .models import ( + PontoonResource, + PontoonResourceTranslation, + PontoonSyncLog, + PontoonSyncLogResource, +) + + +class Importer: + def __init__(self, source_language, logger): + self.source_language = source_language + self.logger = logger + self.log = None + + def create_or_update_translated_page(self, submission, language): + """ + Creates/updates the translated page to reflect the translations in translation memory. + + Note, all strings in the submission must be translated into the target language! + """ + locale = Locale.objects.get( + region_id=Region.objects.default_id(), language=language + ) + + page = submission.revision.as_page_object() + + try: + translated_page = page.get_translation(locale) + created = False + except page.specific_class.DoesNotExist: + # May raise ParentNotTranslatedError + translated_page = page.copy_for_translation(locale) + created = True + + # Fetch all translated segments + segment_page_locations = SegmentPageLocation.objects.filter( + page_revision=submission.revision + ).annotate_translation(language) + + template_page_locations = TemplatePageLocation.objects.filter( + page_revision=submission.revision + ).select_related("template") + + segments = [] + + for page_location in segment_page_locations: + segment = SegmentValue.from_html( + page_location.path, page_location.translation + ) + if page_location.html_attrs: + segment.replace_html_attrs(json.loads(page_location.html_attrs)) + + segments.append(segment) + + for page_location in template_page_locations: + template = page_location.template + segment = TemplateValue( + page_location.path, + template.template_format, + template.template, + template.segment_count, + ) + segments.append(segment) + + # Ingest all translated segments into page + ingest_segments(page, translated_page, page.locale.language, language, segments) + + # Make sure the slug is valid + translated_page.slug = slugify(translated_page.slug) + translated_page.save() + + new_revision = translated_page.save_revision() + new_revision.publish() + + PontoonResourceTranslation.objects.create( + submission=submission, language=language, revision=new_revision + ) + + return new_revision, created + + def import_resource(self, resource, language, old_po, new_po): + for changed_entry in set(new_po) - set(old_po): + try: + segment = Segment.objects.get( + language=self.source_language, text=changed_entry.msgid + ) + translation, created = segment.translations.get_or_create( + language=language, + defaults={ + "text": changed_entry.msgstr, + "updated_at": timezone.now(), + }, + ) + + if not created: + # Update the translation only if the text has changed + if translation.text != changed_entry.msgstr: + translation.text = changed_entry.msgstr + translation.updated_at = timezone.now() + translation.save() + + except Segment.DoesNotExist: + self.logger.warning(f"Unrecognised segment '{changed_entry.msgid}'") + + def try_update_resource_translation(self, resource, language): + # Check if there is a submission ready to be translated + translatable_submission = resource.find_translatable_submission(language) + + if translatable_submission: + self.logger.info( + f"Saving translated page for '{resource.page.title}' in {language.get_display_name()}" + ) + + try: + revision, created = self.create_or_update_translated_page( + translatable_submission, language + ) + except ParentNotTranslatedError: + # These pages will be handled when the parent is created in the code below + self.logger.info( + f"Cannot save translated page for '{resource.page.title}' in {language.get_display_name()} yet as its parent must be translated first" + ) + return + + if created: + # Check if this page has any children that may be ready to translate + child_page_resources = PontoonResource.objects.filter( + page__in=resource.page.get_children() + ) + + for resource in child_page_resources: + self.try_update_resource_translation(resource, language) + + def start_import(self, commit_id): + self.log = PontoonSyncLog.objects.create( + action=PontoonSyncLog.ACTION_PULL, commit_id=commit_id + ) + + def import_file(self, filename, old_content, new_content): + self.logger.info(f"Pull: Importing changes in file '{filename}'") + resource, language = PontoonResource.get_by_po_filename(filename) + + # Log that this resource was updated + PontoonSyncLogResource.objects.create( + log=self.log, resource=resource, language=language + ) + + old_po = polib.pofile(old_content.decode("utf-8")) + new_po = polib.pofile(new_content.decode("utf-8")) + + self.import_resource(resource, language, old_po, new_po) + + # Check if the translated page is ready to be created/updated + self.try_update_resource_translation(resource, language) diff --git a/wagtail_localize/translation_engines/pontoon/management/__init__.py b/wagtail_localize/translation_engines/pontoon/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wagtail_localize/translation_engines/pontoon/management/commands/__init__.py b/wagtail_localize/translation_engines/pontoon/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py b/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py new file mode 100644 index 00000000..ec8234f4 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py @@ -0,0 +1,36 @@ +from django.core.management.base import BaseCommand, CommandError + +from wagtail.core.models import Page +from wagtail_localize.models import TranslatablePageMixin, Locale + +from ...models import submit_to_pontoon, PontoonResourceSubmission + + +class Command(BaseCommand): + def handle(self, **options): + source_locale_id = Locale.objects.default_id() + + for page in Page.objects.live().specific().iterator(): + if not isinstance(page, TranslatablePageMixin): + continue + + if page.locale_id != source_locale_id: + continue + + if page.live_revision is not None: + print( + f"Warning: The page '{page.title}' does not have a live_revision. Using latest revision instead." + ) + revision = page.live_revision + else: + revision = page.get_latest_revision() or page.save_revision( + changed=False + ) + + print(f"Submitting '{page.title}'") + + if PontoonResourceSubmission.objects.filter(revision=revision).exists(): + print("Already submitted!") + continue + + submit_to_pontoon(page, revision) diff --git a/wagtail_localize/translation_engines/pontoon/management/commands/sync_pontoon.py b/wagtail_localize/translation_engines/pontoon/management/commands/sync_pontoon.py new file mode 100644 index 00000000..94446cd8 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/management/commands/sync_pontoon.py @@ -0,0 +1,19 @@ +import logging + +from django.core.management.base import BaseCommand + +from ...sync import SyncManager + + +class Command(BaseCommand): + def handle(self, **options): + logger = logging.getLogger(__name__) + + # Enable logging to console + console = logging.StreamHandler() + console.setLevel(logging.INFO) + console.setFormatter(logging.Formatter("%(levelname)s - %(message)s")) + logger.addHandler(console) + logger.setLevel(logging.INFO) + + SyncManager(logger=logger).sync() diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0001_initial.py b/wagtail_localize/translation_engines/pontoon/migrations/0001_initial.py new file mode 100644 index 00000000..15a1f8b7 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0001_initial.py @@ -0,0 +1,74 @@ +# Generated by Django 2.1.10 on 2019-08-13 12:30 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("wagtailcore", "0041_group_collection_permissions_verbose_name_plural") + ] + + operations = [ + migrations.CreateModel( + name="PontoonResource", + fields=[ + ( + "page", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="+", + serialize=False, + to="wagtailcore.Page", + ), + ), + ("path", models.CharField(max_length=255, unique=True)), + ( + "current_revision", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtailcore.PageRevision", + ), + ), + ], + ), + migrations.CreateModel( + name="PontoonResourceSubmission", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("pushed_at", models.DateTimeField(null=True)), + ("pushed_commit_sha", models.CharField(blank=True, max_length=40)), + ( + "resource", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="submissions", + to="wagtail_localize_pontoon.PontoonResource", + ), + ), + ( + "revision", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="pontoon_submission", + to="wagtailcore.PageRevision", + ), + ), + ], + ), + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0002_pontoonresourcetranslation.py b/wagtail_localize/translation_engines/pontoon/migrations/0002_pontoonresourcetranslation.py new file mode 100644 index 00000000..c2feea69 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0002_pontoonresourcetranslation.py @@ -0,0 +1,56 @@ +# Generated by Django 2.2.4 on 2019-08-14 14:49 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize", "0002_initial_data"), + ("wagtailcore", "0041_group_collection_permissions_verbose_name_plural"), + ("wagtail_localize_pontoon", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="PontoonResourceTranslation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "language", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="pontoon_translations", + to="wagtail_localize.Language", + ), + ), + ( + "revision", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="pontoon_translation", + to="wagtailcore.PageRevision", + ), + ), + ( + "submission", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="translations", + to="wagtail_localize_pontoon.PontoonResourceSubmission", + ), + ), + ], + options={"unique_together": {("submission", "language")}}, + ) + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0003_synclog.py b/wagtail_localize/translation_engines/pontoon/migrations/0003_synclog.py new file mode 100644 index 00000000..74e32cb6 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0003_synclog.py @@ -0,0 +1,45 @@ +# Generated by Django 2.2.5 on 2019-09-03 10:49 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [("wagtail_localize_pontoon", "0002_pontoonresourcetranslation")] + + operations = [ + migrations.CreateModel( + name="PontoonSyncLog", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "action", + models.PositiveIntegerField(choices=[(1, "Push"), (2, "Pull")]), + ), + ("time", models.DateTimeField(auto_now_add=True)), + ("commit_id", models.CharField(max_length=40)), + ], + ), + migrations.RemoveField( + model_name="pontoonresourcesubmission", name="pushed_commit_sha" + ), + migrations.AddField( + model_name="pontoonresourcesubmission", + name="push_log", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="pushed_submissions", + to="wagtail_localize_pontoon.PontoonSyncLog", + ), + ), + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0004_pontoonsynclogresource.py b/wagtail_localize/translation_engines/pontoon/migrations/0004_pontoonsynclogresource.py new file mode 100644 index 00000000..dd3c858a --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0004_pontoonsynclogresource.py @@ -0,0 +1,54 @@ +# Generated by Django 2.2.5 on 2019-09-03 14:30 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize", "0002_initial_data"), + ("wagtail_localize_pontoon", "0003_synclog"), + ] + + operations = [ + migrations.CreateModel( + name="PontoonSyncLogResource", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "language", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="wagtail_localize.Language", + ), + ), + ( + "log", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="resources", + to="wagtail_localize_pontoon.PontoonSyncLog", + ), + ), + ( + "resource", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="logs", + to="wagtail_localize_pontoon.PontoonResource", + ), + ), + ], + ) + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0005_auto_20190923_1405.py b/wagtail_localize/translation_engines/pontoon/migrations/0005_auto_20190923_1405.py new file mode 100644 index 00000000..533953bf --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0005_auto_20190923_1405.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.5 on 2019-09-23 14:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("wagtail_localize_pontoon", "0004_pontoonsynclogresource")] + + operations = [ + migrations.AlterField( + model_name="pontoonsynclog", + name="commit_id", + field=models.CharField(blank=True, max_length=40), + ) + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0006_auto_20191007_1700.py b/wagtail_localize/translation_engines/pontoon/migrations/0006_auto_20191007_1700.py new file mode 100644 index 00000000..596d0dda --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0006_auto_20191007_1700.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.6 on 2019-10-07 17:00 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("wagtail_localize_pontoon", "0005_auto_20190923_1405")] + + operations = [ + migrations.AlterUniqueTogether( + name="pontoonresourcetranslation", unique_together=set() + ) + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/__init__.py b/wagtail_localize/translation_engines/pontoon/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py new file mode 100644 index 00000000..dcfbcf1a --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -0,0 +1,330 @@ +from pathlib import PurePosixPath + +from django.conf import settings +from django.db import models, transaction +from django.db.models import Exists, OuterRef +from django.dispatch import receiver +from django.utils import timezone + +from wagtail.core.signals import page_published +from wagtail_localize.models import TranslatablePageMixin, Locale, Language +from wagtail_localize.segments.extract import extract_segments +from wagtail_localize.translation_memory.models import Segment, SegmentPageLocation +from wagtail_localize.translation_memory.utils import ( + insert_segments, + get_translation_progress, +) + + +class PontoonSyncLog(models.Model): + ACTION_PUSH = 1 + ACTION_PULL = 2 + + ACTION_CHOICES = [(ACTION_PUSH, "Push"), (ACTION_PULL, "Pull")] + + action = models.PositiveIntegerField(choices=ACTION_CHOICES) + time = models.DateTimeField(auto_now_add=True) + commit_id = models.CharField(max_length=40, blank=True) + + +class PontoonResource(models.Model): + page = models.OneToOneField( + "wagtailcore.Page", on_delete=models.CASCADE, primary_key=True, related_name="+" + ) + + # The path within the locale folder in the git repository to push the PO file to + # This is initially the pages URL path but is not updated if the page is moved + path = models.CharField(max_length=255, unique=True) + + # The last revision to be submitted for translation + # This is denormalised from the revision field of the latest submission for this resource + current_revision = models.OneToOneField( + "wagtailcore.PageRevision", + on_delete=models.SET_NULL, + null=True, + related_name="+", + ) + + @classmethod + def get_unique_path_from_urlpath(cls, url_path): + """ + Returns a unique path derived from the given url path taken from a Page object. + + We never change paths, even if a page has been moved in order to not confuse Pontoon. + + But this means that if pages are moved and a new one is created with the same slug in + it's previous position, a path name clash could occur. + """ + path = url_path.strip("/") + + # We're not using the 'fast' technique Wagtail uses to find unique slugs here. This is because + # a prefix query on path could potentially lead to a massive amount of results being returned. + path_suffix = 1 + try_path = path + while cls.objects.filter(path=try_path).exists(): + try_path = f"{path}-{path_suffix}" + path_suffix += 1 + + if path_suffix > 100: + # Unlikely to get here, but I feel uncomfortable about leaving this loop unrestrained! + raise Exception(f"Unable to find a unique path for: {url_path}") + + return try_path + + def get_po_filename(self, language=None): + """ + Returns the filename of this resource within the git repository that is shared with Pontoon. + + If a language is specified, the filename for that language is returned. Otherwise, the filename + of the template is returned. + """ + if language is not None: + base_path = PurePosixPath(f"locales/{language.as_rfc5646_language_tag()}") + else: + base_path = PurePosixPath("templates") + + return (base_path / self.path).with_suffix( + ".pot" if language is None else ".po" + ) + + def get_locale_po_filename_template(self): + """ + Returns the template used for language-specific files for this resource. + + This value is passed to Pontoon in the configuration so it can find the language-specific files. + """ + return (PurePosixPath("locales/{locale}") / self.path).with_suffix(".po") + + @classmethod + def get_by_po_filename(cls, filename): + """ + Finds the resource/language for the given filename of a PO file in the git repo. + + May raise PontoonResource.DoesNotExist or Language.DoesNotExist. + """ + parts = PurePosixPath(filename).parts + if parts[0] == "templates": + path = PurePosixPath(*parts[1:]).with_suffix("") + return cls.objects.get(path=path), None + elif parts[0] == "locales": + path = PurePosixPath(*parts[2:]).with_suffix("") + return ( + cls.objects.get(path=path), + # NOTE: Pontoon Uses RFC 5646 style language tags + # but since this is a site that relies on Pontoon + # for translation, we assumme that Language model + # uses RFC 5646 style language tags as well. + # If this isn't the case, then the following + # commented-out line should be used instead. + # + # TODO: Think of a way to make this configurable. + # + # Language.get_by_rfc5646_language_tag(parts[1]), + Language.objects.get(code=parts[1]), + ) + + raise cls.DoesNotExist( + "Filename must begin with either 'templates' or 'locales'" + ) + + def get_segments(self): + """ + Gets all segments that are in the latest submission to Pontoon. + """ + return Segment.objects.filter( + page_locations__page_revision_id=self.current_revision_id + ) + + def get_all_segments(self, annotate_obsolete=False): + """ + Gets all segments that have ever been submitted to Pontoon. + """ + segments = Segment.objects.filter( + page_locations__page_revision__pontoon_submission__resource_id=self.pk + ) + + if annotate_obsolete: + segments = segments.annotate( + is_obsolete=~Exists( + SegmentPageLocation.objects.filter( + segment=OuterRef("pk"), + page_revision_id=self.current_revision_id, + ) + ) + ) + + return segments.distinct() + + def find_translatable_submission(self, language): + """ + Look to see if a submission is ready for translating. + + The returned submission will be the latest submission that is ready for translation, after any previous + translated submission. + + A submission is considered translatable if all the strings in the submission have been translated into the + target language and the translated page hasn't been updated for a later submission. + """ + submissions_to_check = self.submissions.order_by("-created_at") + + # Exclude submissions that pre-date the last translated submission + last_translated_submission = ( + self.submissions.annotate_translated(language) + .filter(is_translated=True) + .order_by("created_at") + .last() + ) + if last_translated_submission is not None: + submissions_to_check = submissions_to_check.filter( + created_at__gte=last_translated_submission.created_at + ) + + for submission in submissions_to_check: + total_segments, translated_segments = submission.get_translation_progress( + language + ) + + if translated_segments == total_segments: + return submission + + def latest_submission(self): + return self.submissions.latest("created_at") + + def latest_pushed_submission(self): + return self.submissions.filter(pushed_at__isnull=False).latest("created_at") + + def __repr__(self): + return f"" + + +class PontoonSyncLogResourceQuerySet(models.QuerySet): + def unique_resources(self): + return PontoonResource.objects.filter( + page_id__in=self.values_list("resource_id", flat=True) + ) + + def unique_languages(self): + return Language.objects.filter( + id__in=self.values_list("language_id", flat=True) + ) + + +class PontoonSyncLogResource(models.Model): + log = models.ForeignKey( + PontoonSyncLog, on_delete=models.CASCADE, related_name="resources" + ) + resource = models.ForeignKey( + PontoonResource, on_delete=models.CASCADE, related_name="logs" + ) + + # Null if pushing this resource, otherwise set to the language being pulled + language = models.ForeignKey( + "wagtail_localize.Language", + null=True, + on_delete=models.CASCADE, + related_name="+", + ) + + objects = PontoonSyncLogResourceQuerySet.as_manager() + + +class PontoonResourceSubmissionQuerySet(models.QuerySet): + def annotate_translated(self, language): + """ + Adds is_translated flag which is True if the submission has been translated into the specified language. + """ + return self.annotate( + is_translated=Exists( + PontoonResourceTranslation.objects.filter( + submission_id=OuterRef("pk"), language=language + ) + ) + ) + + +class PontoonResourceSubmission(models.Model): + resource = models.ForeignKey( + PontoonResource, on_delete=models.CASCADE, related_name="submissions" + ) + revision = models.OneToOneField( + "wagtailcore.PageRevision", + on_delete=models.CASCADE, + related_name="pontoon_submission", + ) + created_at = models.DateTimeField(auto_now_add=True) + # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT, related_name='+') + + pushed_at = models.DateTimeField(null=True) + push_log = models.ForeignKey( + PontoonSyncLog, + null=True, + on_delete=models.SET_NULL, + related_name="pushed_submissions", + ) + + objects = PontoonResourceSubmissionQuerySet.as_manager() + + def get_translation_progress(self, language): + """ + Get the current translation progress into the specified language. + + Returns two integers: + - The total number of segments in the submission to translate + - The number of segments that have been translated into the target language + """ + return get_translation_progress(self.revision_id, language) + + +class PontoonResourceTranslation(models.Model): + """ + Represents a translation that was sucessfully carried out by pontoon. + """ + + submission = models.ForeignKey( + PontoonResourceSubmission, on_delete=models.CASCADE, related_name="translations" + ) + language = models.ForeignKey( + "wagtail_localize.Language", + on_delete=models.CASCADE, + related_name="pontoon_translations", + ) + + # The revision of the page that was created when the translations were saved + revision = models.OneToOneField( + "wagtailcore.PageRevision", + on_delete=models.CASCADE, + related_name="pontoon_translation", + ) + + created_at = models.DateTimeField(auto_now_add=True) + + +@transaction.atomic +def submit_to_pontoon(page, revision): + # Extract segments from page and save them to translation memory + insert_segments(revision, page.locale.language_id, extract_segments(page)) + + # Get/create resource + try: + resource = PontoonResource.objects.get(page=page) + resource.current_revision = revision + resource.save(update_fields=["current_revision"]) + except PontoonResource.DoesNotExist: + resource = PontoonResource.objects.create( + page=page, + current_revision=revision, + path=PontoonResource.get_unique_path_from_urlpath(page.url_path), + ) + + # Create submission + resource.submissions.create(revision=revision) + + +@receiver(page_published) +def submit_page_to_pontoon(sender, **kwargs): + if issubclass(sender, TranslatablePageMixin): + page = kwargs["instance"] + revision = kwargs["revision"] + + if page.locale_id == Locale.objects.default_id(): + submit_to_pontoon(page, revision) diff --git a/wagtail_localize/translation_engines/pontoon/pofile.py b/wagtail_localize/translation_engines/pontoon/pofile.py new file mode 100644 index 00000000..9ab49a6e --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/pofile.py @@ -0,0 +1,51 @@ +import polib +from django.utils import timezone + + +def generate_source_pofile(resource): + """ + Generate a source PO file for the given resource + """ + po = polib.POFile(wrapwidth=200) + po.metadata = { + "POT-Creation-Date": str(timezone.now()), + "MIME-Version": "1.0", + "Content-Type": "text/html; charset=utf-8", + } + + for segment in resource.get_segments().iterator(): + po.append(polib.POEntry(msgid=segment.text, msgstr="")) + + return str(po) + + +def generate_language_pofile(resource, language): + """ + Generate a translated PO file for the given resource/language + """ + po = polib.POFile(wrapwidth=200) + po.metadata = { + "POT-Creation-Date": str(timezone.now()), + "MIME-Version": "1.0", + "Content-Type": "text/html; charset=utf-8", + "Language": language.as_rfc5646_language_tag(), + } + + # Live segments + for segment in resource.get_segments().annotate_translation(language).iterator(): + po.append(polib.POEntry(msgid=segment.text, msgstr=segment.translation or "")) + + # Add any obsolete segments that have translations for future referene + for segment in ( + resource.get_all_segments(annotate_obsolete=True) + .annotate_translation(language) + .filter(is_obsolete=True, translation__isnull=False) + .iterator() + ): + po.append( + polib.POEntry( + msgid=segment.text, msgstr=segment.translation or "", obsolete=True + ) + ) + + return str(po) diff --git a/wagtail_localize/translation_engines/pontoon/sync.py b/wagtail_localize/translation_engines/pontoon/sync.py new file mode 100644 index 00000000..ad4ab387 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/sync.py @@ -0,0 +1,177 @@ +import logging + +from django.conf import settings +from django.db import transaction +from django.db.models import F +from django.utils import timezone +from django.utils.module_loading import import_string +import polib + +from wagtail_localize.models import Language, ParentNotTranslatedError +from wagtail_localize.translation_memory.models import Segment + +from .git import Repository +from .models import ( + PontoonResourceSubmission, + PontoonResource, + PontoonSyncLog, + PontoonSyncLogResource, +) +from .pofile import generate_source_pofile, generate_language_pofile +from .importer import Importer + + +@transaction.atomic +def _pull(repo, logger): + # Get the last commit ID that we either pulled or pushed + last_log = PontoonSyncLog.objects.order_by("-time").exclude(commit_id="").first() + last_commit_id = None + if last_log is not None: + last_commit_id = last_log.commit_id + + current_commit_id = repo.get_head_commit_id() + + if last_commit_id == current_commit_id: + logger.info("Pull: No changes since last sync") + return + + importer = Importer(Language.objects.default(), logger) + importer.start_import(current_commit_id) + for filename, old_content, new_content in repo.get_changed_files( + last_commit_id, repo.get_head_commit_id() + ): + importer.import_file(filename, old_content, new_content) + + +@transaction.atomic +def _push(repo, logger): + reader = repo.reader() + writer = repo.writer() + writer.copy_unmanaged_files(reader) + + def update_po(filename, new_po_string): + try: + current_po_string = reader.read_file(filename).decode("utf-8") + current_po = polib.pofile(current_po_string, wrapwidth=200) + + # Take metadata from existing PO file + new_po = polib.pofile(new_po_string, wrapwidth=200) + new_po.metadata = current_po.metadata + new_po_string = str(new_po) + + except KeyError: + pass + + writer.write_file(filename, new_po_string) + + languages = Language.objects.filter(is_active=True).exclude( + id=Language.objects.default_id() + ) + + paths = [] + pushed_submission_ids = [] + for submission in ( + PontoonResourceSubmission.objects.filter( + revision_id=F("resource__current_revision_id") + ) + .select_related("resource") + .order_by("resource__path") + ): + source_po = generate_source_pofile(submission.resource) + update_po(str(submission.resource.get_po_filename()), source_po) + + for language in languages: + locale_po = generate_language_pofile(submission.resource, language) + update_po( + str(submission.resource.get_po_filename(language=language)), locale_po + ) + + paths.append( + ( + submission.resource.get_po_filename(), + submission.resource.get_locale_po_filename_template(), + ) + ) + + pushed_submission_ids.append(submission.id) + + writer.write_config( + [language.as_rfc5646_language_tag() for language in languages], paths + ) + + # A queryset of submissions we've just written that haven't been pushed before + pushed_submissions = PontoonResourceSubmission.objects.filter( + id__in=pushed_submission_ids, push_log__isnull=True + ) + + if pushed_submissions.exists(): + # Create a new log for this push + log = PontoonSyncLog.objects.create( + action=PontoonSyncLog.ACTION_PUSH, commit_id="" + ) + + # Add an entry for each resource we just pushed + for resource_id in pushed_submissions.values_list("resource_id", flat=True): + PontoonSyncLogResource.objects.create(log=log, resource_id=resource_id) + + pushed_submissions.update(pushed_at=timezone.now(), push_log=log) + + if writer.has_changes(): + logger.info("Push: Committing changes") + writer.commit("Updates to source content") + + log.commit_id = repo.get_head_commit_id() + log.save(update_fields=["commit_id"]) + else: + logger.info( + "Push: Not committing anything as recent changes haven't affected any translatable content" + ) + + repo.push() + else: + logger.info("Push: No changes since last sync") + + +class SyncManager: + def __init__(self, logger=None): + self.logger = logger or logging.getLogger(__name__) + + def sync(self): + self.logger.info("Pulling repository") + repo = Repository.open() + repo.pull() + + _pull(repo, self.logger) + _push(repo, self.logger) + + self.logger.info("Finished") + + def trigger(self): + """ + Called when user presses the "Sync" button in the admin + + This should enqueue a background task to run the sync() function + """ + self.sync() + + def is_queued(self): + """ + Returns True if the background task is queued + """ + return False + + def is_running(self): + """ + Returns True if the background task is currently running + """ + return False + + +def get_sync_manager(): + sync_manager_class_path = getattr( + settings, + "WAGTAILLOCALIZE_PONTOON_SYNC_MANAGER_CLASS", + "wagtail_localize_pontoon.sync.SyncManager", + ) + sync_manager = import_string(sync_manager_class_path) + return sync_manager() diff --git a/wagtail_localize/translation_engines/pontoon/templates/wagtail_localize_pontoon/dashboard.html b/wagtail_localize/translation_engines/pontoon/templates/wagtail_localize_pontoon/dashboard.html new file mode 100644 index 00000000..b0f0fb13 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/templates/wagtail_localize_pontoon/dashboard.html @@ -0,0 +1,69 @@ +{% extends "wagtailadmin/base.html" %} +{% load static i18n wagtailadmin_tags %} +{% block titletag %}Pontoon{% endblock %} + +{% block content %} + {% include "wagtailadmin/shared/header.html" with title="Pontoon" icon="doc-empty-inverse" %} + +
+ {% if sync_running %} + Synchronising with Pontoon now! + {% elif sync_queued %} + Synchronisation with Pontoon queued + {% else %} +
+ {% csrf_token %} + +
+ {% endif %} + +

Resources

+ + + + + + + + + + {% for resource in resources %} + {% with resource.latest_submission as latest_submission %} + + + + + + + + {% endwith %} + {% endfor %} + +
FilenameSource PageLast updatedLast pushedRequires sync
{{ resource.path }}.pot{{ resource.page.get_admin_display_title }}{{ resource.latest_submission.created_at }}{{ resource.latest_pushed_submission.pushed_at }} + {% if resource.latest_pushed_submission != resource.latest_submission %} + Yes + {% else %} + No + {% endif %} +
+ +

Log

+
    + {% for log in logs %} + {% if log.resources.exists %} +
  • + {{ log.time }} + + {% if log.action == log.ACTION_PUSH %} + Pushed {{ log.resources.unique_resources.count }} resources + {% elif log.action == log.ACTION_PULL %} + Pulled + {{ log.resources.unique_resources.count }} resources + in {{ log.resources.unique_languages.count }} languages + {% endif %} +
    • + {% endif %} + {% endfor %} +
+
+{% endblock %} diff --git a/wagtail_localize/translation_engines/pontoon/tests/__init__.py b/wagtail_localize/translation_engines/pontoon/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py new file mode 100644 index 00000000..6a61dc78 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py @@ -0,0 +1,260 @@ +import logging + +import polib +from django.test import TestCase +from django.utils import timezone + +from wagtail.core.models import Page +from wagtail_localize.models import Language +from wagtail_localize.test.models import TestPage + +from ..importer import Importer +from ..models import PontoonResource, PontoonSyncLog + + +def create_test_page(**kwargs): + parent = kwargs.pop("parent", None) or Page.objects.get(id=1) + page = parent.add_child(instance=TestPage(**kwargs)) + revision = page.save_revision() + revision.publish() + return page + + +def create_test_po(entries): + po = polib.POFile(wrapwidth=200) + po.metadata = { + "POT-Creation-Date": str(timezone.now()), + "MIME-Version": "1.0", + "Content-Type": "text/html; charset=utf-8", + } + + for entry in entries: + po.append(polib.POEntry(msgid=entry[0], msgstr=entry[1])) + + return str(po) + + +class TestImporter(TestCase): + def setUp(self): + self.page = create_test_page( + title="Test page", + slug="test-page", + test_charfield="The test translatable field", + test_synchronizedfield="The test synchronized field", + ) + self.resource = PontoonResource.objects.get(page=self.page) + + self.language = Language.objects.create(code="fr-FR") + + def test_importer(self): + with self.subTest(stage="New page"): + po_v1 = create_test_po([("The test translatable field", "")]).encode( + "utf-8" + ) + + po_v2 = create_test_po( + [("The test translatable field", "Le champ traduisible de test")] + ).encode("utf-8") + + importer = Importer(Language.objects.default(), logging.getLogger("dummy")) + importer.start_import("0" * 40) + importer.import_file( + self.resource.get_po_filename(language=self.language), po_v1, po_v2 + ) + + # Check translated page was created + translated_page = TestPage.objects.get(locale__language=self.language) + self.assertEqual(translated_page.translation_key, self.page.translation_key) + self.assertFalse(translated_page.is_source_translation) + self.assertEqual( + translated_page.test_charfield, "Le champ traduisible de test" + ) + self.assertEqual( + translated_page.test_synchronizedfield, "The test synchronized field" + ) + + # Check log + log = PontoonSyncLog.objects.get() + self.assertEqual(log.action, PontoonSyncLog.ACTION_PULL) + self.assertEqual(log.commit_id, "0" * 40) + log_resource = log.resources.get() + self.assertEqual(log_resource.resource, self.resource) + self.assertEqual(log_resource.language, self.language) + + # Perform another import updating the page + # Much easier to do it this way than trying to construct all the models manually to match the result of the last test + with self.subTest(stage="Update page"): + po_v3 = create_test_po( + [ + ( + "The test translatable field", + "Le champ testable à traduire avec un contenu mis à jour", + ) + ] + ).encode("utf-8") + + importer = Importer(Language.objects.default(), logging.getLogger("dummy")) + importer.start_import("0" * 39 + "1") + importer.import_file( + self.resource.get_po_filename(language=self.language), po_v2, po_v3 + ) + + translated_page.refresh_from_db() + self.assertEqual(translated_page.translation_key, self.page.translation_key) + self.assertFalse(translated_page.is_source_translation) + self.assertEqual( + translated_page.test_charfield, + "Le champ testable à traduire avec un contenu mis à jour", + ) + self.assertEqual( + translated_page.test_synchronizedfield, "The test synchronized field" + ) + + # Check log + log = PontoonSyncLog.objects.exclude(id=log.id).get() + self.assertEqual(log.action, PontoonSyncLog.ACTION_PULL) + self.assertEqual(log.commit_id, "0" * 39 + "1") + log_resource = log.resources.get() + self.assertEqual(log_resource.resource, self.resource) + self.assertEqual(log_resource.language, self.language) + + def test_importer_doesnt_import_if_parent_not_translated(self): + child_page = create_test_page( + title="Test child page", + slug="test-child-page", + test_charfield="The test child's translatable field", + test_synchronizedfield="The test synchronized field", + parent=self.page, + ) + child_resource = PontoonResource.objects.get(page=child_page) + + with self.subTest(stage="Create child page"): + # Translate + po_v1 = create_test_po( + [("The test child's translatable field", "")] + ).encode("utf-8") + + po_v2 = create_test_po( + [ + ( + "The test child's translatable field", + "Le champ traduisible de test", + ) + ] + ).encode("utf-8") + + importer = Importer(Language.objects.default(), logging.getLogger("dummy")) + importer.start_import("0" * 40) + importer.import_file( + child_resource.get_po_filename(language=self.language), po_v1, po_v2 + ) + + # Check translated page was not created + self.assertFalse( + child_page.get_translations() + .filter(locale__language=self.language) + .exists() + ) + + # Check log + log = PontoonSyncLog.objects.get() + self.assertEqual(log.action, PontoonSyncLog.ACTION_PULL) + self.assertEqual(log.commit_id, "0" * 40) + log_resource = log.resources.get() + self.assertEqual(log_resource.resource, child_resource) + self.assertEqual(log_resource.language, self.language) + + with self.subTest(stage="Create parent page"): + po_v1 = create_test_po([("The test translatable field", "")]).encode( + "utf-8" + ) + + po_v2 = create_test_po( + [("The test translatable field", "Le champ traduisible de test")] + ).encode("utf-8") + + importer = Importer(Language.objects.default(), logging.getLogger("dummy")) + importer.start_import("0" * 39 + "1") + importer.import_file( + self.resource.get_po_filename(language=self.language), po_v1, po_v2 + ) + + # Check both translated pages were created + translated_parent = self.page.get_translations().get( + locale__language=self.language + ) + self.assertEqual( + translated_parent.translation_key, self.page.translation_key + ) + self.assertFalse(translated_parent.is_source_translation) + self.assertEqual( + translated_parent.test_charfield, "Le champ traduisible de test" + ) + self.assertEqual( + translated_parent.test_synchronizedfield, "The test synchronized field" + ) + + translated_child = child_page.get_translations().get( + locale__language=self.language + ) + self.assertEqual( + translated_child.translation_key, child_page.translation_key + ) + self.assertFalse(translated_child.is_source_translation) + self.assertEqual( + translated_child.test_charfield, "Le champ traduisible de test" + ) + self.assertEqual( + translated_child.test_synchronizedfield, "The test synchronized field" + ) + + # Check log + log = PontoonSyncLog.objects.exclude(id=log.id).get() + self.assertEqual(log.action, PontoonSyncLog.ACTION_PULL) + self.assertEqual(log.commit_id, "0" * 39 + "1") + log_resource = log.resources.get() + self.assertEqual(log_resource.resource, self.resource) + self.assertEqual(log_resource.language, self.language) + + +class TestImporterRichText(TestCase): + def setUp(self): + self.page = create_test_page( + title="Test page", + slug="test-page", + test_richtextfield='

The test translatable field.

', + ) + self.resource = PontoonResource.objects.get(page=self.page) + + self.language = Language.objects.create(code="fr-FR") + + def test_importer_rich_text(self): + po_v1 = create_test_po( + [('The test translatable field.', "")] + ).encode("utf-8") + + po_v2 = create_test_po( + [ + ( + 'The test translatable field.', + 'Le champ traduisible de test.', + ) + ] + ).encode("utf-8") + + importer = Importer(Language.objects.default(), logging.getLogger("dummy")) + importer.start_import("0" * 40) + importer.import_file( + self.resource.get_po_filename(language=self.language), po_v1, po_v2 + ) + + # Check translated page was created + translated_page = TestPage.objects.get(locale__language=self.language) + self.assertEqual(translated_page.translation_key, self.page.translation_key) + self.assertFalse(translated_page.is_source_translation) + + # Check rich text field was created correctly + self.assertHTMLEqual( + translated_page.test_richtextfield, + '

Le champ traduisible de test.

', + ) diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py b/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py new file mode 100644 index 00000000..765c93a5 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py @@ -0,0 +1,141 @@ +import logging + +import polib +from django.db.models import F +from django.test import TestCase + +from wagtail.core.models import Page +from wagtail_localize.models import Language +from wagtail_localize.test.models import TestPage +from wagtail_localize.translation_memory.models import ( + Segment, + SegmentTranslation, + SegmentPageLocation, +) + +from ..models import PontoonResource +from ..pofile import generate_source_pofile, generate_language_pofile + + +def create_test_page(**kwargs): + root_page = Page.objects.get(id=1) + page = root_page.add_child(instance=TestPage(**kwargs)) + revision = page.save_revision() + revision.publish() + return page + + +class TestGenerateSourcePOFile(TestCase): + def setUp(self): + self.page = create_test_page( + title="Test page", + slug="test-page", + test_charfield="The test translatable field", + test_synchronizedfield="The test synchronized field", + ) + self.resource = PontoonResource.objects.get(page=self.page) + + def test_generate_source_pofile(self): + pofile = generate_source_pofile(self.resource) + parsed_po = polib.pofile(pofile) + self.assertEqual( + [(m.msgid, m.msgstr) for m in parsed_po], + [("The test translatable field", "")], + ) + + def test_generate_source_pofile_with_multiple_revisions(self): + # Create another revision with the same segment text, but in a different otder + # We check this to make sure that the segment is not duplicated in the PO file + # See issue: https://github.com/mozilla/donate-wagtail/issues/559 + + new_revision = self.page.save_revision() + new_revision.publish() + SegmentPageLocation.objects.filter(page_revision=new_revision).update( + order=F("order") + 1 + ) + + pofile = generate_source_pofile(self.resource) + parsed_po = polib.pofile(pofile) + self.assertEqual( + [(m.msgid, m.msgstr) for m in parsed_po], + [("The test translatable field", "")], + ) + + +class TestGenerateLanguagePOFile(TestCase): + def setUp(self): + self.page = create_test_page( + title="Test page", + slug="test-page", + test_charfield="The test translatable field", + test_synchronizedfield="The test synchronized field", + ) + self.resource = PontoonResource.objects.get(page=self.page) + self.language = Language.objects.create(code="fr") + + def test_generate_language_pofile(self): + pofile = generate_language_pofile(self.resource, self.language) + parsed_po = polib.pofile(pofile) + self.assertEqual( + [(m.msgid, m.msgstr) for m in parsed_po], + [("The test translatable field", "")], + ) + + def test_generate_language_pofile_with_existing_translation(self): + segment = Segment.objects.get(text="The test translatable field") + SegmentTranslation.objects.create( + translation_of=segment, + language=self.language, + text="Le champ traduisible de test", + ) + + pofile = generate_language_pofile(self.resource, self.language) + parsed_po = polib.pofile(pofile) + self.assertEqual( + [(m.msgid, m.msgstr) for m in parsed_po], + [("The test translatable field", "Le champ traduisible de test")], + ) + + def test_generate_language_pofile_with_existing_obsolete_translation(self): + # Update the existing segment. The save_revision bit below will generate a new segment with the current text + segment = Segment.objects.get() + segment.text = "Some obsolete text" + segment.text_id = Segment.get_text_id(segment.text) + segment.save() + + SegmentTranslation.objects.create( + translation_of=segment, language=self.language, text="Du texte obsolète" + ) + + # Create a new revision. This will create a new segment like how the current segment was before I changed it + # It will also update the revision field on the resource so we need to refresh that + self.page.save_revision().publish() + self.resource.refresh_from_db() + + pofile = generate_language_pofile(self.resource, self.language) + parsed_po = polib.pofile(pofile) + self.assertEqual( + [(m.msgid, m.msgstr, m.obsolete) for m in parsed_po], + [ + ("The test translatable field", "", 0), + ("Some obsolete text", "Du texte obsolète", 1), + ], + ) + + def test_generate_language_pofile_with_multiple_revisions(self): + # Create another revision with the same segment text, but in a different otder + # We check this to make sure that the segment is not duplicated in the PO file + # See issue: https://github.com/mozilla/donate-wagtail/issues/559 + + new_revision = self.page.save_revision() + new_revision.publish() + SegmentPageLocation.objects.filter(page_revision=new_revision).update( + order=F("order") + 1 + ) + + pofile = generate_language_pofile(self.resource, self.language) + parsed_po = polib.pofile(pofile) + self.assertEqual( + [(m.msgid, m.msgstr) for m in parsed_po], + [("The test translatable field", "")], + ) diff --git a/wagtail_localize/translation_engines/pontoon/views.py b/wagtail_localize/translation_engines/pontoon/views.py new file mode 100644 index 00000000..d0906280 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/views.py @@ -0,0 +1,26 @@ +from django.shortcuts import render, redirect + +from .models import PontoonSyncLog, PontoonResource +from .sync import get_sync_manager + + +def dashboard(request): + sync_manager = get_sync_manager() + return render( + request, + "wagtail_localize_pontoon/dashboard.html", + { + "resources": PontoonResource.objects.all(), + "logs": PontoonSyncLog.objects.order_by("-time"), + "sync_running": sync_manager.is_running(), + "sync_queued": sync_manager.is_queued(), + }, + ) + + +def force_sync(request): + sync_manager = get_sync_manager() + if not sync_manager.is_queued(): + sync_manager.trigger() + + return redirect("wagtail_localize_pontoon:dashboard") diff --git a/wagtail_localize/translation_engines/pontoon/wagtail_hooks.py b/wagtail_localize/translation_engines/pontoon/wagtail_hooks.py new file mode 100644 index 00000000..27d180c5 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/wagtail_hooks.py @@ -0,0 +1,40 @@ +from django.conf.urls import url, include +from django.urls import reverse +from django.utils.translation import ugettext_lazy as _ + +from wagtail.admin.menu import MenuItem +from wagtail.core import hooks + +from . import views + + +@hooks.register("register_admin_urls") +def register_admin_urls(): + urls = [ + url("^$", views.dashboard, name="dashboard"), + url("^force-sync/$", views.force_sync, name="force_sync"), + ] + + return [ + url( + "^localize/pontoon/", + include( + (urls, "wagtail_localize_pontoon"), namespace="wagtail_localize_pontoon" + ), + ) + ] + + +class PontoonMenuItem(MenuItem): + def is_shown(self, request): + return True + + +@hooks.register("register_settings_menu_item") +def register_menu_item(): + return PontoonMenuItem( + _("Pontoon"), + reverse("wagtail_localize_pontoon:dashboard"), + classnames="icon icon-site", + order=500, + ) From dbfc9e7b1d0956a16f9dfd4c38bc90a478b94407 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 18:36:10 +0000 Subject: [PATCH 02/21] Add some test for git repository writer --- .../pontoon/tests/test_git.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 wagtail_localize/translation_engines/pontoon/tests/test_git.py diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_git.py b/wagtail_localize/translation_engines/pontoon/tests/test_git.py new file mode 100644 index 00000000..65347bf2 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/tests/test_git.py @@ -0,0 +1,105 @@ +import io +import tempfile + +import toml +import pygit2 +from django.test import TestCase +from git import Repo + +from ..git import Repository + + +class GitTestCase(TestCase): + def setUp(self): + self.repo_dir = tempfile.TemporaryDirectory() + self.gitpython = Repo.init(self.repo_dir.name, bare=True) + self.initial_commit = self.gitpython.index.commit("initial commit") + self.gitpython.create_head("master") + self.pygit = pygit2.Repository(self.repo_dir.name) + self.repo = Repository(self.pygit, self.gitpython) + + def assert_file_in_tree(self, tree, name, mode=33188, check_contents=None): + # FIXME allow more than one file + blob = tree.blobs[0] + self.assertEqual(blob.name, name) + self.assertEqual(blob.mode, mode) + + if check_contents is not None: + contents = io.BytesIO() + blob.stream_data(contents) + check_contents(contents.getvalue()) + + def tearDown(self): + self.repo_dir.cleanup() + + +class TestRepositoryWriter(GitTestCase): + def test_write_file_and_commit(self): + writer = self.repo.writer() + + writer.write_file("test.txt", "this is a test") + writer.commit("Added test.txt") + + # Check new commit + commit = self.gitpython.head.commit + self.assertEqual(commit.parents, (self.initial_commit,)) + self.assertEqual(commit.author.name, "Wagtail Localize") + self.assertEqual(commit.author.email, "wagtail_localize_pontoon@wagtail.io") + self.assertEqual(commit.message, "Added test.txt") + + # Check file has been committed + def check_contents(contents): + self.assertEqual(contents.decode(), "this is a test") + + self.assert_file_in_tree(commit.tree, "test.txt", check_contents=check_contents) + + def test_has_changes(self): + writer = self.repo.writer() + + # Initially, should be no changes + self.assertFalse(writer.has_changes()) + + # Writing data should make it return True + writer.write_file("test.txt", "this is a test") + self.assertTrue(writer.has_changes()) + + # After committing that change, should return False again + writer.commit("Added test.txt") + self.assertFalse(writer.has_changes()) + + # Writing a file, but not changing anything should make it still return False + writer.write_file("test.txt", "this is a test") + self.assertFalse(writer.has_changes()) + + # But making actual changes to the file should make it return True + writer.write_file("test.txt", "this is an updated test") + self.assertTrue(writer.has_changes()) + + def test_write_config(self): + writer = self.repo.writer() + + writer.write_config( + ["en", "de", "fr"], + [("templates/mytemplate.pot", "locales/de/mytranslation.po"),], + ) + + writer.commit("Wrote config") + + # Check file has been committed + def check_contents(contents): + self.assertEqual( + toml.loads(contents.decode()), + { + "locales": ["en", "de", "fr"], + "paths": [ + { + "l10n": "locales/de/mytranslation.po", + "reference": "templates/mytemplate.pot", + } + ], + }, + ) + + self.assert_file_in_tree( + self.gitpython.head.commit.tree, "l10n.toml", check_contents=check_contents + ) From d9417db3a237a945e6316ce6a2d5c1cfa6391209 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 13:57:32 +0000 Subject: [PATCH 03/21] Add base class for extracted values --- wagtail_localize/segments/__init__.py | 164 +++++++++++--------------- 1 file changed, 70 insertions(+), 94 deletions(-) diff --git a/wagtail_localize/segments/__init__.py b/wagtail_localize/segments/__init__.py index b95da902..42cd4c62 100644 --- a/wagtail_localize/segments/__init__.py +++ b/wagtail_localize/segments/__init__.py @@ -6,7 +6,63 @@ from .html import extract_html_elements, restore_html_elements -class SegmentValue: +class BaseValue: + def __init__(self, path, order=0): + self.path = path + self.order = order + + def clone(self): + """ + Clones this segment. Must be overridden in subclass. + """ + raise NotImplementedError + + def with_order(self, order): + """ + Sets the order of this segment. + """ + clone = self.clone() + clone.order = order + return clone + + def wrap(self, base_path): + """ + Appends a component to the beginning of the path. + + For example: + + >>> s = SegmentValue('field', 'foo') + >>> s.wrap('wrapped') + SegmentValue('wrapped.field', 'foo') + """ + new_path = base_path + + if self.path: + new_path += "." + self.path + + clone = self.clone() + clone.path = new_path + return clone + + def unwrap(self): + """ + Pops a component from the beginning of the path. Reversing .wrap(). + + For example: + + >>> s = SegmentValue('wrapped.field', 'foo') + >>> s.unwrap() + 'wrapped', SegmentValue('field', 'foo') + """ + first_component, *remaining_components = self.path.split(".") + new_path = ".".join(remaining_components) + + clone = self.clone() + clone.path = new_path + return first_component, clone + + +class SegmentValue(BaseValue): class HTMLElement: """ Represents the position of an inline element within an HTML segment value. @@ -48,12 +104,17 @@ def __eq__(self, other): def __repr__(self): return f"" - def __init__(self, path, text, html_elements=None, order=0): - self.path = path - self.order = order + def __init__(self, path, text, html_elements=None, **kwargs): self.text = text self.html_elements = html_elements + super().__init__(path, **kwargs) + + def clone(self): + return SegmentValue( + self.path, self.text, html_elements=self.html_elements, order=self.order + ) + @classmethod def from_html(cls, path, html): text, elements = extract_html_elements(html) @@ -70,46 +131,6 @@ def from_html(cls, path, html): return cls(path, text, html_elements) - def with_order(self, order): - """ - Sets the order of this segment. - """ - return SegmentValue(self.path, self.text, self.html_elements, order=order) - - def wrap(self, base_path): - """ - Appends a component to the beginning of the path. - - For example: - - >>> s = SegmentValue('field', "The text") - >>> s.wrap('relation') - SegmentValue('relation.field', "The text") - """ - new_path = base_path - - if self.path: - new_path += "." + self.path - - return SegmentValue(new_path, self.text, self.html_elements, order=self.order) - - def unwrap(self): - """ - Pops a component from the beginning of the path. Reversing .wrap(). - - For example: - - >>> s = SegmentValue('relation.field', "The text") - >>> s.unwrap() - 'relation', SegmentValue('field', "The text") - """ - base_path, *remaining_components = self.path.split(".") - new_path = ".".join(remaining_components) - return ( - base_path, - SegmentValue(new_path, self.text, self.html_elements, order=self.order), - ) - @property def html(self): if not self.html_elements: @@ -200,62 +221,17 @@ def __repr__(self): return ''.format(self.path, self.html) -class TemplateValue: - def __init__(self, path, format, template, segment_count, order=0): - self.path = path - self.order = order +class TemplateValue(BaseValue): + def __init__(self, path, format, template, segment_count, **kwargs): self.format = format self.template = template self.segment_count = segment_count - def with_order(self, order): - """ - Sets the order of this segment. - """ - return TemplateValue( - self.path, self.format, self.template, self.segment_count, order=order - ) - - def wrap(self, base_path): - """ - Appends a component to the beginning of the path. - - For example: - - >>> s = TemplateValue('field', 'html', ", 1) - >>> s.wrap('relation') - TemplateValue('relation.field', 'html', ", 1) - """ - new_path = base_path - - if self.path: - new_path += "." + self.path + super().__init__(path, **kwargs) + def clone(self): return TemplateValue( - new_path, self.format, self.template, self.segment_count, order=self.order - ) - - def unwrap(self): - """ - Pops a component from the beginning of the path. Reversing .wrap(). - - For example: - - >>> s = TemplateValue('relation.field', 'html', ", 1) - >>> s.unwrap() - 'relation', TemplateValue('field', 'html', ", 1) - """ - base_path, *remaining_components = self.path.split(".") - new_path = ".".join(remaining_components) - return ( - base_path, - TemplateValue( - new_path, - self.format, - self.template, - self.segment_count, - order=self.order, - ), + self.path, self.format, self.template, self.segment_count, order=self.order ) def is_empty(self): From b1a69c41b1d53764d4552cc68562805610343225 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 14:07:22 +0000 Subject: [PATCH 04/21] Don't extract ManyToOneRels that don't come from ParentalKey --- wagtail_localize/segments/extract.py | 7 ++- .../segments/tests/test_segment_extraction.py | 17 ++++- .../0006_testnonparentalchildobject.py | 62 +++++++++++++++++++ wagtail_localize/test/models.py | 9 +++ 4 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 wagtail_localize/test/migrations/0006_testnonparentalchildobject.py diff --git a/wagtail_localize/segments/extract.py b/wagtail_localize/segments/extract.py index c7479902..37c6e9de 100644 --- a/wagtail_localize/segments/extract.py +++ b/wagtail_localize/segments/extract.py @@ -1,5 +1,6 @@ from django.db import models +from modelcluster.fields import ParentalKey from wagtail.core import blocks from wagtail.core.fields import RichTextField, StreamField from wagtail.core.rich_text import RichText @@ -136,8 +137,10 @@ def extract_segments(instance): for segment in extract_segments(related_instance) ) - elif isinstance(field, (models.ManyToOneRel)) and issubclass( - field.related_model, TranslatableMixin + elif ( + isinstance(field, (models.ManyToOneRel)) + and isinstance(field.remote_field, ParentalKey) + and issubclass(field.related_model, TranslatableMixin) ): manager = getattr(instance, field.name) diff --git a/wagtail_localize/segments/tests/test_segment_extraction.py b/wagtail_localize/segments/tests/test_segment_extraction.py index 7c419a07..38ae7f7d 100644 --- a/wagtail_localize/segments/tests/test_segment_extraction.py +++ b/wagtail_localize/segments/tests/test_segment_extraction.py @@ -6,7 +6,12 @@ from wagtail.core.blocks import StreamValue from wagtail.core.models import Page -from wagtail_localize.test.models import TestPage, TestSnippet, TestChildObject +from wagtail_localize.test.models import ( + TestPage, + TestSnippet, + TestChildObject, + TestNonParentalChildObject, +) from wagtail_localize.segments import SegmentValue, TemplateValue from wagtail_localize.segments.extract import extract_segments @@ -111,6 +116,16 @@ def test_childobjects(self): ], ) + def test_nonparentalchildobjects(self): + page = make_test_page() + page.save() + TestNonParentalChildObject.objects.create(page=page, field="Test content") + + segments = extract_segments(page) + + # No segments this time as we don't extract ManyToOneRel's that don't use ParentalKeys + self.assertEqual(segments, []) + def test_customfield(self): page = make_test_page(test_customfield="Test content") diff --git a/wagtail_localize/test/migrations/0006_testnonparentalchildobject.py b/wagtail_localize/test/migrations/0006_testnonparentalchildobject.py new file mode 100644 index 00000000..5a0903b1 --- /dev/null +++ b/wagtail_localize/test/migrations/0006_testnonparentalchildobject.py @@ -0,0 +1,62 @@ +# Generated by Django 2.2.9 on 2019-12-18 14:03 + +from django.db import migrations, models +import django.db.models.deletion +import uuid +import wagtail_localize.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize", "0002_initial_data"), + ("wagtail_localize_test", "0005_create_testhomepage"), + ] + + operations = [ + migrations.CreateModel( + name="TestNonParentalChildObject", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "sort_order", + models.IntegerField(blank=True, editable=False, null=True), + ), + ( + "translation_key", + models.UUIDField(default=uuid.uuid4, editable=False), + ), + ("is_source_translation", models.BooleanField(default=True)), + ("field", models.TextField()), + ( + "locale", + models.ForeignKey( + default=wagtail_localize.models.default_locale_id, + on_delete=django.db.models.deletion.PROTECT, + related_name="+", + to="wagtail_localize.Locale", + ), + ), + ( + "page", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="test_nonparentalchildobjects", + to="wagtail_localize_test.TestPage", + ), + ), + ], + options={ + "abstract": False, + "unique_together": {("translation_key", "locale")}, + }, + ), + ] diff --git a/wagtail_localize/test/models.py b/wagtail_localize/test/models.py index 75182e2c..00c5b632 100644 --- a/wagtail_localize/test/models.py +++ b/wagtail_localize/test/models.py @@ -112,5 +112,14 @@ class TestChildObject(TranslatableMixin, Orderable): translatable_fields = [TranslatableField("field")] +class TestNonParentalChildObject(TranslatableMixin, Orderable): + page = models.ForeignKey( + TestPage, on_delete=models.CASCADE, related_name="test_nonparentalchildobjects" + ) + field = models.TextField() + + translatable_fields = [TranslatableField("field")] + + class TestHomePage(TranslatablePageMixin, TranslatablePageRoutingMixin, Page): pass From ba45280f820db026599fce7b6ed8a670fe30df52 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Mon, 16 Dec 2019 17:49:07 +0000 Subject: [PATCH 05/21] Added generic version models --- .../migrations/0007_versioning_models.py | 76 +++++++++++++++++++ wagtail_localize/translation_memory/models.py | 63 +++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 wagtail_localize/translation_memory/migrations/0007_versioning_models.py diff --git a/wagtail_localize/translation_memory/migrations/0007_versioning_models.py b/wagtail_localize/translation_memory/migrations/0007_versioning_models.py new file mode 100644 index 00000000..e5f2e2b6 --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0007_versioning_models.py @@ -0,0 +1,76 @@ +# Generated by Django 2.2.9 on 2019-12-18 14:33 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtailcore", "0041_group_collection_permissions_verbose_name_plural"), + ("wagtail_localize", "0002_initial_data"), + ("contenttypes", "0002_remove_content_type_name"), + ("wagtail_localize_translation_memory", "0006_segmentpagelocation_html_attrs"), + ] + + operations = [ + migrations.CreateModel( + name="TranslatableObject", + fields=[ + ( + "translation_key", + models.UUIDField(primary_key=True, serialize=False), + ), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="contenttypes.ContentType", + ), + ), + ], + options={"unique_together": {("content_type", "translation_key")},}, + ), + migrations.CreateModel( + name="TranslatableRevision", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("content_json", models.TextField()), + ("created_at", models.DateTimeField()), + ( + "locale", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wagtail_localize.Locale", + ), + ), + ( + "object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="revisions", + to="wagtail_localize_translation_memory.TranslatableObject", + ), + ), + ( + "page_revision", + models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="wagtaillocalize_revision", + to="wagtailcore.PageRevision", + ), + ), + ], + ), + ] diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index cd768f19..000a0d8b 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -1,6 +1,7 @@ import json import uuid +from django.contrib.contenttypes.models import ContentType from django.db import models, transaction from django.db.models import Subquery, OuterRef @@ -12,6 +13,68 @@ def pk(obj): return obj +class TranslatableObjectManager(models.Manager): + def get_or_create_from_instance(self, instance): + return self.get_or_create( + translation_key=instance.translation_key, + content_type=ContentType.objects.get_for_model( + instance.get_translation_model() + ), + ) + + +class TranslatableObject(models.Model): + """ + Represents something that can be translated. + + Note that one instance of this represents all translations for the object. + """ + + translation_key = models.UUIDField(primary_key=True) + content_type = models.ForeignKey( + ContentType, on_delete=models.CASCADE, related_name="+" + ) + + objects = TranslatableObjectManager() + + def get_instance(self, locale): + return self.content_type.get_object_for_this_type( + translation_key=self.translation_key, locale=locale + ) + + class Meta: + unique_together = [("content_type", "translation_key")] + + +class TranslatableRevision(models.Model): + """ + A piece of content that to be used as a source for translations. + """ + + object = models.ForeignKey( + TranslatableObject, on_delete=models.CASCADE, related_name="revisions" + ) + locale = models.ForeignKey("wagtail_localize.Locale", on_delete=models.CASCADE) + content_json = models.TextField() + created_at = models.DateTimeField() + + page_revision = models.OneToOneField( + "wagtailcore.PageRevision", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="wagtaillocalize_revision", + ) + + def as_instance(self): + if self.page_revision is not None: + return self.page_revision.as_page_object() + + raise NotImplementedError( + "revisions of non-page objects not currently supported" + ) + + class SegmentQuerySet(models.QuerySet): def annotate_translation(self, language): """ From 4a01d170bb4d2ffac7a6410080ea41c3f066a40f Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Mon, 16 Dec 2019 17:53:09 +0000 Subject: [PATCH 06/21] New location models --- .../migrations/0008_new_location_models.py | 79 +++++++++++++++++ wagtail_localize/translation_memory/models.py | 86 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 wagtail_localize/translation_memory/migrations/0008_new_location_models.py diff --git a/wagtail_localize/translation_memory/migrations/0008_new_location_models.py b/wagtail_localize/translation_memory/migrations/0008_new_location_models.py new file mode 100644 index 00000000..3e884afe --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0008_new_location_models.py @@ -0,0 +1,79 @@ +# Generated by Django 2.2.9 on 2019-12-18 15:12 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_translation_memory", "0007_versioning_models"), + ] + + operations = [ + migrations.CreateModel( + name="TemplateLocation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("path", models.TextField()), + ("order", models.PositiveIntegerField()), + ( + "revision", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wagtail_localize_translation_memory.TranslatableRevision", + ), + ), + ( + "template", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="locations", + to="wagtail_localize_translation_memory.Template", + ), + ), + ], + options={"abstract": False,}, + ), + migrations.CreateModel( + name="SegmentLocation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("path", models.TextField()), + ("order", models.PositiveIntegerField()), + ("html_attrs", models.TextField(blank=True)), + ( + "revision", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wagtail_localize_translation_memory.TranslatableRevision", + ), + ), + ( + "segment", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="locations", + to="wagtail_localize_translation_memory.Segment", + ), + ), + ], + options={"abstract": False,}, + ), + ] diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index 000a0d8b..0005c545 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -171,6 +171,92 @@ def from_template_value(cls, template_value): return template +class BaseLocation(models.Model): + revision = models.ForeignKey(TranslatableRevision, on_delete=models.CASCADE) + path = models.TextField() + order = models.PositiveIntegerField() + + class Meta: + abstract = True + + +class SegmentLocationQuerySet(models.QuerySet): + def annotate_translation(self, language): + """ + Adds a 'translation' field to the segments containing the + text content of the segment translated into the specified + language. + """ + return self.annotate( + translation=Subquery( + SegmentTranslation.objects.filter( + translation_of_id=OuterRef("segment_id"), language_id=pk(language) + ).values("text") + ) + ) + + +class SegmentLocation(BaseLocation): + segment = models.ForeignKey( + Segment, on_delete=models.CASCADE, related_name="locations" + ) + + # When we extract the segment, we replace HTML attributes with id tags + # The attributes that were removed are stored here. These must be + # added into the translated strings. + # These are stored as a mapping of element ids to KV mappings of + # attributes in JSON format. For example: + # + # For this segment: Link to example.com + # + # The value of this field could be: + # + # { + # "a#a1": { + # "href": "https://www.example.com" + # } + # } + html_attrs = models.TextField(blank=True) + + objects = SegmentLocationQuerySet.as_manager() + + @classmethod + def from_segment_value(cls, revision, language, segment_value): + segment = Segment.from_text(language, segment_value.html_with_ids) + + segment_loc, created = cls.objects.get_or_create( + revision_id=pk(revision), + path=segment_value.path, + order=segment_value.order, + segment=segment, + html_attrs=json.dumps(segment_value.get_html_attrs()), + ) + + return segment_loc + + +class TemplateLocation(BaseLocation): + template = models.ForeignKey( + Template, on_delete=models.CASCADE, related_name="locations" + ) + + @classmethod + def from_template_value(cls, revision, template_value): + template = Template.from_template_value(template_value) + + template_loc, created = cls.objects.get_or_create( + revision_id=pk(revision), + path=template_value.path, + order=template_value.order, + template=template, + ) + + return template_loc + + +# LEGACY PageLocation models + + class BasePageLocation(models.Model): page_revision = models.ForeignKey( "wagtailcore.PageRevision", on_delete=models.CASCADE From 50eb212ada81c02a096d0cc4c5d22d6c26cd021b Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 15:21:19 +0000 Subject: [PATCH 07/21] Add migration to new location models --- .../0009_migrate_to_new_location_models.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 wagtail_localize/translation_memory/migrations/0009_migrate_to_new_location_models.py diff --git a/wagtail_localize/translation_memory/migrations/0009_migrate_to_new_location_models.py b/wagtail_localize/translation_memory/migrations/0009_migrate_to_new_location_models.py new file mode 100644 index 00000000..b3406a7e --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0009_migrate_to_new_location_models.py @@ -0,0 +1,82 @@ +# Generated by Django 2.2.9 on 2019-12-18 14:36 + +from django.db import migrations + + +def migrate_to_new_location_models(apps, schema_editor): + TranslatableObject = apps.get_model( + "wagtail_localize_translation_memory.TranslatableObject" + ) + TranslatableRevision = apps.get_model( + "wagtail_localize_translation_memory.TranslatableRevision" + ) + SegmentPageLocation = apps.get_model( + "wagtail_localize_translation_memory.SegmentPageLocation" + ) + SegmentLocation = apps.get_model( + "wagtail_localize_translation_memory.SegmentLocation" + ) + TemplatePageLocation = apps.get_model( + "wagtail_localize_translation_memory.TemplatePageLocation" + ) + TemplateLocation = apps.get_model( + "wagtail_localize_translation_memory.TemplateLocation" + ) + ContentType = apps.get_model("contenttypes.ContentType") + + def get_revision_from_page_revision(page_revision): + try: + return TranslatableRevision.objects.get(page_revision=page_revision) + except TranslatableRevision.DoesNotExist: + page = page_revision.page + page_model = apps.get_model( + page.content_type.app_label, page.content_type.model + ) + page_translation_model = page_model._meta.get_field("locale").model + page_specific = page_model.objects.get(id=page.id) + + object, created = TranslatableObject.objects.get_or_create( + translation_key=page_specific.translation_key, + content_type=ContentType.objects.get_for_model(page_translation_model), + ) + + return TranslatableRevision.objects.create( + object=object, + locale=page_specific.locale, + content_json=page_revision.content_json, + created_at=page_revision.created_at, + page_revision=page_revision, + ) + + for segment_page_location in SegmentPageLocation.objects.select_related( + "page_revision" + ).iterator(): + revision = get_revision_from_page_revision(segment_page_location.page_revision) + SegmentLocation.objects.create( + revision=revision, + path=segment_page_location.path, + order=segment_page_location.order, + segment_id=segment_page_location.segment_id, + ) + + for template_page_location in TemplatePageLocation.objects.select_related( + "page_revision" + ).iterator(): + revision = get_revision_from_page_revision(template_page_location.page_revision) + TemplateLocation.objects.create( + revision=revision, + path=template_page_location.path, + order=template_page_location.order, + template_id=template_page_location.template_id, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_translation_memory", "0008_new_location_models"), + ] + + operations = [ + migrations.RunPython(migrate_to_new_location_models), + ] From 10d1c9a7b252b5c326b162f625ef570feb38c7a7 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 15:39:45 +0000 Subject: [PATCH 08/21] Pontoon: Rename revision => page_revison --- .../translation_engines/pontoon/importer.py | 8 ++--- .../commands/submit_whole_site_to_pontoon.py | 10 +++--- .../0007_rename_revision_to_page_revision.py | 28 ++++++++++++++++ .../translation_engines/pontoon/models.py | 32 +++++++++---------- .../translation_engines/pontoon/sync.py | 2 +- 5 files changed, 55 insertions(+), 25 deletions(-) create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0007_rename_revision_to_page_revision.py diff --git a/wagtail_localize/translation_engines/pontoon/importer.py b/wagtail_localize/translation_engines/pontoon/importer.py index 7de88b8a..06c73632 100644 --- a/wagtail_localize/translation_engines/pontoon/importer.py +++ b/wagtail_localize/translation_engines/pontoon/importer.py @@ -39,7 +39,7 @@ def create_or_update_translated_page(self, submission, language): region_id=Region.objects.default_id(), language=language ) - page = submission.revision.as_page_object() + page = submission.page_revision.as_page_object() try: translated_page = page.get_translation(locale) @@ -51,11 +51,11 @@ def create_or_update_translated_page(self, submission, language): # Fetch all translated segments segment_page_locations = SegmentPageLocation.objects.filter( - page_revision=submission.revision + page_revision=submission.page_revision ).annotate_translation(language) template_page_locations = TemplatePageLocation.objects.filter( - page_revision=submission.revision + page_revision=submission.page_revision ).select_related("template") segments = [] @@ -90,7 +90,7 @@ def create_or_update_translated_page(self, submission, language): new_revision.publish() PontoonResourceTranslation.objects.create( - submission=submission, language=language, revision=new_revision + submission=submission, language=language, page_revision=new_revision ) return new_revision, created diff --git a/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py b/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py index ec8234f4..cdbd3b57 100644 --- a/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py +++ b/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py @@ -21,16 +21,18 @@ def handle(self, **options): print( f"Warning: The page '{page.title}' does not have a live_revision. Using latest revision instead." ) - revision = page.live_revision + page_revision = page.live_revision else: - revision = page.get_latest_revision() or page.save_revision( + page_revision = page.get_latest_revision() or page.save_revision( changed=False ) print(f"Submitting '{page.title}'") - if PontoonResourceSubmission.objects.filter(revision=revision).exists(): + if PontoonResourceSubmission.objects.filter( + page_revision=page_revision + ).exists(): print("Already submitted!") continue - submit_to_pontoon(page, revision) + submit_to_pontoon(page, page_revision) diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0007_rename_revision_to_page_revision.py b/wagtail_localize/translation_engines/pontoon/migrations/0007_rename_revision_to_page_revision.py new file mode 100644 index 00000000..4143fb5d --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0007_rename_revision_to_page_revision.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.9 on 2019-12-18 15:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_pontoon", "0006_auto_20191007_1700"), + ] + + operations = [ + migrations.RenameField( + model_name="pontoonresource", + old_name="current_revision", + new_name="current_page_revision", + ), + migrations.RenameField( + model_name="pontoonresourcesubmission", + old_name="revision", + new_name="page_revision", + ), + migrations.RenameField( + model_name="pontoonresourcetranslation", + old_name="revision", + new_name="page_revision", + ), + ] diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index dcfbcf1a..89de97de 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -36,9 +36,9 @@ class PontoonResource(models.Model): # This is initially the pages URL path but is not updated if the page is moved path = models.CharField(max_length=255, unique=True) - # The last revision to be submitted for translation - # This is denormalised from the revision field of the latest submission for this resource - current_revision = models.OneToOneField( + # The last page_revision to be submitted for translation + # This is denormalised from the page_revision field of the latest submission for this resource + current_page_revision = models.OneToOneField( "wagtailcore.PageRevision", on_delete=models.SET_NULL, null=True, @@ -132,7 +132,7 @@ def get_segments(self): Gets all segments that are in the latest submission to Pontoon. """ return Segment.objects.filter( - page_locations__page_revision_id=self.current_revision_id + page_locations__page_revision_id=self.current_page_revision_id ) def get_all_segments(self, annotate_obsolete=False): @@ -148,7 +148,7 @@ def get_all_segments(self, annotate_obsolete=False): is_obsolete=~Exists( SegmentPageLocation.objects.filter( segment=OuterRef("pk"), - page_revision_id=self.current_revision_id, + page_revision_id=self.current_page_revision_id, ) ) ) @@ -246,7 +246,7 @@ class PontoonResourceSubmission(models.Model): resource = models.ForeignKey( PontoonResource, on_delete=models.CASCADE, related_name="submissions" ) - revision = models.OneToOneField( + page_revision = models.OneToOneField( "wagtailcore.PageRevision", on_delete=models.CASCADE, related_name="pontoon_submission", @@ -272,7 +272,7 @@ def get_translation_progress(self, language): - The total number of segments in the submission to translate - The number of segments that have been translated into the target language """ - return get_translation_progress(self.revision_id, language) + return get_translation_progress(self.page_revision_id, language) class PontoonResourceTranslation(models.Model): @@ -290,7 +290,7 @@ class PontoonResourceTranslation(models.Model): ) # The revision of the page that was created when the translations were saved - revision = models.OneToOneField( + page_revision = models.OneToOneField( "wagtailcore.PageRevision", on_delete=models.CASCADE, related_name="pontoon_translation", @@ -300,31 +300,31 @@ class PontoonResourceTranslation(models.Model): @transaction.atomic -def submit_to_pontoon(page, revision): +def submit_to_pontoon(page, page_revision): # Extract segments from page and save them to translation memory - insert_segments(revision, page.locale.language_id, extract_segments(page)) + insert_segments(page_revision, page.locale.language_id, extract_segments(page)) # Get/create resource try: resource = PontoonResource.objects.get(page=page) - resource.current_revision = revision - resource.save(update_fields=["current_revision"]) + resource.current_page_revision = page_revision + resource.save(update_fields=["current_page_revision"]) except PontoonResource.DoesNotExist: resource = PontoonResource.objects.create( page=page, - current_revision=revision, + current_page_revision=page_revision, path=PontoonResource.get_unique_path_from_urlpath(page.url_path), ) # Create submission - resource.submissions.create(revision=revision) + resource.submissions.create(page_revision=page_revision) @receiver(page_published) def submit_page_to_pontoon(sender, **kwargs): if issubclass(sender, TranslatablePageMixin): page = kwargs["instance"] - revision = kwargs["revision"] + page_revision = kwargs["revision"] if page.locale_id == Locale.objects.default_id(): - submit_to_pontoon(page, revision) + submit_to_pontoon(page, page_revision) diff --git a/wagtail_localize/translation_engines/pontoon/sync.py b/wagtail_localize/translation_engines/pontoon/sync.py index ad4ab387..8238ad1b 100644 --- a/wagtail_localize/translation_engines/pontoon/sync.py +++ b/wagtail_localize/translation_engines/pontoon/sync.py @@ -72,7 +72,7 @@ def update_po(filename, new_po_string): pushed_submission_ids = [] for submission in ( PontoonResourceSubmission.objects.filter( - revision_id=F("resource__current_revision_id") + page_revision_id=F("resource__current_page_revision_id") ) .select_related("resource") .order_by("resource__path") From d336a54f2d40c2061b38950d93e623424e23c628 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 16:17:50 +0000 Subject: [PATCH 09/21] Pontoon: Add new revision fields --- .../0008_add_new_revision_fields.py | 45 +++++++++++ .../0009_populate_new_revision_fields.py | 77 +++++++++++++++++++ .../translation_engines/pontoon/models.py | 21 +++++ 3 files changed, 143 insertions(+) create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0008_add_new_revision_fields.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0009_populate_new_revision_fields.py diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0008_add_new_revision_fields.py b/wagtail_localize/translation_engines/pontoon/migrations/0008_add_new_revision_fields.py new file mode 100644 index 00000000..c45735d9 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0008_add_new_revision_fields.py @@ -0,0 +1,45 @@ +# Generated by Django 2.2.9 on 2019-12-18 16:00 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_translation_memory", "0009_migrate_to_new_location_models"), + ("wagtail_localize_pontoon", "0007_rename_revision_to_page_revision"), + ] + + operations = [ + migrations.AddField( + model_name="pontoonresource", + name="current_revision", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtail_localize_translation_memory.TranslatableRevision", + ), + ), + migrations.AddField( + model_name="pontoonresource", + name="object", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="wagtail_localize_translation_memory.TranslatableObject", + ), + ), + migrations.AddField( + model_name="pontoonresourcesubmission", + name="revision", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="pontoon_submission", + to="wagtail_localize_translation_memory.TranslatableRevision", + ), + ), + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0009_populate_new_revision_fields.py b/wagtail_localize/translation_engines/pontoon/migrations/0009_populate_new_revision_fields.py new file mode 100644 index 00000000..d3e540d5 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0009_populate_new_revision_fields.py @@ -0,0 +1,77 @@ +# Generated by Django 2.2.9 on 2019-12-18 16:01 + +from django.db import migrations + + +def populate_new_revision_fields(apps, schema_editor): + PontoonResource = apps.get_model("wagtail_localize_pontoon.PontoonResource") + PontoonResourceSubmission = apps.get_model( + "wagtail_localize_pontoon.PontoonResourceSubmission" + ) + TranslatableObject = apps.get_model( + "wagtail_localize_translation_memory.TranslatableObject" + ) + TranslatableRevision = apps.get_model( + "wagtail_localize_translation_memory.TranslatableRevision" + ) + ContentType = apps.get_model("contenttypes.ContentType") + + def get_page_specific_instance(page): + page_model = apps.get_model( + page.content_type.app_label, page.content_type.model + ) + return page_model.objects.get(id=page.id) + + def get_translatable_object_from_page(page): + page_translation_model = page._meta.get_field("locale").model + + object, created = TranslatableObject.objects.get_or_create( + translation_key=page.translation_key, + content_type=ContentType.objects.get_for_model(page_translation_model), + ) + + return object + + def get_revision_from_page_revision(page_revision): + try: + return TranslatableRevision.objects.get(page_revision=page_revision) + except TranslatableRevision.DoesNotExist: + page_specific = get_page_specific_instance(page_revision.page) + + return TranslatableRevision.objects.create( + object=get_translatable_object_from_page(page_specific), + locale=page_specific.locale, + content_json=page_revision.content_json, + created_at=page_revision.created_at, + page_revision=page_revision, + ) + + for resource in PontoonResource.objects.select_related( + "page", "current_page_revision" + ).iterator(): + page_specific = get_page_specific_instance(resource.page) + resource.object = get_translatable_object_from_page(page_specific) + + if resource.current_page_revision: + resource.current_revision = get_revision_from_page_revision( + resource.current_page_revision + ) + + resource.save() + + for submission in PontoonResourceSubmission.objects.select_related( + "page_revision" + ).iterator(): + submission.revision = get_revision_from_page_revision(submission.page_revision) + submission.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_pontoon", "0008_add_new_revision_fields"), + ] + + operations = [ + migrations.RunPython(populate_new_revision_fields, migrations.RunPython.noop), + ] diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index 89de97de..08b68a59 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -32,6 +32,13 @@ class PontoonResource(models.Model): "wagtailcore.Page", on_delete=models.CASCADE, primary_key=True, related_name="+" ) + object = models.OneToOneField( + "wagtail_localize_translation_memory.TranslatableObject", + on_delete=models.CASCADE, + null=True, + related_name="+", + ) + # The path within the locale folder in the git repository to push the PO file to # This is initially the pages URL path but is not updated if the page is moved path = models.CharField(max_length=255, unique=True) @@ -45,6 +52,13 @@ class PontoonResource(models.Model): related_name="+", ) + current_revision = models.OneToOneField( + "wagtail_localize_translation_memory.TranslatableRevision", + on_delete=models.SET_NULL, + null=True, + related_name="+", + ) + @classmethod def get_unique_path_from_urlpath(cls, url_path): """ @@ -251,6 +265,12 @@ class PontoonResourceSubmission(models.Model): on_delete=models.CASCADE, related_name="pontoon_submission", ) + revision = models.OneToOneField( + "wagtail_localize_translation_memory.TranslatableRevision", + on_delete=models.CASCADE, + null=True, + related_name="pontoon_submission", + ) created_at = models.DateTimeField(auto_now_add=True) # created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT, related_name='+') @@ -290,6 +310,7 @@ class PontoonResourceTranslation(models.Model): ) # The revision of the page that was created when the translations were saved + # Note: This field is not used anywhere page_revision = models.OneToOneField( "wagtailcore.PageRevision", on_delete=models.CASCADE, From 9aacee84e5d88d702dc66f8953a1db6996789474 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Tue, 3 Dec 2019 15:26:04 +0000 Subject: [PATCH 10/21] Remove paragraph indices from content paths These aren't stable between revisions. Removing them would allow us to use the content path as segment context as it would no longer change whenever a new paragraph is added. --- wagtail_localize/segments/extract.py | 6 ++--- wagtail_localize/segments/ingest.py | 22 +++++++------------ .../segments/tests/test_segment_extraction.py | 12 +++++----- .../segments/tests/test_segment_ingestion.py | 15 ++++++++----- 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/wagtail_localize/segments/extract.py b/wagtail_localize/segments/extract.py index c7479902..d733e189 100644 --- a/wagtail_localize/segments/extract.py +++ b/wagtail_localize/segments/extract.py @@ -27,8 +27,7 @@ def handle_block(self, block_type, block_value): template, texts = extract_html_segments(block_value.source) return [TemplateValue("", "html", template, len(texts))] + [ - SegmentValue.from_html(str(position), text) - for position, text in enumerate(texts) + SegmentValue.from_html("", text) for text in texts ] elif isinstance(block_type, (ImageChooserBlock, SnippetChooserBlock)): @@ -113,8 +112,7 @@ def extract_segments(instance): template, texts = extract_html_segments(field.value_from_object(instance)) field_segments = [TemplateValue("", "html", template, len(texts))] + [ - SegmentValue.from_html(str(position), text) - for position, text in enumerate(texts) + SegmentValue.from_html("", text) for text in texts ] segments.extend(segment.wrap(field.name) for segment in field_segments) diff --git a/wagtail_localize/segments/ingest.py b/wagtail_localize/segments/ingest.py index 6bf6c8c2..d031f385 100644 --- a/wagtail_localize/segments/ingest.py +++ b/wagtail_localize/segments/ingest.py @@ -15,20 +15,14 @@ def organise_template_segments(segments): - template = None - segments_by_position = {} - - for segment in segments: - if isinstance(segment, TemplateValue): - template = segment - else: - segments_by_position[int(segment.path)] = segment - - segments = [] - for position in range(template.segment_count): - segments.append(segments_by_position[position].html) - - return template.format, template.template, segments + # The first segment is always the template, followed by the texts in order of their position + segments.sort(key=lambda segment: segment.order) + template = segments[0] + return ( + template.format, + template.template, + [segment.html for segment in segments[1:]], + ) def handle_related_object(related_object, src_locale, tgt_locale, segments): diff --git a/wagtail_localize/segments/tests/test_segment_extraction.py b/wagtail_localize/segments/tests/test_segment_extraction.py index 7c419a07..c3754516 100644 --- a/wagtail_localize/segments/tests/test_segment_extraction.py +++ b/wagtail_localize/segments/tests/test_segment_extraction.py @@ -26,14 +26,14 @@ def make_test_page(**kwargs): '

', 3, ), - SegmentValue("0", "This is a heading", html_elements=[]), + SegmentValue("", "This is a heading", html_elements=[]), SegmentValue( - "1", + "", "This is a paragraph. Bold text", html_elements=[SegmentValue.HTMLElement(27, 36, "b1", ("b", {}))], ), SegmentValue( - "2", + "", "This is a link.", html_elements=[ SegmentValue.HTMLElement(0, 14, "a1", ("a", {"href": "http://example.com"})) @@ -266,10 +266,8 @@ def test_listblock(self): self.assertEqual( segments, [ - SegmentValue(f"test_streamfield.{block_id}.0", "Test content"), - SegmentValue( - f"test_streamfield.{block_id}.1", "Some more test content" - ), + SegmentValue(f"test_streamfield.{block_id}", "Test content"), + SegmentValue(f"test_streamfield.{block_id}", "Some more test content"), ], ) diff --git a/wagtail_localize/segments/tests/test_segment_ingestion.py b/wagtail_localize/segments/tests/test_segment_ingestion.py index 70e5c78d..e8f56c28 100644 --- a/wagtail_localize/segments/tests/test_segment_ingestion.py +++ b/wagtail_localize/segments/tests/test_segment_ingestion.py @@ -27,19 +27,22 @@ def make_test_page(**kwargs): "html", '

', 3, + order=9, ), - SegmentValue("0", "Ceci est une rubrique", html_elements=[]), + SegmentValue("", "Ceci est une rubrique", html_elements=[], order=10), SegmentValue( - "1", + "", "Ceci est un paragraphe. Texte en gras", html_elements=[SegmentValue.HTMLElement(30, 43, "b1", ("b", {}))], + order=11, ), SegmentValue( - "2", + "", "Ceci est un lien", html_elements=[ SegmentValue.HTMLElement(0, 16, "a1", ("a", {"href": "http://example.com"})) ], + order=12, ), ] @@ -510,9 +513,11 @@ def test_listblock(self): self.src_locale, self.locale, [ - SegmentValue(f"test_streamfield.{block_id}.0", "Tester le contenu"), SegmentValue( - f"test_streamfield.{block_id}.1", "Encore du contenu de test" + f"test_streamfield.{block_id}", "Tester le contenu", order=0 + ), + SegmentValue( + f"test_streamfield.{block_id}", "Encore du contenu de test", order=1 ), ], ) From fa91773d00ee59e79882c6a52d1fdae41cd30b19 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 17:07:52 +0000 Subject: [PATCH 11/21] Pontoon: Update logic to use new revision fields --- .../translation_engines/pontoon/importer.py | 41 ++++++---- .../commands/submit_whole_site_to_pontoon.py | 6 +- .../translation_engines/pontoon/models.py | 80 ++++++++++++------- .../translation_engines/pontoon/sync.py | 2 +- .../pontoon/tests/test_importer.py | 30 ++++++- .../pontoon/tests/test_pofile_generation.py | 15 ++-- 6 files changed, 118 insertions(+), 56 deletions(-) diff --git a/wagtail_localize/translation_engines/pontoon/importer.py b/wagtail_localize/translation_engines/pontoon/importer.py index 06c73632..a437ab95 100644 --- a/wagtail_localize/translation_engines/pontoon/importer.py +++ b/wagtail_localize/translation_engines/pontoon/importer.py @@ -6,13 +6,18 @@ import polib from wagtail.core.models import Page -from wagtail_localize.models import Locale, Region, ParentNotTranslatedError +from wagtail_localize.models import ( + Locale, + Region, + ParentNotTranslatedError, + TranslatablePageMixin, +) from wagtail_localize.segments import SegmentValue, TemplateValue from wagtail_localize.segments.ingest import ingest_segments from wagtail_localize.translation_memory.models import ( Segment, - SegmentPageLocation, - TemplatePageLocation, + SegmentLocation, + TemplateLocation, ) from .models import ( @@ -39,7 +44,7 @@ def create_or_update_translated_page(self, submission, language): region_id=Region.objects.default_id(), language=language ) - page = submission.page_revision.as_page_object() + page = submission.revision.page_revision.as_page_object() try: translated_page = page.get_translation(locale) @@ -50,17 +55,17 @@ def create_or_update_translated_page(self, submission, language): created = True # Fetch all translated segments - segment_page_locations = SegmentPageLocation.objects.filter( - page_revision=submission.page_revision + segment_locations = SegmentLocation.objects.filter( + revision=submission.revision ).annotate_translation(language) - template_page_locations = TemplatePageLocation.objects.filter( - page_revision=submission.page_revision + template_locations = TemplateLocation.objects.filter( + revision=submission.revision ).select_related("template") segments = [] - for page_location in segment_page_locations: + for page_location in segment_locations: segment = SegmentValue.from_html( page_location.path, page_location.translation ) @@ -69,7 +74,7 @@ def create_or_update_translated_page(self, submission, language): segments.append(segment) - for page_location in template_page_locations: + for page_location in template_locations: template = page_location.template segment = TemplateValue( page_location.path, @@ -90,7 +95,7 @@ def create_or_update_translated_page(self, submission, language): new_revision.publish() PontoonResourceTranslation.objects.create( - submission=submission, language=language, page_revision=new_revision + submission=submission, language=language ) return new_revision, created @@ -125,7 +130,7 @@ def try_update_resource_translation(self, resource, language): if translatable_submission: self.logger.info( - f"Saving translated page for '{resource.page.title}' in {language.get_display_name()}" + f"Saving translation for '{resource.path}' in {language.get_display_name()}" ) try: @@ -135,14 +140,20 @@ def try_update_resource_translation(self, resource, language): except ParentNotTranslatedError: # These pages will be handled when the parent is created in the code below self.logger.info( - f"Cannot save translated page for '{resource.page.title}' in {language.get_display_name()} yet as its parent must be translated first" + f"Cannot save translation for '{resource.path}' in {language.get_display_name()} yet as its parent must be translated first" ) return - if created: + if created and translatable_submission.revision.page_revision is not None: + source_page = translatable_submission.revision.page_revision.page + # Check if this page has any children that may be ready to translate child_page_resources = PontoonResource.objects.filter( - page__in=resource.page.get_children() + object__translation_key__in=[ + child.translation_key + for child in source_page.get_children().specific() + if isinstance(child, TranslatablePageMixin) + ] ) for resource in child_page_resources: diff --git a/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py b/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py index cdbd3b57..c0b65ba1 100644 --- a/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py +++ b/wagtail_localize/translation_engines/pontoon/management/commands/submit_whole_site_to_pontoon.py @@ -3,7 +3,7 @@ from wagtail.core.models import Page from wagtail_localize.models import TranslatablePageMixin, Locale -from ...models import submit_to_pontoon, PontoonResourceSubmission +from ...models import submit_page_to_pontoon, PontoonResourceSubmission class Command(BaseCommand): @@ -30,9 +30,9 @@ def handle(self, **options): print(f"Submitting '{page.title}'") if PontoonResourceSubmission.objects.filter( - page_revision=page_revision + revision__page_revision=page_revision ).exists(): print("Already submitted!") continue - submit_to_pontoon(page, page_revision) + submit_page_to_pontoon(page_revision) diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index 08b68a59..bc2899fd 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -9,7 +9,12 @@ from wagtail.core.signals import page_published from wagtail_localize.models import TranslatablePageMixin, Locale, Language from wagtail_localize.segments.extract import extract_segments -from wagtail_localize.translation_memory.models import Segment, SegmentPageLocation +from wagtail_localize.translation_memory.models import ( + Segment, + SegmentLocation, + TranslatableObject, + TranslatableRevision, +) from wagtail_localize.translation_memory.utils import ( insert_segments, get_translation_progress, @@ -43,8 +48,6 @@ class PontoonResource(models.Model): # This is initially the pages URL path but is not updated if the page is moved path = models.CharField(max_length=255, unique=True) - # The last page_revision to be submitted for translation - # This is denormalised from the page_revision field of the latest submission for this resource current_page_revision = models.OneToOneField( "wagtailcore.PageRevision", on_delete=models.SET_NULL, @@ -52,6 +55,8 @@ class PontoonResource(models.Model): related_name="+", ) + # The last revision to be submitted for translation + # This is denormalised from the revision field of the latest submission for this resource current_revision = models.OneToOneField( "wagtail_localize_translation_memory.TranslatableRevision", on_delete=models.SET_NULL, @@ -145,24 +150,21 @@ def get_segments(self): """ Gets all segments that are in the latest submission to Pontoon. """ - return Segment.objects.filter( - page_locations__page_revision_id=self.current_page_revision_id - ) + return Segment.objects.filter(locations__revision_id=self.current_revision_id) def get_all_segments(self, annotate_obsolete=False): """ Gets all segments that have ever been submitted to Pontoon. """ segments = Segment.objects.filter( - page_locations__page_revision__pontoon_submission__resource_id=self.pk + locations__revision__pontoon_submission__resource_id=self.pk ) if annotate_obsolete: segments = segments.annotate( is_obsolete=~Exists( - SegmentPageLocation.objects.filter( - segment=OuterRef("pk"), - page_revision_id=self.current_page_revision_id, + SegmentLocation.objects.filter( + segment=OuterRef("pk"), revision_id=self.current_revision_id, ) ) ) @@ -292,7 +294,7 @@ def get_translation_progress(self, language): - The total number of segments in the submission to translate - The number of segments that have been translated into the target language """ - return get_translation_progress(self.page_revision_id, language) + return get_translation_progress(self.revision_id, language) class PontoonResourceTranslation(models.Model): @@ -320,32 +322,52 @@ class PontoonResourceTranslation(models.Model): created_at = models.DateTimeField(auto_now_add=True) +def submit_page_to_pontoon(page_revision): + object, created = TranslatableObject.objects.get_or_create_from_instance( + page_revision.page + ) + + resource, created = PontoonResource.objects.get_or_create( + object=object, + defaults={ + "path": PontoonResource.get_unique_path_from_urlpath( + page_revision.page.url_path + ), + }, + ) + + revision, created = TranslatableRevision.objects.get_or_create( + object=object, + page_revision=page_revision, + defaults={ + "locale_id": page_revision.page.locale_id, + "content_json": page_revision.content_json, + "created_at": page_revision.created_at, + }, + ) + + submit_to_pontoon(resource, revision) + + @transaction.atomic -def submit_to_pontoon(page, page_revision): - # Extract segments from page and save them to translation memory - insert_segments(page_revision, page.locale.language_id, extract_segments(page)) - - # Get/create resource - try: - resource = PontoonResource.objects.get(page=page) - resource.current_page_revision = page_revision - resource.save(update_fields=["current_page_revision"]) - except PontoonResource.DoesNotExist: - resource = PontoonResource.objects.create( - page=page, - current_page_revision=page_revision, - path=PontoonResource.get_unique_path_from_urlpath(page.url_path), - ) +def submit_to_pontoon(resource, revision): + # Extract segments from revision and save them to translation memory + insert_segments( + revision, revision.locale.language_id, extract_segments(revision.as_instance()) + ) + + resource.current_revision = revision + resource.save(update_fields=["current_revision"]) # Create submission - resource.submissions.create(page_revision=page_revision) + resource.submissions.create(revision=revision) @receiver(page_published) -def submit_page_to_pontoon(sender, **kwargs): +def submit_page_to_pontoon_on_publish(sender, **kwargs): if issubclass(sender, TranslatablePageMixin): page = kwargs["instance"] page_revision = kwargs["revision"] if page.locale_id == Locale.objects.default_id(): - submit_to_pontoon(page, page_revision) + submit_page_to_pontoon(page_revision) diff --git a/wagtail_localize/translation_engines/pontoon/sync.py b/wagtail_localize/translation_engines/pontoon/sync.py index 8238ad1b..ad4ab387 100644 --- a/wagtail_localize/translation_engines/pontoon/sync.py +++ b/wagtail_localize/translation_engines/pontoon/sync.py @@ -72,7 +72,7 @@ def update_po(filename, new_po_string): pushed_submission_ids = [] for submission in ( PontoonResourceSubmission.objects.filter( - page_revision_id=F("resource__current_page_revision_id") + revision_id=F("resource__current_revision_id") ) .select_related("resource") .order_by("resource__path") diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py index 6a61dc78..9c7dbfad 100644 --- a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py +++ b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py @@ -42,11 +42,15 @@ def setUp(self): test_charfield="The test translatable field", test_synchronizedfield="The test synchronized field", ) - self.resource = PontoonResource.objects.get(page=self.page) + self.resource = PontoonResource.objects.get( + object__translation_key=self.page.translation_key + ) self.language = Language.objects.create(code="fr-FR") def test_importer(self): + new_page_succeeded = False + with self.subTest(stage="New page"): po_v1 = create_test_po([("The test translatable field", "")]).encode( "utf-8" @@ -81,6 +85,13 @@ def test_importer(self): self.assertEqual(log_resource.resource, self.resource) self.assertEqual(log_resource.language, self.language) + new_page_succeeded = True + + # subTest swallows errors, but we don't want to proceed if there was an error + # I know we're not exactly using them as intended + if not new_page_succeeded: + return + # Perform another import updating the page # Much easier to do it this way than trying to construct all the models manually to match the result of the last test with self.subTest(stage="Update page"): @@ -126,7 +137,11 @@ def test_importer_doesnt_import_if_parent_not_translated(self): test_synchronizedfield="The test synchronized field", parent=self.page, ) - child_resource = PontoonResource.objects.get(page=child_page) + child_resource = PontoonResource.objects.get( + object__translation_key=child_page.translation_key + ) + + create_child_page_succeeded = False with self.subTest(stage="Create child page"): # Translate @@ -164,6 +179,13 @@ def test_importer_doesnt_import_if_parent_not_translated(self): self.assertEqual(log_resource.resource, child_resource) self.assertEqual(log_resource.language, self.language) + create_child_page_succeeded = True + + # subTest swallows errors, but we don't want to proceed if there was an error + # I know we're not exactly using them as intended + if not create_child_page_succeeded: + return + with self.subTest(stage="Create parent page"): po_v1 = create_test_po([("The test translatable field", "")]).encode( "utf-8" @@ -224,7 +246,9 @@ def setUp(self): slug="test-page", test_richtextfield='

The test translatable field.

', ) - self.resource = PontoonResource.objects.get(page=self.page) + self.resource = PontoonResource.objects.get( + object__translation_key=self.page.translation_key + ) self.language = Language.objects.create(code="fr-FR") diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py b/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py index 765c93a5..50e0ab18 100644 --- a/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py +++ b/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py @@ -10,7 +10,7 @@ from wagtail_localize.translation_memory.models import ( Segment, SegmentTranslation, - SegmentPageLocation, + SegmentLocation, ) from ..models import PontoonResource @@ -33,7 +33,9 @@ def setUp(self): test_charfield="The test translatable field", test_synchronizedfield="The test synchronized field", ) - self.resource = PontoonResource.objects.get(page=self.page) + self.resource = PontoonResource.objects.get( + object__translation_key=self.page.translation_key + ) def test_generate_source_pofile(self): pofile = generate_source_pofile(self.resource) @@ -50,7 +52,8 @@ def test_generate_source_pofile_with_multiple_revisions(self): new_revision = self.page.save_revision() new_revision.publish() - SegmentPageLocation.objects.filter(page_revision=new_revision).update( + + SegmentLocation.objects.filter(revision__page_revision=new_revision).update( order=F("order") + 1 ) @@ -70,7 +73,9 @@ def setUp(self): test_charfield="The test translatable field", test_synchronizedfield="The test synchronized field", ) - self.resource = PontoonResource.objects.get(page=self.page) + self.resource = PontoonResource.objects.get( + object__translation_key=self.page.translation_key + ) self.language = Language.objects.create(code="fr") def test_generate_language_pofile(self): @@ -129,7 +134,7 @@ def test_generate_language_pofile_with_multiple_revisions(self): new_revision = self.page.save_revision() new_revision.publish() - SegmentPageLocation.objects.filter(page_revision=new_revision).update( + SegmentLocation.objects.filter(revision__page_revision=new_revision).update( order=F("order") + 1 ) From 461e04d23509713e4a89586bd42bf2ada9908ae5 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 17:06:43 +0000 Subject: [PATCH 12/21] Pontoon: Remove old revision fields --- .../migrations/0010_remove_page_fields.py | 41 +++++++++++++++++++ .../translation_engines/pontoon/models.py | 23 +---------- 2 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0010_remove_page_fields.py diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0010_remove_page_fields.py b/wagtail_localize/translation_engines/pontoon/migrations/0010_remove_page_fields.py new file mode 100644 index 00000000..8ba4a4a1 --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0010_remove_page_fields.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.7 on 2019-12-18 16:41 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_pontoon", "0009_populate_new_revision_fields"), + ] + + operations = [ + migrations.RemoveField( + model_name="pontoonresource", name="current_page_revision", + ), + migrations.AlterField( + model_name="pontoonresource", + name="object", + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="wagtail_localize_translation_memory.TranslatableObject", + ), + ), + migrations.RemoveField( + model_name="pontoonresourcesubmission", name="page_revision", + ), + migrations.AlterField( + model_name="pontoonresourcesubmission", + name="revision", + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="pontoon_submission", + to="wagtail_localize_translation_memory.TranslatableRevision", + ), + ), + migrations.RemoveField( + model_name="pontoonresourcetranslation", name="page_revision", + ), + ] diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index bc2899fd..1ffcd0fc 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -40,7 +40,6 @@ class PontoonResource(models.Model): object = models.OneToOneField( "wagtail_localize_translation_memory.TranslatableObject", on_delete=models.CASCADE, - null=True, related_name="+", ) @@ -48,13 +47,6 @@ class PontoonResource(models.Model): # This is initially the pages URL path but is not updated if the page is moved path = models.CharField(max_length=255, unique=True) - current_page_revision = models.OneToOneField( - "wagtailcore.PageRevision", - on_delete=models.SET_NULL, - null=True, - related_name="+", - ) - # The last revision to be submitted for translation # This is denormalised from the revision field of the latest submission for this resource current_revision = models.OneToOneField( @@ -262,15 +254,10 @@ class PontoonResourceSubmission(models.Model): resource = models.ForeignKey( PontoonResource, on_delete=models.CASCADE, related_name="submissions" ) - page_revision = models.OneToOneField( - "wagtailcore.PageRevision", - on_delete=models.CASCADE, - related_name="pontoon_submission", - ) + revision = models.OneToOneField( "wagtail_localize_translation_memory.TranslatableRevision", on_delete=models.CASCADE, - null=True, related_name="pontoon_submission", ) created_at = models.DateTimeField(auto_now_add=True) @@ -311,14 +298,6 @@ class PontoonResourceTranslation(models.Model): related_name="pontoon_translations", ) - # The revision of the page that was created when the translations were saved - # Note: This field is not used anywhere - page_revision = models.OneToOneField( - "wagtailcore.PageRevision", - on_delete=models.CASCADE, - related_name="pontoon_translation", - ) - created_at = models.DateTimeField(auto_now_add=True) From ae537598ea5d31ca1ff3d4749a7986e322ca680d Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 18 Dec 2019 18:56:55 +0000 Subject: [PATCH 13/21] Pontoon: Change PontoonResource primary key --- .../0011_replace_pontoonresource.py | 76 +++++++++++++++++++ .../0012_populate_new_resource_model.py | 43 +++++++++++ .../0013_remove_old_resource_model.py | 54 +++++++++++++ .../translation_engines/pontoon/models.py | 5 +- 4 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0011_replace_pontoonresource.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0012_populate_new_resource_model.py create mode 100644 wagtail_localize/translation_engines/pontoon/migrations/0013_remove_old_resource_model.py diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0011_replace_pontoonresource.py b/wagtail_localize/translation_engines/pontoon/migrations/0011_replace_pontoonresource.py new file mode 100644 index 00000000..8bb2a57f --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0011_replace_pontoonresource.py @@ -0,0 +1,76 @@ +# Generated by Django 2.2.9 on 2019-12-18 18:44 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ("wagtail_localize_translation_memory", "0009_migrate_to_new_location_models"), + ("wagtailcore", "0041_group_collection_permissions_verbose_name_plural"), + ("wagtail_localize_pontoon", "0010_remove_page_fields"), + ] + + operations = [ + migrations.RenameModel( + old_name="PontoonResource", new_name="OldPontoonResource", + ), + migrations.CreateModel( + name="NewPontoonResource", + fields=[ + ( + "object", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="+", + serialize=False, + to="wagtail_localize_translation_memory.TranslatableObject", + ), + ), + ("path", models.CharField(max_length=255, unique=True)), + ( + "current_revision", + models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtail_localize_translation_memory.TranslatableRevision", + ), + ), + ], + ), + migrations.RenameField( + model_name="pontoonresourcesubmission", + old_name="resource", + new_name="old_resource", + ), + migrations.RenameField( + model_name="pontoonsynclogresource", + old_name="resource", + new_name="old_resource", + ), + migrations.AddField( + model_name="pontoonresourcesubmission", + name="new_resource", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="submissions", + to="wagtail_localize_pontoon.NewPontoonResource", + ), + ), + migrations.AddField( + model_name="pontoonsynclogresource", + name="new_resource", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="logs", + to="wagtail_localize_pontoon.NewPontoonResource", + ), + ), + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0012_populate_new_resource_model.py b/wagtail_localize/translation_engines/pontoon/migrations/0012_populate_new_resource_model.py new file mode 100644 index 00000000..25dd353a --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0012_populate_new_resource_model.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.9 on 2019-12-18 18:47 + +from django.db import migrations + + +def populate_new_resource_model(apps, schema_editor): + OldResource = apps.get_model("wagtail_localize_pontoon.OldPontoonResource") + NewResource = apps.get_model("wagtail_localize_pontoon.NewPontoonResource") + SyncLogResource = apps.get_model("wagtail_localize_pontoon.PontoonSyncLogResource") + ResourceSubmission = apps.get_model( + "wagtail_localize_pontoon.PontoonResourceSubmission" + ) + + # Create new resources + id_mapping = {} + for resource in OldResource.objects.iterator(): + id_mapping[resource.page_id] = resource.object_id + + NewResource.objects.create( + object_id=resource.object_id, + path=resource.path, + current_revision_id=resource.current_revision_id, + ) + + # Populate new foreign key fields + for page_id, object_id in id_mapping.items(): + SyncLogResource.objects.filter(old_resource_id=page_id).update( + new_resource_id=object_id + ) + ResourceSubmission.objects.filter(old_resource_id=page_id).update( + new_resource_id=object_id + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_pontoon", "0011_replace_pontoonresource"), + ] + + operations = [ + migrations.RunPython(populate_new_resource_model), + ] diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0013_remove_old_resource_model.py b/wagtail_localize/translation_engines/pontoon/migrations/0013_remove_old_resource_model.py new file mode 100644 index 00000000..1b90aeed --- /dev/null +++ b/wagtail_localize/translation_engines/pontoon/migrations/0013_remove_old_resource_model.py @@ -0,0 +1,54 @@ +# Generated by Django 2.2.9 on 2019-12-18 18:54 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_pontoon", "0012_populate_new_resource_model"), + ] + + operations = [ + migrations.RemoveField( + model_name="pontoonresourcesubmission", name="old_resource", + ), + migrations.RemoveField( + model_name="pontoonsynclogresource", name="old_resource", + ), + migrations.AlterField( + model_name="pontoonresourcesubmission", + name="new_resource", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="submissions", + to="wagtail_localize_pontoon.NewPontoonResource", + ), + ), + migrations.AlterField( + model_name="pontoonsynclogresource", + name="new_resource", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="logs", + to="wagtail_localize_pontoon.NewPontoonResource", + ), + ), + migrations.DeleteModel(name="OldPontoonResource",), + # Must be after deleting OldPontoonResource to prevent name clashes with + # previous indexes. Django sometimes doesn't rename them. + migrations.RenameModel( + old_name="NewPontoonResource", new_name="PontoonResource", + ), + migrations.RenameField( + model_name="pontoonresourcesubmission", + old_name="new_resource", + new_name="resource", + ), + migrations.RenameField( + model_name="pontoonsynclogresource", + old_name="new_resource", + new_name="resource", + ), + ] diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index 1ffcd0fc..6f40aff1 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -33,13 +33,10 @@ class PontoonSyncLog(models.Model): class PontoonResource(models.Model): - page = models.OneToOneField( - "wagtailcore.Page", on_delete=models.CASCADE, primary_key=True, related_name="+" - ) - object = models.OneToOneField( "wagtail_localize_translation_memory.TranslatableObject", on_delete=models.CASCADE, + primary_key=True, related_name="+", ) From 4b5ec73e900a7acfb57b78f58a6d400b0206e049 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Fri, 20 Dec 2019 10:31:45 +0000 Subject: [PATCH 14/21] Update translation memory utils to use new location models --- wagtail_localize/translation_memory/utils.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/wagtail_localize/translation_memory/utils.py b/wagtail_localize/translation_memory/utils.py index 12fa0ff0..034c8c5f 100644 --- a/wagtail_localize/translation_memory/utils.py +++ b/wagtail_localize/translation_memory/utils.py @@ -15,12 +15,12 @@ from .models import ( Segment, SegmentTranslation, - SegmentPageLocation, - TemplatePageLocation, + SegmentLocation, + TemplateLocation, ) -def get_translation_progress(page_revision_id, language): +def get_translation_progress(revision_id, language): """ For the specified page revision, get the current translation progress into the specified language. @@ -31,9 +31,9 @@ def get_translation_progress(page_revision_id, language): """ # Get QuerySet of Segments that need to be translated required_segments = Segment.objects.filter( - id__in=SegmentPageLocation.objects.filter( - page_revision_id=page_revision_id - ).values_list("segment_id") + id__in=SegmentLocation.objects.filter(revision_id=revision_id).values_list( + "segment_id" + ) ) # Annotate each Segment with a flag that indicates whether the segment is translated @@ -58,12 +58,12 @@ def get_translation_progress(page_revision_id, language): return aggs["total_segments"], aggs["translated_segments"] -def insert_segments(page_revision, language, segments): +def insert_segments(revision, language, segments): """ Inserts the list of untranslated segments into translation memory """ for segment in segments: if isinstance(segment, TemplateValue): - TemplatePageLocation.from_template_value(page_revision, segment) + TemplateLocation.from_template_value(revision, segment) else: - SegmentPageLocation.from_segment_value(page_revision, language, segment) + SegmentLocation.from_segment_value(revision, language, segment) From 129d472f5abb4eb0e233fd3c5518862b63ccc352 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Fri, 20 Dec 2019 10:33:40 +0000 Subject: [PATCH 15/21] Delete old location models --- .../0010_delete_old_location_models.py | 19 ++++ wagtail_localize/translation_memory/models.py | 88 ------------------- 2 files changed, 19 insertions(+), 88 deletions(-) create mode 100644 wagtail_localize/translation_memory/migrations/0010_delete_old_location_models.py diff --git a/wagtail_localize/translation_memory/migrations/0010_delete_old_location_models.py b/wagtail_localize/translation_memory/migrations/0010_delete_old_location_models.py new file mode 100644 index 00000000..cae38b7d --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0010_delete_old_location_models.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.7 on 2019-12-20 10:33 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_translation_memory", "0009_migrate_to_new_location_models"), + ] + + operations = [ + migrations.RemoveField( + model_name="templatepagelocation", name="page_revision", + ), + migrations.RemoveField(model_name="templatepagelocation", name="template",), + migrations.DeleteModel(name="SegmentPageLocation",), + migrations.DeleteModel(name="TemplatePageLocation",), + ] diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index 0005c545..74431485 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -252,91 +252,3 @@ def from_template_value(cls, revision, template_value): ) return template_loc - - -# LEGACY PageLocation models - - -class BasePageLocation(models.Model): - page_revision = models.ForeignKey( - "wagtailcore.PageRevision", on_delete=models.CASCADE - ) - path = models.TextField() - order = models.PositiveIntegerField() - - class Meta: - abstract = True - - -class SegmentPageLocationQuerySet(models.QuerySet): - def annotate_translation(self, language): - """ - Adds a 'translation' field to the segments containing the - text content of the segment translated into the specified - language. - """ - return self.annotate( - translation=Subquery( - SegmentTranslation.objects.filter( - translation_of_id=OuterRef("segment_id"), language_id=pk(language) - ).values("text") - ) - ) - - -class SegmentPageLocation(BasePageLocation): - segment = models.ForeignKey( - Segment, on_delete=models.CASCADE, related_name="page_locations" - ) - - # When we extract the segment, we replace HTML attributes with id tags - # The attributes that were removed are stored here. These must be - # added into the translated strings. - # These are stored as a mapping of element ids to KV mappings of - # attributes in JSON format. For example: - # - # For this segment: Link to example.com - # - # The value of this field could be: - # - # { - # "a#a1": { - # "href": "https://www.example.com" - # } - # } - html_attrs = models.TextField(blank=True) - - objects = SegmentPageLocationQuerySet.as_manager() - - @classmethod - def from_segment_value(cls, page_revision, language, segment_value): - segment = Segment.from_text(language, segment_value.html_with_ids) - - segment_page_loc, created = cls.objects.get_or_create( - page_revision_id=pk(page_revision), - path=segment_value.path, - order=segment_value.order, - segment=segment, - html_attrs=json.dumps(segment_value.get_html_attrs()), - ) - - return segment_page_loc - - -class TemplatePageLocation(BasePageLocation): - template = models.ForeignKey( - Template, on_delete=models.CASCADE, related_name="page_locations" - ) - - @classmethod - def from_template_value(cls, page_revision, template_value): - template = Template.from_template_value(template_value) - - template_page_loc, created = cls.objects.get_or_create( - page_revision_id=pk(page_revision), - path=template_value.path, - order=template_value.order, - template=template, - ) - - return template_page_loc From 30b6e8e276e4fa8c357d2d33692bacd978a1890f Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Fri, 20 Dec 2019 15:48:57 +0000 Subject: [PATCH 16/21] Implement serialization of non-page objects into translatable revision --- .../translation_engines/pontoon/models.py | 16 +---- wagtail_localize/translation_memory/models.py | 69 +++++++++++++++++-- 2 files changed, 67 insertions(+), 18 deletions(-) diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index 6f40aff1..1e9b9218 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -299,12 +299,12 @@ class PontoonResourceTranslation(models.Model): def submit_page_to_pontoon(page_revision): - object, created = TranslatableObject.objects.get_or_create_from_instance( - page_revision.page + revision, created = TranslatableRevision.get_or_create_from_page_revision( + page_revision ) resource, created = PontoonResource.objects.get_or_create( - object=object, + object=revision.object, defaults={ "path": PontoonResource.get_unique_path_from_urlpath( page_revision.page.url_path @@ -312,16 +312,6 @@ def submit_page_to_pontoon(page_revision): }, ) - revision, created = TranslatableRevision.objects.get_or_create( - object=object, - page_revision=page_revision, - defaults={ - "locale_id": page_revision.page.locale_id, - "content_json": page_revision.content_json, - "created_at": page_revision.created_at, - }, - ) - submit_to_pontoon(resource, revision) diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index 74431485..95979bac 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -2,8 +2,16 @@ import uuid from django.contrib.contenttypes.models import ContentType +from django.core.serializers.json import DjangoJSONEncoder from django.db import models, transaction from django.db.models import Subquery, OuterRef +from django.utils import timezone +from modelcluster.models import ( + ClusterableModel, + get_serializable_data_for_fields, + model_from_serializable_data, +) +from wagtail.core.models import Page def pk(obj): @@ -66,14 +74,65 @@ class TranslatableRevision(models.Model): related_name="wagtaillocalize_revision", ) - def as_instance(self): - if self.page_revision is not None: - return self.page_revision.as_page_object() + @classmethod + def get_or_create_from_page_revision(cls, page_revision): + object, created = TranslatableObject.objects.get_or_create_from_instance( + page_revision.page + ) + + return TranslatableRevision.objects.get_or_create( + object=object, + page_revision=page_revision, + defaults={ + "locale_id": page_revision.page.locale_id, + "content_json": page_revision.content_json, + "created_at": page_revision.created_at, + }, + ) + + @classmethod + def from_instance(cls, instance): + object, created = TranslatableObject.objects.get_or_create_from_instance( + instance + ) - raise NotImplementedError( - "revisions of non-page objects not currently supported" + if isinstance(instance, ClusterableModel): + content_json = instance.to_json() + else: + serializable_data = get_serializable_data_for_fields(instance) + content_json = json.dumps(serializable_data, cls=DjangoJSONEncoder) + + return cls.objects.create( + object=object, + locale=instance.locale, + content_json=content_json, + created_at=timezone.now(), ) + def as_instance(self): + """ + Builds an instance of the object with the content at this revision. + """ + instance = self.object.get_instance(self.locale) + + if isinstance(instance, Page): + return instance.with_content_json(self.content_json) + + elif isinstance(instance, ClusterableModel): + new_instance = instance.__class__.from_json(self.content_json) + + else: + new_instance = model_from_serializable_data( + instance.__class__, json.loads(self.content_json) + ) + + new_instance.pk = instance.pk + new_instance.locale = instance.locale + new_instance.translation_key = instance.translation_key + new_instance.is_source_translation = instance.is_source_translation + + return new_instance + class SegmentQuerySet(models.QuerySet): def annotate_translation(self, language): From 8784c7daccd996190a0051c4b4affba04b6e2618 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Sat, 28 Dec 2019 20:37:21 +0000 Subject: [PATCH 17/21] Moved translation code into translation memory --- .../translation_engines/pontoon/importer.py | 81 +++---------------- wagtail_localize/translation_memory/models.py | 76 +++++++++++++++++ 2 files changed, 89 insertions(+), 68 deletions(-) diff --git a/wagtail_localize/translation_engines/pontoon/importer.py b/wagtail_localize/translation_engines/pontoon/importer.py index a437ab95..328e35d3 100644 --- a/wagtail_localize/translation_engines/pontoon/importer.py +++ b/wagtail_localize/translation_engines/pontoon/importer.py @@ -34,72 +34,6 @@ def __init__(self, source_language, logger): self.logger = logger self.log = None - def create_or_update_translated_page(self, submission, language): - """ - Creates/updates the translated page to reflect the translations in translation memory. - - Note, all strings in the submission must be translated into the target language! - """ - locale = Locale.objects.get( - region_id=Region.objects.default_id(), language=language - ) - - page = submission.revision.page_revision.as_page_object() - - try: - translated_page = page.get_translation(locale) - created = False - except page.specific_class.DoesNotExist: - # May raise ParentNotTranslatedError - translated_page = page.copy_for_translation(locale) - created = True - - # Fetch all translated segments - segment_locations = SegmentLocation.objects.filter( - revision=submission.revision - ).annotate_translation(language) - - template_locations = TemplateLocation.objects.filter( - revision=submission.revision - ).select_related("template") - - segments = [] - - for page_location in segment_locations: - segment = SegmentValue.from_html( - page_location.path, page_location.translation - ) - if page_location.html_attrs: - segment.replace_html_attrs(json.loads(page_location.html_attrs)) - - segments.append(segment) - - for page_location in template_locations: - template = page_location.template - segment = TemplateValue( - page_location.path, - template.template_format, - template.template, - template.segment_count, - ) - segments.append(segment) - - # Ingest all translated segments into page - ingest_segments(page, translated_page, page.locale.language, language, segments) - - # Make sure the slug is valid - translated_page.slug = slugify(translated_page.slug) - translated_page.save() - - new_revision = translated_page.save_revision() - new_revision.publish() - - PontoonResourceTranslation.objects.create( - submission=submission, language=language - ) - - return new_revision, created - def import_resource(self, resource, language, old_po, new_po): for changed_entry in set(new_po) - set(old_po): try: @@ -129,13 +63,24 @@ def try_update_resource_translation(self, resource, language): translatable_submission = resource.find_translatable_submission(language) if translatable_submission: + locale = Locale.objects.get( + region_id=Region.objects.default_id(), language=language + ) + self.logger.info( f"Saving translation for '{resource.path}' in {language.get_display_name()}" ) try: - revision, created = self.create_or_update_translated_page( - translatable_submission, language + ( + translation, + created, + ) = translatable_submission.revision.create_or_update_translation( + locale + ) + + PontoonResourceTranslation.objects.create( + submission=translatable_submission, language=language ) except ParentNotTranslatedError: # These pages will be handled when the parent is created in the code below diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index 95979bac..ce8f903b 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -6,6 +6,7 @@ from django.db import models, transaction from django.db.models import Subquery, OuterRef from django.utils import timezone +from django.utils.text import slugify from modelcluster.models import ( ClusterableModel, get_serializable_data_for_fields, @@ -13,6 +14,9 @@ ) from wagtail.core.models import Page +from wagtail_localize.segments import SegmentValue, TemplateValue +from wagtail_localize.segments.ingest import ingest_segments + def pk(obj): if isinstance(obj, models.Model): @@ -45,6 +49,11 @@ class TranslatableObject(models.Model): objects = TranslatableObjectManager() + def has_translation(self, locale): + return self.content_type.get_all_objects_for_this_type( + translation_key=self.translation_key, locale=locale + ).exists() + def get_instance(self, locale): return self.content_type.get_object_for_this_type( translation_key=self.translation_key, locale=locale @@ -133,6 +142,73 @@ def as_instance(self): return new_instance + def create_or_update_translation(self, locale, translation=None): + """ + Creates/updates a translation of the object into the specified locale + based on the content of this source and the translated strings + currently in translation memory. + + If the translation already exists, you can pass the object in with the + `translation` keyword argument. Otherwise, this object will be fetched + or created by this method. + """ + original = self.as_instance() + created = False + + if translation is None: + try: + translation = self.object.get_instance(locale) + except models.ObjectDoesNotExist: + translation = original.copy_for_translation(locale) + created = True + + # Fetch all translated segments + segment_locations = SegmentLocation.objects.filter( + revision=self + ).annotate_translation(locale.language) + + template_locations = TemplateLocation.objects.filter( + revision=self + ).select_related("template") + + segments = [] + + for location in segment_locations: + segment = SegmentValue.from_html(location.path, location.translation).with_order(location.order) + if location.html_attrs: + segment.replace_html_attrs(json.loads(location.html_attrs)) + + segments.append(segment) + + for location in template_locations: + template = location.template + segment = TemplateValue( + location.path, + template.template_format, + template.template, + template.segment_count, + order=location.order, + ) + segments.append(segment) + + # Ingest all translated segments + ingest_segments(original, translation, self.locale, locale, segments) + + # TODO: Copy synchronised fields + + if isinstance(translation, Page): + # Make sure the slug is valid + translation.slug = slugify(translation.slug) + translation.save() + + # Create a new revision + new_revision = translation.save_revision() + new_revision.publish() + else: + translation.save() + + return translation, created + class SegmentQuerySet(models.QuerySet): def annotate_translation(self, language): From 7f7280892b193ec47590aebb0cdeffb8339804dc Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Mon, 23 Dec 2019 15:40:44 +0000 Subject: [PATCH 18/21] Treat related objects as separate resources --- wagtail_localize/segments/__init__.py | 42 ++++++++ wagtail_localize/segments/extract.py | 13 +-- wagtail_localize/segments/ingest.py | 19 +--- .../segments/tests/test_segment_extraction.py | 6 +- .../segments/tests/test_segment_ingestion.py | 23 ++++- .../translation_engines/pontoon/importer.py | 60 +++++++++--- .../0013_remove_old_resource_model.py | 2 + .../translation_engines/pontoon/models.py | 97 +++++++++++++++++-- .../pontoon/tests/test_importer.py | 81 +++++++++++++++- .../migrations/0011_relatedobjectlocation.py | 46 +++++++++ wagtail_localize/translation_memory/models.py | 57 +++++++++-- wagtail_localize/translation_memory/utils.py | 7 +- 12 files changed, 391 insertions(+), 62 deletions(-) create mode 100644 wagtail_localize/translation_memory/migrations/0011_relatedobjectlocation.py diff --git a/wagtail_localize/segments/__init__.py b/wagtail_localize/segments/__init__.py index 42cd4c62..949109db 100644 --- a/wagtail_localize/segments/__init__.py +++ b/wagtail_localize/segments/__init__.py @@ -1,5 +1,6 @@ from collections import Counter +from django.contrib.contenttypes.models import ContentType from django.forms.utils import flatatt from django.utils.html import escape @@ -250,3 +251,44 @@ def __repr__(self): return "".format( self.path, self.format, self.segment_count ) + + +class RelatedObjectValue(BaseValue): + def __init__(self, path, content_type, translation_key, **kwargs): + self.content_type = content_type + self.translation_key = translation_key + + super().__init__(path, **kwargs) + + @classmethod + def from_instance(cls, path, instance): + model = instance.get_translation_model() + return cls( + path, ContentType.objects.get_for_model(model), instance.translation_key + ) + + def get_instance(self, locale): + return self.content_type.get_object_for_this_type( + translation_key=self.translation_key, locale=locale + ) + + def clone(self): + return RelatedObjectValue( + self.path, self.content_type, self.translation_key, order=self.order + ) + + def is_empty(self): + return self.content_type is None and self.translation_key is None + + def __eq__(self, other): + return ( + isinstance(other, RelatedObjectValue) + and self.path == other.path + and self.content_type == other.content_type + and self.translation_key == other.translation_key + ) + + def __repr__(self): + return "".format( + self.path, self.content_type, self.translation_key + ) diff --git a/wagtail_localize/segments/extract.py b/wagtail_localize/segments/extract.py index 37c6e9de..4641bd07 100644 --- a/wagtail_localize/segments/extract.py +++ b/wagtail_localize/segments/extract.py @@ -8,7 +8,7 @@ from wagtail.snippets.blocks import SnippetChooserBlock from wagtail_localize.models import TranslatableMixin -from wagtail_localize.segments import SegmentValue, TemplateValue +from wagtail_localize.segments import SegmentValue, TemplateValue, RelatedObjectValue from .html import extract_html_segments @@ -58,7 +58,7 @@ def handle_related_object_block(self, related_object): if related_object is None or not isinstance(related_object, TranslatableMixin): return [] - return extract_segments(related_object) + return RelatedObjectValue.from_instance("", related_object) def handle_struct_block(self, struct_block): segments = [] @@ -132,9 +132,8 @@ def extract_segments(instance): related_instance = getattr(instance, field.name) if related_instance: - segments.extend( - segment.wrap(field.name) - for segment in extract_segments(related_instance) + segments.append( + RelatedObjectValue.from_instance(field.name, related_instance) ) elif ( @@ -146,9 +145,7 @@ def extract_segments(instance): for child_instance in manager.all(): segments.extend( - segment.wrap( - "{}.{}".format(field.name, child_instance.translation_key) - ) + segment.wrap(str(child_instance.translation_key)).wrap(field.name) for segment in extract_segments(child_instance) ) diff --git a/wagtail_localize/segments/ingest.py b/wagtail_localize/segments/ingest.py index 6bf6c8c2..f976de4f 100644 --- a/wagtail_localize/segments/ingest.py +++ b/wagtail_localize/segments/ingest.py @@ -32,22 +32,9 @@ def organise_template_segments(segments): def handle_related_object(related_object, src_locale, tgt_locale, segments): - if related_object is None or not isinstance(related_object, TranslatableMixin): - return related_object - - # Note: when called from streamfield, we may be given the translated object - related_original = related_object.get_translation(src_locale) - related_translated = related_object.get_translation_or_none(tgt_locale) - - if related_translated is None: - # Create translated version by copying the original version - related_translated = related_original.copy_for_translation(tgt_locale) - - ingest_segments( - related_original, related_translated, src_locale, tgt_locale, segments - ) - related_translated.save() - return related_translated + # FIXME: Check that segments is a single item list + related_object_value = segments[0] + return related_object_value.get_instance(tgt_locale) class StreamFieldSegmentsWriter: diff --git a/wagtail_localize/segments/tests/test_segment_extraction.py b/wagtail_localize/segments/tests/test_segment_extraction.py index 38ae7f7d..7b672598 100644 --- a/wagtail_localize/segments/tests/test_segment_extraction.py +++ b/wagtail_localize/segments/tests/test_segment_extraction.py @@ -12,7 +12,7 @@ TestChildObject, TestNonParentalChildObject, ) -from wagtail_localize.segments import SegmentValue, TemplateValue +from wagtail_localize.segments import SegmentValue, TemplateValue, RelatedObjectValue from wagtail_localize.segments.extract import extract_segments @@ -96,7 +96,9 @@ def test_snippet(self): page = make_test_page(test_snippet=test_snippet) segments = extract_segments(page) - self.assertEqual(segments, [SegmentValue("test_snippet.field", "Test content")]) + self.assertEqual( + segments, [RelatedObjectValue.from_instance("test_snippet", test_snippet)] + ) def test_childobjects(self): page = make_test_page() diff --git a/wagtail_localize/segments/tests/test_segment_ingestion.py b/wagtail_localize/segments/tests/test_segment_ingestion.py index 70e5c78d..18065288 100644 --- a/wagtail_localize/segments/tests/test_segment_ingestion.py +++ b/wagtail_localize/segments/tests/test_segment_ingestion.py @@ -8,7 +8,7 @@ from wagtail_localize.models import Language, Locale from wagtail_localize.test.models import TestPage, TestSnippet, TestChildObject -from wagtail_localize.segments import SegmentValue, TemplateValue +from wagtail_localize.segments import SegmentValue, TemplateValue, RelatedObjectValue from wagtail_localize.segments.extract import extract_segments from wagtail_localize.segments.ingest import ingest_segments @@ -141,6 +141,23 @@ def test_richtextfield(self): def test_snippet(self): test_snippet = TestSnippet.objects.create(field="Test content") + translated_snippet = test_snippet.copy_for_translation(self.locale) + translated_snippet.save() + + # Ingest segments into the snippet + ingest_segments( + test_snippet, + translated_snippet, + self.src_locale, + self.locale, + [SegmentValue("field", "Tester le contenu")], + ) + + translated_snippet.save() + + self.assertEqual(translated_snippet.field, "Tester le contenu") + + # Now ingest a RelatedObjectValue into the page page = make_test_page(test_snippet=test_snippet) translated_page = page.copy_for_translation(self.locale) @@ -149,10 +166,10 @@ def test_snippet(self): translated_page, self.src_locale, self.locale, - [SegmentValue("test_snippet.field", "Tester le contenu")], + [RelatedObjectValue.from_instance("test_snippet", test_snippet)], ) - # Check the snippet was duplicated + # Check the translated snippet was linked to the translated page self.assertNotEqual(page.test_snippet_id, translated_page.test_snippet_id) self.assertEqual(page.test_snippet.locale, self.src_locale) self.assertEqual(translated_page.test_snippet.locale, self.locale) diff --git a/wagtail_localize/translation_engines/pontoon/importer.py b/wagtail_localize/translation_engines/pontoon/importer.py index 328e35d3..9ad06f0b 100644 --- a/wagtail_localize/translation_engines/pontoon/importer.py +++ b/wagtail_localize/translation_engines/pontoon/importer.py @@ -67,6 +67,13 @@ def try_update_resource_translation(self, resource, language): region_id=Region.objects.default_id(), language=language ) + for dependency in translatable_submission.get_dependencies(): + if not dependency.object.has_translation(locale): + self.logger.info( + f"Can't translate '{resource.path}' into {language.get_display_name()} because its dependency '{dependency.path}' hasn't been translated yet" + ) + return + self.logger.info( f"Saving translation for '{resource.path}' in {language.get_display_name()}" ) @@ -89,20 +96,45 @@ def try_update_resource_translation(self, resource, language): ) return - if created and translatable_submission.revision.page_revision is not None: - source_page = translatable_submission.revision.page_revision.page - - # Check if this page has any children that may be ready to translate - child_page_resources = PontoonResource.objects.filter( - object__translation_key__in=[ - child.translation_key - for child in source_page.get_children().specific() - if isinstance(child, TranslatablePageMixin) - ] - ) - - for resource in child_page_resources: - self.try_update_resource_translation(resource, language) + if created: + # The translation was created. + + # The logic in this part checks to find any other submissions that were waiting + # for this object to be translated first and triggers them to be translated as + # well. + + # Look for any submissions that were waiting for this object to be translated. + # For example, any linked snippets, images, etc must be translated before + # the page. So if a page (the dependee) is ready to be translated and its last + # linked item has just been translated, we can translate that page now. + for dependee in resource.get_dependees(): + # Check that the next translatable submission of the dependee resource is + # the dependee submission that's linked to this resource. + # This prevents us from translating an old submission that's been replaced. + if ( + dependee.resource.find_translatable_submission(language) + != dependee + ): + continue + + # FIXME: Could we pass in the translatable submission instead? + self.try_update_resource_translation(dependee.resource, language) + + # If a page was created, check to see if it has any children that are + # now ready to translate. + if translatable_submission.revision.page_revision is not None: + source_page = translatable_submission.revision.page_revision.page + + child_page_resources = PontoonResource.objects.filter( + object__translation_key__in=[ + child.translation_key + for child in source_page.get_children().specific() + if isinstance(child, TranslatablePageMixin) + ] + ) + + for resource in child_page_resources: + self.try_update_resource_translation(resource, language) def start_import(self, commit_id): self.log = PontoonSyncLog.objects.create( diff --git a/wagtail_localize/translation_engines/pontoon/migrations/0013_remove_old_resource_model.py b/wagtail_localize/translation_engines/pontoon/migrations/0013_remove_old_resource_model.py index 1b90aeed..8f6ff0e5 100644 --- a/wagtail_localize/translation_engines/pontoon/migrations/0013_remove_old_resource_model.py +++ b/wagtail_localize/translation_engines/pontoon/migrations/0013_remove_old_resource_model.py @@ -10,6 +10,8 @@ class Migration(migrations.Migration): ("wagtail_localize_pontoon", "0012_populate_new_resource_model"), ] + atomic = False + operations = [ migrations.RemoveField( model_name="pontoonresourcesubmission", name="old_resource", diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index 1e9b9218..5ebea840 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -5,15 +5,18 @@ from django.db.models import Exists, OuterRef from django.dispatch import receiver from django.utils import timezone +from django.utils.text import slugify from wagtail.core.signals import page_published from wagtail_localize.models import TranslatablePageMixin, Locale, Language +from wagtail_localize.segments import RelatedObjectValue from wagtail_localize.segments.extract import extract_segments from wagtail_localize.translation_memory.models import ( Segment, SegmentLocation, TranslatableObject, TranslatableRevision, + RelatedObjectLocation, ) from wagtail_localize.translation_memory.utils import ( insert_segments, @@ -198,6 +201,20 @@ def latest_submission(self): def latest_pushed_submission(self): return self.submissions.filter(pushed_at__isnull=False).latest("created_at") + def get_dependees(self): + """ + Gets a QuerySet of PontoonResourceSubmissions that depend on this Resource. + + This is where this resource reprensents a reusable snippet or images that + other resources link to and can't be translated until this resource is + translated. + + This is the opposite of PontoonResourceSubmission.get_dependencies + """ + return PontoonResourceSubmission.objects.filter( + revision_id__in=self.object.references.values_list("revision_id", flat=True) + ) + def __repr__(self): return f"" @@ -280,6 +297,21 @@ def get_translation_progress(self, language): """ return get_translation_progress(self.revision_id, language) + def get_dependencies(self): + """ + Gets a QuerySet of resources that this submission depends on. + + These are any linked objects (eg, snippets, images) that must + be translated before this resource can be translated. + + This is the opposite of PontoonResource.get_dependees + """ + return PontoonResource.objects.filter( + object_id__in=RelatedObjectLocation.objects.filter( + revision_id=self.revision_id + ).values_list("object_id", flat=True) + ) + class PontoonResourceTranslation(models.Model): """ @@ -298,28 +330,73 @@ class PontoonResourceTranslation(models.Model): created_at = models.DateTimeField(auto_now_add=True) -def submit_page_to_pontoon(page_revision): - revision, created = TranslatableRevision.get_or_create_from_page_revision( - page_revision - ) +@transaction.atomic +def submit_to_pontoon(instance): + revision, created = TranslatableRevision.from_instance(instance) + + if not created: + # Check if there is already a submission for this revision + if PontoonResourceSubmission.objects.filter(revision=revision).exists(): + return + + # Extract segments from revision and save them to translation memory + segments = extract_segments(instance) + insert_segments(revision, revision.locale.language_id, segments) + + # Recurse into any related objects + for segment in segments: + if not isinstance(segment, RelatedObjectValue): + continue + + submit_to_pontoon(segment.get_instance(revision.locale)) + + base_model = instance.get_translation_model() resource, created = PontoonResource.objects.get_or_create( object=revision.object, defaults={ "path": PontoonResource.get_unique_path_from_urlpath( - page_revision.page.url_path + f"{slugify(base_model._meta.verbose_name_plural)}/{instance.pk}" ), }, ) - submit_to_pontoon(resource, revision) + resource.current_revision = revision + resource.save(update_fields=["current_revision"]) + + # Create submission + resource.submissions.create(revision=revision) @transaction.atomic -def submit_to_pontoon(resource, revision): - # Extract segments from revision and save them to translation memory - insert_segments( - revision, revision.locale.language_id, extract_segments(revision.as_instance()) +def submit_page_to_pontoon(page_revision): + revision, created = TranslatableRevision.get_or_create_from_page_revision( + page_revision + ) + + if not created: + # Check if there is already a submission for this revision + if PontoonResourceSubmission.objects.filter(revision=revision).exists(): + return + + # Extract segments from revision and save them into translation memory + segments = extract_segments(page_revision.as_page_object()) + insert_segments(revision, revision.locale.language_id, segments) + + # Recurse into any related objects + for segment in segments: + if not isinstance(segment, RelatedObjectValue): + continue + + submit_to_pontoon(segment.get_instance(revision.locale)) + + resource, created = PontoonResource.objects.get_or_create( + object=revision.object, + defaults={ + "path": PontoonResource.get_unique_path_from_urlpath( + "pages" + page_revision.page.url_path + ), + }, ) resource.current_revision = revision diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py index 9c7dbfad..19fab1c0 100644 --- a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py +++ b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py @@ -6,7 +6,7 @@ from wagtail.core.models import Page from wagtail_localize.models import Language -from wagtail_localize.test.models import TestPage +from wagtail_localize.test.models import TestPage, TestSnippet from ..importer import Importer from ..models import PontoonResource, PontoonSyncLog @@ -238,6 +238,85 @@ def test_importer_doesnt_import_if_parent_not_translated(self): self.assertEqual(log_resource.resource, self.resource) self.assertEqual(log_resource.language, self.language) + def test_importer_doesnt_import_if_dependency_not_translated(self): + self.page.test_snippet = TestSnippet.objects.create(field="Test content") + self.page.save_revision().publish() + + snippet_resource = PontoonResource.objects.get( + object__translation_key=self.page.test_snippet.translation_key + ) + + with self.subTest(stage="New page"): + po_v1 = create_test_po([("The test translatable field", "")]).encode( + "utf-8" + ) + + po_v2 = create_test_po( + [("The test translatable field", "Le champ traduisible de test")] + ).encode("utf-8") + + importer = Importer(Language.objects.default(), logging.getLogger("dummy")) + importer.start_import("0" * 40) + importer.import_file( + self.resource.get_po_filename(language=self.language), po_v1, po_v2 + ) + + # Check translated page was not created + self.assertFalse( + self.page.get_translations() + .filter(locale__language=self.language) + .exists() + ) + + # Check log + log = PontoonSyncLog.objects.get() + self.assertEqual(log.action, PontoonSyncLog.ACTION_PULL) + self.assertEqual(log.commit_id, "0" * 40) + log_resource = log.resources.get() + self.assertEqual(log_resource.resource, self.resource) + self.assertEqual(log_resource.language, self.language) + + new_page_succeeded = True + + # subTest swallows errors, but we don't want to proceed if there was an error + # I know we're not exactly using them as intended + if not new_page_succeeded: + return + + with self.subTest(stage="Create snippet"): + po_v1 = create_test_po([("Test content", "")]).encode("utf-8") + + po_v2 = create_test_po([("Test content", "Tester le contenu")]).encode( + "utf-8" + ) + + importer = Importer(Language.objects.default(), logging.getLogger("dummy")) + importer.start_import("0" * 39 + "1") + importer.import_file( + snippet_resource.get_po_filename(language=self.language), po_v1, po_v2 + ) + + # Check translated snippet was created + translated_snippet = TestSnippet.objects.get(locale__language=self.language) + self.assertEqual( + translated_snippet.translation_key, + self.page.test_snippet.translation_key, + ) + self.assertFalse(translated_snippet.is_source_translation) + self.assertEqual(translated_snippet.field, "Tester le contenu") + + # Check the translated page was created and linked to the translated snippet + translated_page = TestPage.objects.get(locale__language=self.language) + self.assertEqual(translated_page.test_snippet, translated_snippet) + + # Check log + log = PontoonSyncLog.objects.exclude(id=log.id).get() + self.assertEqual(log.action, PontoonSyncLog.ACTION_PULL) + self.assertEqual(log.commit_id, "0" * 39 + "1") + log_resource = log.resources.get() + self.assertEqual(log_resource.resource, snippet_resource) + self.assertEqual(log_resource.language, self.language) + class TestImporterRichText(TestCase): def setUp(self): diff --git a/wagtail_localize/translation_memory/migrations/0011_relatedobjectlocation.py b/wagtail_localize/translation_memory/migrations/0011_relatedobjectlocation.py new file mode 100644 index 00000000..4762aa89 --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0011_relatedobjectlocation.py @@ -0,0 +1,46 @@ +# Generated by Django 2.2.7 on 2019-12-20 15:52 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_translation_memory", "0010_delete_old_location_models"), + ] + + operations = [ + migrations.CreateModel( + name="RelatedObjectLocation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("path", models.TextField()), + ("order", models.PositiveIntegerField()), + ( + "object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="references", + to="wagtail_localize_translation_memory.TranslatableObject", + ), + ), + ( + "revision", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wagtail_localize_translation_memory.TranslatableRevision", + ), + ), + ], + options={"abstract": False,}, + ), + ] diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index ce8f903b..23356461 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -14,7 +14,7 @@ ) from wagtail.core.models import Page -from wagtail_localize.segments import SegmentValue, TemplateValue +from wagtail_localize.segments import SegmentValue, TemplateValue, RelatedObjectValue from wagtail_localize.segments.ingest import ingest_segments @@ -100,7 +100,7 @@ def get_or_create_from_page_revision(cls, page_revision): ) @classmethod - def from_instance(cls, instance): + def from_instance(cls, instance, force=False): object, created = TranslatableObject.objects.get_or_create_from_instance( instance ) @@ -111,11 +111,21 @@ def from_instance(cls, instance): serializable_data = get_serializable_data_for_fields(instance) content_json = json.dumps(serializable_data, cls=DjangoJSONEncoder) - return cls.objects.create( - object=object, - locale=instance.locale, - content_json=content_json, - created_at=timezone.now(), + if not force: + # Check if the instance has changed at all since the previous revision + previous_revision = object.revisions.order_by("created_at").last() + if previous_revision: + if content_json == previous_revision.content_json: + return previous_revision, False + + return ( + cls.objects.create( + object=object, + locale=instance.locale, + content_json=content_json, + created_at=timezone.now(), + ), + True, ) def as_instance(self): @@ -171,6 +181,10 @@ def create_or_update_translation(self, locale, translation=None): revision=self ).select_related("template") + related_object_locations = RelatedObjectLocation.objects.filter( + revision=self + ).select_related("object") + segments = [] for location in segment_locations: @@ -191,6 +205,15 @@ def create_or_update_translation(self, locale, translation=None): ) segments.append(segment) + for location in related_object_locations: + segment = RelatedObjectValue( + location.path, + location.object.content_type, + location.object.translation_key, + order=location.order, + ) + segments.append(segment) + # Ingest all translated segments ingest_segments(original, translation, self.locale, locale, segments) @@ -387,3 +410,23 @@ def from_template_value(cls, revision, template_value): ) return template_loc + + +class RelatedObjectLocation(BaseLocation): + object = models.ForeignKey( + TranslatableObject, on_delete=models.CASCADE, related_name="references" + ) + + @classmethod + def from_related_object_value(cls, revision, related_object_value): + related_object_loc, created = cls.objects.get_or_create( + revision_id=pk(revision), + path=related_object_value.path, + order=related_object_value.order, + object=TranslatableObject.objects.get_or_create( + content_type=related_object_value.content_type, + translation_key=related_object_value.translation_key, + )[0], + ) + + return related_object_loc diff --git a/wagtail_localize/translation_memory/utils.py b/wagtail_localize/translation_memory/utils.py index 034c8c5f..e13fe396 100644 --- a/wagtail_localize/translation_memory/utils.py +++ b/wagtail_localize/translation_memory/utils.py @@ -10,13 +10,16 @@ When, ) -from wagtail_localize.segments import TemplateValue +from wagtail_localize.segments import TemplateValue, RelatedObjectValue +from wagtail_localize.segments.extract import extract_segments from .models import ( Segment, SegmentTranslation, SegmentLocation, TemplateLocation, + RelatedObjectLocation, + TranslatableRevision, ) @@ -65,5 +68,7 @@ def insert_segments(revision, language, segments): for segment in segments: if isinstance(segment, TemplateValue): TemplateLocation.from_template_value(revision, segment) + elif isinstance(segment, RelatedObjectValue): + RelatedObjectLocation.from_related_object_value(revision, segment) else: SegmentLocation.from_segment_value(revision, language, segment) From 74927196439bbbd594d9bc71aa55490054832889 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 1 Jan 2020 21:21:08 +0000 Subject: [PATCH 19/21] Update pofile translation engine to support related objects --- wagtail_localize/test/fixtures/test_user.json | 20 ++ wagtail_localize/test/settings.py | 1 + .../translation_engines/pofile/tests.py | 190 ++++++++++++++++++ .../translation_engines/pofile/views.py | 162 +++++++++------ wagtail_localize/translation_memory/models.py | 4 +- 5 files changed, 318 insertions(+), 59 deletions(-) create mode 100644 wagtail_localize/test/fixtures/test_user.json create mode 100644 wagtail_localize/translation_engines/pofile/tests.py diff --git a/wagtail_localize/test/fixtures/test_user.json b/wagtail_localize/test/fixtures/test_user.json new file mode 100644 index 00000000..bf8f98b1 --- /dev/null +++ b/wagtail_localize/test/fixtures/test_user.json @@ -0,0 +1,20 @@ +[ + { + "model": "auth.user", + "pk": 1, + "fields": { + "password": "md5$seasalt$1e9bf2bf5606aa5c39852cc30f0f6f22", + "last_login": "2017-04-23T19:58:19.460Z", + "is_superuser": true, + "username": "admin", + "first_name": "Admin", + "last_name": "User", + "email": "admin@example.com", + "is_staff": true, + "is_active": true, + "date_joined": "2017-02-17T07:37:53.700Z", + "groups": [], + "user_permissions": [] + } + } +] diff --git a/wagtail_localize/test/settings.py b/wagtail_localize/test/settings.py index e5139cf6..265ab205 100644 --- a/wagtail_localize/test/settings.py +++ b/wagtail_localize/test/settings.py @@ -38,6 +38,7 @@ "wagtail_localize.admin.workflow", "wagtail_localize.translation_memory", "wagtail_localize.translation_engines.pontoon", + "wagtail_localize.translation_engines.pofile", "wagtail.contrib.search_promotions", "wagtail.contrib.forms", "wagtail.contrib.redirects", diff --git a/wagtail_localize/translation_engines/pofile/tests.py b/wagtail_localize/translation_engines/pofile/tests.py new file mode 100644 index 00000000..0bc410eb --- /dev/null +++ b/wagtail_localize/translation_engines/pofile/tests.py @@ -0,0 +1,190 @@ +import polib +from django.contrib.auth.models import User +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import TestCase +from django.urls import reverse +from django.utils import timezone +from wagtail.core.models import Page + +from wagtail_localize.admin.workflow.models import ( + TranslationRequest, + TranslationRequestPage, +) +from wagtail_localize.models import Language, Locale +from wagtail_localize.test.models import TestPage, TestSnippet + + +def create_test_page(**kwargs): + parent = kwargs.pop("parent", None) or Page.objects.get(id=1) + page = parent.add_child(instance=TestPage(**kwargs)) + revision = page.save_revision() + revision.publish() + return page + + +class BasePoFileTestCase(TestCase): + fixtures = ["test_user.json"] + + def setUp(self): + self.client.login(username="admin", password="password") + + Language.objects.create(code="fr") + self.user = User.objects.get(username="admin") + self.translation_request = TranslationRequest.objects.create( + source_locale=Locale.objects.default(), + target_locale=Locale.objects.get(language__code="fr"), + target_root=Page.objects.get(id=1), + created_at=timezone.now(), + created_by=self.user, + ) + + def add_page_to_request(self, page, **kwargs): + return TranslationRequestPage.objects.create( + request=self.translation_request, + source_revision=page.get_latest_revision(), + **kwargs, + ) + + +class TestDownload(BasePoFileTestCase): + def test_download(self): + page = create_test_page( + title="Test page", + slug="test-page", + test_charfield="Some translatable content", + ) + self.add_page_to_request(page) + + response = self.client.get( + reverse( + "wagtail_localize_pofile:download", args=[self.translation_request.id] + ) + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response["Content-Type"], "text/x-gettext-translation") + + self.assertIn( + b'#: pages/test-page/:test_charfield\nmsgid "Some translatable content"\nmsgstr ""\n', + response.content, + ) + + def test_download_with_nested_snippet(self): + snippet = TestSnippet.objects.create(field="Some test snippet content") + page = create_test_page( + title="Test page", slug="test-page", test_snippet=snippet + ) + self.add_page_to_request(page) + + response = self.client.get( + reverse( + "wagtail_localize_pofile:download", args=[self.translation_request.id] + ) + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response["Content-Type"], "text/x-gettext-translation") + + self.assertIn( + f'#: test-snippets/{snippet.id}:field\nmsgid "Some test snippet content"\nmsgstr ""\n', + response.content.decode(), + ) + + +class TestUpload(BasePoFileTestCase): + def test_upload(self): + page = create_test_page( + title="Test page", + slug="test-page", + test_charfield="Some translatable content", + ) + request_page = self.add_page_to_request(page) + + po = polib.POFile() + po.metadata = { + "POT-Creation-Date": str(timezone.now()), + "MIME-Version": "1.0", + "Content-Type": "text/plain; charset=utf-8", + } + + po.append( + polib.POEntry( + msgid="Some translatable content", + msgstr="Du contenu traduisible", + occurrences="", + ) + ) + + response = self.client.post( + reverse( + "wagtail_localize_pofile:upload", args=[self.translation_request.id] + ), + {"file": SimpleUploadedFile("test.po", str(po).encode("utf-8")),}, + ) + self.assertRedirects( + response, + reverse( + "wagtail_localize_workflow_management:detail", + args=[self.translation_request.id], + ), + ) + + request_page.refresh_from_db() + self.assertTrue(request_page.is_completed) + + completed_revision = request_page.completed_revision + completed_page = completed_revision.as_page_object() + self.assertEqual(completed_page.locale, self.translation_request.target_locale) + self.assertEqual(completed_page.translation_key, page.translation_key) + self.assertEqual(completed_page.test_charfield, "Du contenu traduisible") + + def test_upload_with_nested_snippet(self): + snippet = TestSnippet.objects.create(field="Some test snippet content") + page = create_test_page( + title="Test page", slug="test-page", test_snippet=snippet + ) + request_page = self.add_page_to_request(page) + + po = polib.POFile() + po.metadata = { + "POT-Creation-Date": str(timezone.now()), + "MIME-Version": "1.0", + "Content-Type": "text/plain; charset=utf-8", + } + + po.append( + polib.POEntry( + msgid="Some test snippet content", + msgstr="Du contenu d'extrait de test", + occurrences="", + ) + ) + + response = self.client.post( + reverse( + "wagtail_localize_pofile:upload", args=[self.translation_request.id] + ), + {"file": SimpleUploadedFile("test.po", str(po).encode("utf-8")),}, + ) + self.assertRedirects( + response, + reverse( + "wagtail_localize_workflow_management:detail", + args=[self.translation_request.id], + ), + ) + + request_page.refresh_from_db() + self.assertTrue(request_page.is_completed) + + completed_revision = request_page.completed_revision + completed_page = completed_revision.as_page_object() + self.assertEqual( + completed_page.test_snippet.locale, self.translation_request.target_locale + ) + self.assertEqual( + completed_page.test_snippet.translation_key, snippet.translation_key + ) + self.assertEqual( + completed_page.test_snippet.field, "Du contenu d'extrait de test" + ) diff --git a/wagtail_localize/translation_engines/pofile/views.py b/wagtail_localize/translation_engines/pofile/views.py index 93a5191e..91f09922 100644 --- a/wagtail_localize/translation_engines/pofile/views.py +++ b/wagtail_localize/translation_engines/pofile/views.py @@ -9,13 +9,41 @@ from django.shortcuts import get_object_or_404, redirect from django.utils import timezone from django.utils.text import slugify +from wagtail.core.models import Page from wagtail_localize.admin.workflow.models import TranslationRequest -from wagtail_localize.segments import SegmentValue, TemplateValue +from wagtail_localize.segments import SegmentValue, TemplateValue, RelatedObjectValue from wagtail_localize.segments.extract import extract_segments from wagtail_localize.segments.ingest import ingest_segments +class MessageExtractor: + def __init__(self, locale): + self.locale = locale + self.seen_objects = set() + self.messages = defaultdict(list) + + def get_path(self, instance): + if isinstance(instance, Page): + return "pages" + instance.url_path + else: + base_model = instance.get_translation_model() + return f"{slugify(base_model._meta.verbose_name_plural)}/{instance.pk}" + + def extract_messages(self, instance): + if instance.translation_key in self.seen_objects: + return + self.seen_objects.add(instance.translation_key) + + for segment in extract_segments(instance): + if isinstance(segment, SegmentValue): + self.messages[segment.text].append( + (self.get_path(instance), segment.path) + ) + elif isinstance(segment, RelatedObjectValue): + self.extract_messages(segment.get_instance(self.locale)) + + @require_GET def download(request, translation_request_id): translation_request = get_object_or_404( @@ -23,19 +51,10 @@ def download(request, translation_request_id): ) # Extract messages from pages - messages = defaultdict(list) + message_extractor = MessageExtractor(translation_request.source_locale) for page in translation_request.pages.all(): instance = page.source_revision.as_page_object() - - segments = extract_segments(instance) - - # Filter out templates - text_segments = [ - segment for segment in segments if isinstance(segment, SegmentValue) - ] - - for segment in text_segments: - messages[segment.text].append((instance.url_path, segment.path)) + message_extractor.extract_messages(instance) # Build a PO file po = polib.POFile() @@ -45,7 +64,7 @@ def download(request, translation_request_id): "Content-Type": "text/plain; charset=utf-8", } - for text, occurances in messages.items(): + for text, occurances in message_extractor.messages.items(): po.append( polib.POEntry( msgid=text, @@ -70,6 +89,70 @@ def __init__(self, instance, num_missing): super().__init__() +class MessageIngestor: + def __init__(self, source_locale, target_locale, translations): + self.source_locale = source_locale + self.target_locale = target_locale + self.translations = translations + self.seen_objects = set() + + def ingest_messages(self, instance): + if instance.translation_key in self.seen_objects: + return + self.seen_objects.add(instance.translation_key) + + segments = extract_segments(instance) + + # Ingest segments for dependencies first + for segment in segments: + if isinstance(segment, RelatedObjectValue): + self.ingest_messages(segment.get_instance(self.source_locale)) + + text_segments = [ + segment for segment in segments if isinstance(segment, SegmentValue) + ] + + # Initialise translated segments by copying templates and related objects + translated_segments = [ + segment + for segment in segments + if isinstance(segment, (TemplateValue, RelatedObjectValue)) + ] + + missing_segments = 0 + for segment in text_segments: + if segment.text in self.translations: + translated_segments.append( + SegmentValue(segment.path, self.translations[segment.text]) + ) + else: + missing_segments += 1 + + if missing_segments: + raise MissingSegmentsException(instance, missing_segments) + + try: + translation = instance.get_translation(self.target_locale) + except instance.__class__.DoesNotExist: + translation = instance.copy_for_translation(self.target_locale) + + ingest_segments( + instance, + translation, + self.source_locale, + self.target_locale, + translated_segments, + ) + + if isinstance(translation, Page): + translation.slug = slugify(translation.slug) + revision = translation.save_revision() + else: + translation.save() + + return translation + + @require_POST def upload(request, translation_request_id): translation_request = get_object_or_404( @@ -85,56 +168,19 @@ def upload(request, translation_request_id): try: with transaction.atomic(): + message_ingestor = MessageIngestor( + translation_request.source_locale, + translation_request.target_locale, + translations, + ) + for page in translation_request.pages.filter(is_completed=False): instance = page.source_revision.as_page_object() - - segments = extract_segments(instance) - - text_segments = [ - segment for segment in segments if isinstance(segment, SegmentValue) - ] - template_segments = [ - segment - for segment in segments - if isinstance(segment, TemplateValue) - ] - - translated_segments = template_segments.copy() - - missing_segments = 0 - for segment in text_segments: - if segment.text in translations: - translated_segments.append( - SegmentValue(segment.path, translations[segment.text]) - ) - else: - missing_segments += 1 - - if missing_segments: - raise MissingSegmentsException(instance, missing_segments) - - try: - translation = instance.get_translation( - translation_request.target_locale - ) - except instance.__class__.DoesNotExist: - translation = instance.copy_for_translation( - translation_request.target_locale - ) - - ingest_segments( - instance, - translation, - translation_request.source_locale, - translation_request.target_locale, - translated_segments, - ) - translation.slug = slugify(translation.slug) - revision = translation.save_revision() + translation = message_ingestor.ingest_messages(instance) # Update translation request page.is_completed = True - page.completed_revision = revision + page.completed_revision = translation.get_latest_revision() page.save(update_fields=["is_completed", "completed_revision"]) except MissingSegmentsException as e: diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index 23356461..e9ef0fa5 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -188,7 +188,9 @@ def create_or_update_translation(self, locale, translation=None): segments = [] for location in segment_locations: - segment = SegmentValue.from_html(location.path, location.translation).with_order(location.order) + segment = SegmentValue.from_html( + location.path, location.translation + ).with_order(location.order) if location.html_attrs: segment.replace_html_attrs(json.loads(location.html_attrs)) From f4c38ce1cb9bb76ec793ae293b0158607d352056 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 2 Jan 2020 09:47:09 +0000 Subject: [PATCH 20/21] Update google translate translation engine to support related objects --- setup.py | 3 + tox.ini | 2 +- wagtail_localize/test/settings.py | 1 + .../google_translate/tests.py | 190 ++++++++++++++++++ .../google_translate/views.py | 100 +++++---- 5 files changed, 244 insertions(+), 52 deletions(-) create mode 100644 wagtail_localize/translation_engines/google_translate/tests.py diff --git a/setup.py b/setup.py index 13bfac4b..f324015d 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,9 @@ 'gitpython>=3.0,<4.0', 'toml>=0.10,<0.11', ], + 'google_translate': [ + 'googletrans>=2.4,<3.0', + ], }, zip_safe=False, ) diff --git a/tox.ini b/tox.ini index e02c335c..b02f6e1b 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ envlist = py{37}-dj{22,master}-wa{26,27}-{postgres} ignore = D100,D101,D102,D103,D105,D200,D202,D204,D205,D209,D400,D401,E303,E501,W503,N805,N806 [testenv] -install_command = pip install -e ".[testing,pontoon]" -U {opts} {packages} +install_command = pip install -e ".[testing,pontoon,google_translate]" -U {opts} {packages} commands = coverage run testmanage.py test basepython = diff --git a/wagtail_localize/test/settings.py b/wagtail_localize/test/settings.py index 265ab205..a169edf1 100644 --- a/wagtail_localize/test/settings.py +++ b/wagtail_localize/test/settings.py @@ -37,6 +37,7 @@ "wagtail_localize.admin.regions", "wagtail_localize.admin.workflow", "wagtail_localize.translation_memory", + "wagtail_localize.translation_engines.google_translate", "wagtail_localize.translation_engines.pontoon", "wagtail_localize.translation_engines.pofile", "wagtail.contrib.search_promotions", diff --git a/wagtail_localize/translation_engines/google_translate/tests.py b/wagtail_localize/translation_engines/google_translate/tests.py new file mode 100644 index 00000000..dbc74cf5 --- /dev/null +++ b/wagtail_localize/translation_engines/google_translate/tests.py @@ -0,0 +1,190 @@ +from unittest import mock + +from django.contrib.auth.models import User +from django.test import TestCase +from django.urls import reverse +from django.utils import timezone +from googletrans.models import Translated +from wagtail.core.models import Page + +from wagtail_localize.admin.workflow.models import ( + TranslationRequest, + TranslationRequestPage, +) +from wagtail_localize.models import Language, Locale +from wagtail_localize.test.models import TestPage, TestSnippet + + +def create_test_page(**kwargs): + parent = kwargs.pop("parent", None) or Page.objects.get(id=1) + page = parent.add_child(instance=TestPage(**kwargs)) + revision = page.save_revision() + revision.publish() + return page + + +@mock.patch("googletrans.Translator") +class TestTranslate(TestCase): + fixtures = ["test_user.json"] + + def setUp(self): + self.client.login(username="admin", password="password") + + Language.objects.create(code="fr") + self.user = User.objects.get(username="admin") + self.translation_request = TranslationRequest.objects.create( + source_locale=Locale.objects.default(), + target_locale=Locale.objects.get(language__code="fr"), + target_root=Page.objects.get(id=1), + created_at=timezone.now(), + created_by=self.user, + ) + + def add_page_to_request(self, page, **kwargs): + return TranslationRequestPage.objects.create( + request=self.translation_request, + source_revision=page.get_latest_revision(), + **kwargs, + ) + + def test_translate(self, Translator): + page = create_test_page( + title="Test page", + slug="test-page", + test_charfield="Some translatable content", + ) + request_page = self.add_page_to_request(page) + + # Mock response from Google Translate + Translator().translate.return_value = [ + Translated( + "en", + "fr", + "Some translatable content", + "Certains contenus traduisibles", + "Certains contenus traduisibles", + ) + ] + + response = self.client.post( + reverse( + "wagtail_localize_google_translate:translate", + args=[self.translation_request.id], + ), + {"publish": "on"}, + ) + + self.assertRedirects( + response, + reverse( + "wagtail_localize_workflow_management:detail", + args=[self.translation_request.id], + ), + ) + + Translator().translate.assert_called_with( + ["Some translatable content"], dest="fr", src="en" + ) + + request_page.refresh_from_db() + self.assertTrue(request_page.is_completed) + + translated_page = page.get_translation(Locale.objects.get(language__code="fr")) + self.assertTrue(translated_page.live) + self.assertEqual( + translated_page.test_charfield, "Certains contenus traduisibles" + ) + + def test_translate_without_publishing(self, Translator): + page = create_test_page( + title="Test page", + slug="test-page", + test_charfield="Some translatable content", + ) + request_page = self.add_page_to_request(page) + + # Mock response from Google Translate + Translator().translate.return_value = [ + Translated( + "en", + "fr", + "Some translatable content", + "Certains contenus traduisibles", + "Certains contenus traduisibles", + ) + ] + + response = self.client.post( + reverse( + "wagtail_localize_google_translate:translate", + args=[self.translation_request.id], + ) + ) + + self.assertRedirects( + response, + reverse( + "wagtail_localize_workflow_management:detail", + args=[self.translation_request.id], + ), + ) + + Translator().translate.assert_called_with( + ["Some translatable content"], dest="fr", src="en" + ) + + request_page.refresh_from_db() + self.assertTrue(request_page.is_completed) + + translated_page = page.get_translation(Locale.objects.get(language__code="fr")) + self.assertFalse(translated_page.live) + self.assertEqual( + translated_page.get_latest_revision_as_page().test_charfield, + "Certains contenus traduisibles", + ) + + def test_translate_with_nested_snippet(self, Translator): + snippet = TestSnippet.objects.create(field="Some test snippet content") + page = create_test_page( + title="Test page", slug="test-page", test_snippet=snippet + ) + request_page = self.add_page_to_request(page) + + # Mock response from Google Translate + Translator().translate.return_value = [ + Translated( + "en", + "fr", + "Some test snippet content", + "Du contenu d'extrait de test", + "Du contenu d'extrait de test", + ) + ] + + response = self.client.post( + reverse( + "wagtail_localize_google_translate:translate", + args=[self.translation_request.id], + ), + {"publish": "on"}, + ) + + self.assertRedirects( + response, + reverse( + "wagtail_localize_workflow_management:detail", + args=[self.translation_request.id], + ), + ) + + Translator().translate.assert_called_with( + ["Some test snippet content"], dest="fr", src="en" + ) + + request_page.refresh_from_db() + self.assertTrue(request_page.is_completed) + + translated_page = page.get_translation(Locale.objects.get(language__code="fr")) + self.assertEqual( + translated_page.test_snippet.field, "Du contenu d'extrait de test" + ) diff --git a/wagtail_localize/translation_engines/google_translate/views.py b/wagtail_localize/translation_engines/google_translate/views.py index 759ab10d..4d3fd47d 100644 --- a/wagtail_localize/translation_engines/google_translate/views.py +++ b/wagtail_localize/translation_engines/google_translate/views.py @@ -11,8 +11,14 @@ from wagtail_localize.segments.extract import extract_segments from wagtail_localize.segments.ingest import ingest_segments +from wagtail_localize.translation_engines.pofile.views import ( + MessageExtractor, + MessageIngestor, + MissingSegmentsException, +) + # TODO: Switch to official Google API client -from googletrans import Translator +import googletrans def language_code(code): @@ -30,69 +36,61 @@ def translate(request, translation_request_id): translation_request = get_object_or_404( TranslationRequest, id=translation_request_id ) - translator = Translator() - - publish = request.POST.get("publish", "") == "on" + message_extractor = MessageExtractor(translation_request.source_locale) for page in translation_request.pages.filter(is_completed=False): instance = page.source_revision.as_page_object() + message_extractor.extract_messages(instance) - segments = extract_segments(instance) - - text_segments = [ - segment for segment in segments if isinstance(segment, SegmentValue) - ] - template_segments = [ - segment for segment in segments if isinstance(segment, TemplateValue) - ] + translator = googletrans.Translator() + google_translations = translator.translate( + list(message_extractor.messages.keys()), + src=language_code(translation_request.source_locale.language.code), + dest=language_code(translation_request.target_locale.language.code), + ) - # Group segments by source text so we only submit them once - text_segments_grouped = defaultdict(list) - for segment in text_segments: - text_segments_grouped[segment.text].append(segment.path) + translations = { + translation.origin: translation.text for translation in google_translations + } - translations = translator.translate( - list(text_segments_grouped.keys()), - src=language_code(translation_request.source_locale.language.code), - dest=language_code(translation_request.target_locale.language.code), - ) - - translated_segments = template_segments.copy() - for translation in translations: - translated_segments.extend( - [ - SegmentValue(path, translation.text) - for path in text_segments_grouped[translation.origin] - ] - ) + publish = request.POST.get("publish", "") == "on" + try: with transaction.atomic(): - try: - translation = instance.get_translation( - translation_request.target_locale - ) - except instance.__class__.DoesNotExist: - translation = instance.copy_for_translation( - translation_request.target_locale - ) - - ingest_segments( - instance, - translation, + message_ingestor = MessageIngestor( translation_request.source_locale, translation_request.target_locale, - translated_segments, + translations, ) - translation.slug = slugify(translation.slug) - revision = translation.save_revision() - if publish: - revision.publish() + for page in translation_request.pages.filter(is_completed=False): + instance = page.source_revision.as_page_object() + translation = message_ingestor.ingest_messages(instance) + revision = translation.get_latest_revision() + + if publish: + revision.publish() + + # Update translation request + page.is_completed = True + page.completed_revision = revision + page.save(update_fields=["is_completed", "completed_revision"]) + + except MissingSegmentsException as e: + # TODO: Plural + messages.error( + request, + "Unable to translate %s. %s missing segments." + % (e.instance.get_admin_display_title(), e.num_missing), + ) - # Update translation request - page.is_completed = True - page.completed_revision = revision - page.save(update_fields=["is_completed", "completed_revision"]) + else: + # TODO: Plural + messages.success( + request, + "%d pages successfully translated with PO file" + % translation_request.pages.count(), + ) # TODO: Plural messages.success( From f3163c464edcc110be7fc4b88dab8ec7aced3d82 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Thu, 2 Jan 2020 12:53:19 +0000 Subject: [PATCH 21/21] Implement segment context --- .../translation_engines/pontoon/importer.py | 28 +++- .../translation_engines/pontoon/models.py | 27 ++-- .../translation_engines/pontoon/pofile.py | 43 ++++-- .../pontoon/tests/test_importer.py | 103 +++++++++++--- .../pontoon/tests/test_pofile_generation.py | 13 +- .../0012_segmenttranslationcontext.py | 88 ++++++++++++ ...0013_populate_segmenttranslationcontext.py | 76 +++++++++++ .../migrations/0014_remove_location_paths.py | 44 ++++++ wagtail_localize/translation_memory/models.py | 126 ++++++++++++------ wagtail_localize/translation_memory/utils.py | 10 +- 10 files changed, 467 insertions(+), 91 deletions(-) create mode 100644 wagtail_localize/translation_memory/migrations/0012_segmenttranslationcontext.py create mode 100644 wagtail_localize/translation_memory/migrations/0013_populate_segmenttranslationcontext.py create mode 100644 wagtail_localize/translation_memory/migrations/0014_remove_location_paths.py diff --git a/wagtail_localize/translation_engines/pontoon/importer.py b/wagtail_localize/translation_engines/pontoon/importer.py index 9ad06f0b..1aceb2dd 100644 --- a/wagtail_localize/translation_engines/pontoon/importer.py +++ b/wagtail_localize/translation_engines/pontoon/importer.py @@ -18,6 +18,7 @@ Segment, SegmentLocation, TemplateLocation, + SegmentTranslationContext, ) from .models import ( @@ -34,14 +35,39 @@ def __init__(self, source_language, logger): self.logger = logger self.log = None + def changed_entries(self, old_po, new_po): + """ + Iterator of all entries that exists in new_po but not + old_po. + + This retrieves any strings that have been added or changed. + """ + old_entry_keys = set(str(entry) for entry in old_po) + new_entry_keys = set(str(entry) for entry in new_po) + new_entries = { + str(entry): entry for entry in new_po + } + + for changed_key in new_entry_keys - old_entry_keys: + yield new_entries[changed_key] + def import_resource(self, resource, language, old_po, new_po): - for changed_entry in set(new_po) - set(old_po): + for changed_entry in self.changed_entries(old_po, new_po): + # Don't import black strings + if not changed_entry.msgstr: + continue + try: segment = Segment.objects.get( language=self.source_language, text=changed_entry.msgid ) translation, created = segment.translations.get_or_create( language=language, + context=SegmentTranslationContext.get_from_string( + changed_entry.msgctxt + ) + if changed_entry.msgctxt + else None, defaults={ "text": changed_entry.msgstr, "updated_at": timezone.now(), diff --git a/wagtail_localize/translation_engines/pontoon/models.py b/wagtail_localize/translation_engines/pontoon/models.py index 5ebea840..97c6ec8a 100644 --- a/wagtail_localize/translation_engines/pontoon/models.py +++ b/wagtail_localize/translation_engines/pontoon/models.py @@ -17,6 +17,7 @@ TranslatableObject, TranslatableRevision, RelatedObjectLocation, + SegmentTranslation, ) from wagtail_localize.translation_memory.utils import ( insert_segments, @@ -142,26 +143,22 @@ def get_segments(self): """ Gets all segments that are in the latest submission to Pontoon. """ - return Segment.objects.filter(locations__revision_id=self.current_revision_id) + return SegmentLocation.objects.filter(revision_id=self.current_revision_id) - def get_all_segments(self, annotate_obsolete=False): + def get_obsolete_translations(self, language): """ - Gets all segments that have ever been submitted to Pontoon. + Gets all past translations for this resource that are not used in + the latest submission. """ - segments = Segment.objects.filter( - locations__revision__pontoon_submission__resource_id=self.pk - ) - - if annotate_obsolete: - segments = segments.annotate( - is_obsolete=~Exists( - SegmentLocation.objects.filter( - segment=OuterRef("pk"), revision_id=self.current_revision_id, - ) + return SegmentTranslation.objects.annotate( + is_in_latest_submission=Exists( + SegmentLocation.objects.filter( + revision_id=self.current_revision_id, + segment_id=OuterRef("translation_of_id"), + context_id=OuterRef("context_id"), ) ) - - return segments.distinct() + ).filter(language=language, is_in_latest_submission=False,) def find_translatable_submission(self, language): """ diff --git a/wagtail_localize/translation_engines/pontoon/pofile.py b/wagtail_localize/translation_engines/pontoon/pofile.py index 9ab49a6e..fd0e15ea 100644 --- a/wagtail_localize/translation_engines/pontoon/pofile.py +++ b/wagtail_localize/translation_engines/pontoon/pofile.py @@ -13,8 +13,16 @@ def generate_source_pofile(resource): "Content-Type": "text/html; charset=utf-8", } - for segment in resource.get_segments().iterator(): - po.append(polib.POEntry(msgid=segment.text, msgstr="")) + for segment in ( + resource.get_segments().select_related("segment", "context").iterator() + ): + po.append( + polib.POEntry( + msgid=segment.segment.text, + msgstr="", + msgctxt=segment.context.as_string(), + ) + ) return str(po) @@ -32,19 +40,36 @@ def generate_language_pofile(resource, language): } # Live segments - for segment in resource.get_segments().annotate_translation(language).iterator(): - po.append(polib.POEntry(msgid=segment.text, msgstr=segment.translation or "")) - - # Add any obsolete segments that have translations for future referene for segment in ( - resource.get_all_segments(annotate_obsolete=True) + resource.get_segments() + .select_related("segment", "context") .annotate_translation(language) - .filter(is_obsolete=True, translation__isnull=False) .iterator() ): po.append( polib.POEntry( - msgid=segment.text, msgstr=segment.translation or "", obsolete=True + msgid=segment.segment.text, + msgstr=segment.translation or "", + msgctxt=segment.context.as_string(), + ) + ) + + # Add any obsolete segments that have translations for future reference + # We find this by looking for obsolete contexts and annotate the latest + # translation for each one. Contexts that were never translated are + # excluded + for translation in ( + resource.get_obsolete_translations(language) + .select_related("translation_of", "context") + .filter(context__isnull=False) + .iterator() + ): + po.append( + polib.POEntry( + msgid=translation.translation_of.text, + msgstr=translation.text or "", + msgctxt=translation.context.as_string(), + obsolete=True, ) ) diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py index 19fab1c0..7d14a80e 100644 --- a/wagtail_localize/translation_engines/pontoon/tests/test_importer.py +++ b/wagtail_localize/translation_engines/pontoon/tests/test_importer.py @@ -29,7 +29,7 @@ def create_test_po(entries): } for entry in entries: - po.append(polib.POEntry(msgid=entry[0], msgstr=entry[1])) + po.append(polib.POEntry(msgctxt=entry[0], msgid=entry[1], msgstr=entry[2])) return str(po) @@ -52,12 +52,24 @@ def test_importer(self): new_page_succeeded = False with self.subTest(stage="New page"): - po_v1 = create_test_po([("The test translatable field", "")]).encode( - "utf-8" - ) + po_v1 = create_test_po( + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "", + ) + ] + ).encode("utf-8") po_v2 = create_test_po( - [("The test translatable field", "Le champ traduisible de test")] + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "Le champ traduisible de test", + ) + ] ).encode("utf-8") importer = Importer(Language.objects.default(), logging.getLogger("dummy")) @@ -98,6 +110,7 @@ def test_importer(self): po_v3 = create_test_po( [ ( + f"{self.page.translation_key}:test_charfield", "The test translatable field", "Le champ testable à traduire avec un contenu mis à jour", ) @@ -146,12 +159,19 @@ def test_importer_doesnt_import_if_parent_not_translated(self): with self.subTest(stage="Create child page"): # Translate po_v1 = create_test_po( - [("The test child's translatable field", "")] + [ + ( + f"{child_page.translation_key}:test_charfield", + "The test child's translatable field", + "", + ) + ] ).encode("utf-8") po_v2 = create_test_po( [ ( + f"{child_page.translation_key}:test_charfield", "The test child's translatable field", "Le champ traduisible de test", ) @@ -187,12 +207,24 @@ def test_importer_doesnt_import_if_parent_not_translated(self): return with self.subTest(stage="Create parent page"): - po_v1 = create_test_po([("The test translatable field", "")]).encode( - "utf-8" - ) + po_v1 = create_test_po( + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "", + ) + ] + ).encode("utf-8") po_v2 = create_test_po( - [("The test translatable field", "Le champ traduisible de test")] + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "Le champ traduisible de test", + ) + ] ).encode("utf-8") importer = Importer(Language.objects.default(), logging.getLogger("dummy")) @@ -247,12 +279,24 @@ def test_importer_doesnt_import_if_dependency_not_translated(self): ) with self.subTest(stage="New page"): - po_v1 = create_test_po([("The test translatable field", "")]).encode( - "utf-8" - ) + po_v1 = create_test_po( + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "", + ) + ] + ).encode("utf-8") po_v2 = create_test_po( - [("The test translatable field", "Le champ traduisible de test")] + [ + ( + f"{self.page.translation_key}:test_charfield", + "The test translatable field", + "Le champ traduisible de test", + ) + ] ).encode("utf-8") importer = Importer(Language.objects.default(), logging.getLogger("dummy")) @@ -284,11 +328,25 @@ def test_importer_doesnt_import_if_dependency_not_translated(self): return with self.subTest(stage="Create snippet"): - po_v1 = create_test_po([("Test content", "")]).encode("utf-8") + po_v1 = create_test_po( + [ + ( + f"{self.page.test_snippet.translation_key}:field", + "Test content", + "", + ) + ] + ).encode("utf-8") - po_v2 = create_test_po([("Test content", "Tester le contenu")]).encode( - "utf-8" - ) + po_v2 = create_test_po( + [ + ( + f"{self.page.test_snippet.translation_key}:field", + "Test content", + "Tester le contenu", + ) + ] + ).encode("utf-8") importer = Importer(Language.objects.default(), logging.getLogger("dummy")) importer.start_import("0" * 39 + "1") @@ -333,12 +391,19 @@ def setUp(self): def test_importer_rich_text(self): po_v1 = create_test_po( - [('The test translatable field.', "")] + [ + ( + f"{self.page.translation_key}:test_richtextfield", + 'The test translatable field.', + "", + ) + ] ).encode("utf-8") po_v2 = create_test_po( [ ( + f"{self.page.translation_key}:test_richtextfield", 'The test translatable field.', 'Le champ traduisible de test.', ) diff --git a/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py b/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py index 50e0ab18..a641cab1 100644 --- a/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py +++ b/wagtail_localize/translation_engines/pontoon/tests/test_pofile_generation.py @@ -9,6 +9,7 @@ from wagtail_localize.test.models import TestPage from wagtail_localize.translation_memory.models import ( Segment, + SegmentTranslationContext, SegmentTranslation, SegmentLocation, ) @@ -88,9 +89,13 @@ def test_generate_language_pofile(self): def test_generate_language_pofile_with_existing_translation(self): segment = Segment.objects.get(text="The test translatable field") + context = SegmentTranslationContext.objects.get( + object_id=self.page.translation_key, path="test_charfield" + ) SegmentTranslation.objects.create( translation_of=segment, language=self.language, + context=context, text="Le champ traduisible de test", ) @@ -108,8 +113,14 @@ def test_generate_language_pofile_with_existing_obsolete_translation(self): segment.text_id = Segment.get_text_id(segment.text) segment.save() + context = SegmentTranslationContext.objects.get( + object_id=self.page.translation_key, path="test_charfield" + ) SegmentTranslation.objects.create( - translation_of=segment, language=self.language, text="Du texte obsolète" + translation_of=segment, + context=context, + language=self.language, + text="Du texte obsolète", ) # Create a new revision. This will create a new segment like how the current segment was before I changed it diff --git a/wagtail_localize/translation_memory/migrations/0012_segmenttranslationcontext.py b/wagtail_localize/translation_memory/migrations/0012_segmenttranslationcontext.py new file mode 100644 index 00000000..55112aa5 --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0012_segmenttranslationcontext.py @@ -0,0 +1,88 @@ +# Generated by Django 2.2.9 on 2019-12-31 15:30 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize", "0002_initial_data"), + ("wagtail_localize_translation_memory", "0011_relatedobjectlocation"), + ] + + operations = [ + migrations.CreateModel( + name="SegmentTranslationContext", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("path_id", models.UUIDField()), + ("path", models.TextField()), + ( + "object", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="wagtail_localize_translation_memory.TranslatableObject", + ), + ), + ], + options={"unique_together": {("object", "path_id")},}, + ), + migrations.AddField( + model_name="relatedobjectlocation", + name="context", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AddField( + model_name="segmentlocation", + name="context", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AddField( + model_name="segmenttranslation", + name="context", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="translations", + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AddField( + model_name="templatelocation", + name="context", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AlterUniqueTogether( + name="segmenttranslation", + unique_together={("language", "translation_of", "context")}, + ), + ] diff --git a/wagtail_localize/translation_memory/migrations/0013_populate_segmenttranslationcontext.py b/wagtail_localize/translation_memory/migrations/0013_populate_segmenttranslationcontext.py new file mode 100644 index 00000000..6cf46b25 --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0013_populate_segmenttranslationcontext.py @@ -0,0 +1,76 @@ +# Generated by Django 2.2.9 on 2019-12-31 16:30 + +import uuid +from django.db import migrations + + +def get_path_id(path): + return uuid.uuid5(uuid.UUID("fcab004a-2b50-11ea-978f-2e728ce88125"), path) + + +def populate_segmenttranslationcontext(apps, schema_editor): + SegmentLocation = apps.get_model( + "wagtail_localize_translation_memory.SegmentLocation" + ) + TemplateLocation = apps.get_model( + "wagtail_localize_translation_memory.TemplateLocation" + ) + RelatedObjectLocation = apps.get_model( + "wagtail_localize_translation_memory.RelatedObjectLocation" + ) + SegmentTranslation = apps.get_model( + "wagtail_localize_translation_memory.SegmentTranslation" + ) + SegmentTranslationContext = apps.get_model( + "wagtail_localize_translation_memory.SegmentTranslationContext" + ) + + def update_location_model(model): + for location in model.objects.select_related("revision"): + context, created = SegmentTranslationContext.objects.get_or_create( + object_id=location.revision.object_id, + path_id=get_path_id(location.path), + defaults={"path": location.path,}, + ) + + location.context = context + location.save(update_fields=["context"]) + + update_location_model(SegmentLocation) + update_location_model(TemplateLocation) + update_location_model(RelatedObjectLocation) + + for translation in SegmentTranslation.objects.all(): + contexts = ( + SegmentLocation.objects.filter(segment_id=translation.translation_of_id) + .order_by("context_id") + .values_list("context_id", flat=True) + .distinct() + ) + + if len(contexts) > 0: + translation.context_id = contexts[0] + translation.save(update_fields=["context_id"]) + + # Create duplicate SegmentTranslations for remaining contexts + if len(contexts) > 1: + for context in contexts[1:]: + SegmentTranslation.objects.create( + translation_of_id=translation.translation_of_id, + language_id=translation.language_id, + context_id=context, + text=translation.text, + created_at=translation.created_at, + updated_at=translation.updated_at, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("wagtail_localize_translation_memory", "0012_segmenttranslationcontext"), + ] + + operations = [ + migrations.RunPython(populate_segmenttranslationcontext), + ] diff --git a/wagtail_localize/translation_memory/migrations/0014_remove_location_paths.py b/wagtail_localize/translation_memory/migrations/0014_remove_location_paths.py new file mode 100644 index 00000000..c3463c6d --- /dev/null +++ b/wagtail_localize/translation_memory/migrations/0014_remove_location_paths.py @@ -0,0 +1,44 @@ +# Generated by Django 2.2.7 on 2020-01-02 12:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "wagtail_localize_translation_memory", + "0013_populate_segmenttranslationcontext", + ), + ] + + operations = [ + migrations.RemoveField(model_name="relatedobjectlocation", name="path",), + migrations.RemoveField(model_name="segmentlocation", name="path",), + migrations.RemoveField(model_name="templatelocation", name="path",), + migrations.AlterField( + model_name="relatedobjectlocation", + name="context", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AlterField( + model_name="segmentlocation", + name="context", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + migrations.AlterField( + model_name="templatelocation", + name="context", + field=models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="wagtail_localize_translation_memory.SegmentTranslationContext", + ), + ), + ] diff --git a/wagtail_localize/translation_memory/models.py b/wagtail_localize/translation_memory/models.py index e9ef0fa5..62361127 100644 --- a/wagtail_localize/translation_memory/models.py +++ b/wagtail_localize/translation_memory/models.py @@ -173,23 +173,29 @@ def create_or_update_translation(self, locale, translation=None): created = True # Fetch all translated segments - segment_locations = SegmentLocation.objects.filter( - revision=self - ).annotate_translation(locale.language) + segment_locations = ( + SegmentLocation.objects.filter(revision=self) + .annotate_translation(locale.language) + .select_related("context") + ) - template_locations = TemplateLocation.objects.filter( - revision=self - ).select_related("template") + template_locations = ( + TemplateLocation.objects.filter(revision=self) + .select_related("template") + .select_related("context") + ) - related_object_locations = RelatedObjectLocation.objects.filter( - revision=self - ).select_related("object") + related_object_locations = ( + RelatedObjectLocation.objects.filter(revision=self) + .select_related("object") + .select_related("context") + ) segments = [] for location in segment_locations: segment = SegmentValue.from_html( - location.path, location.translation + location.context.path, location.translation ).with_order(location.order) if location.html_attrs: segment.replace_html_attrs(json.loads(location.html_attrs)) @@ -199,7 +205,7 @@ def create_or_update_translation(self, locale, translation=None): for location in template_locations: template = location.template segment = TemplateValue( - location.path, + location.context.path, template.template_format, template.template, template.segment_count, @@ -209,7 +215,7 @@ def create_or_update_translation(self, locale, translation=None): for location in related_object_locations: segment = RelatedObjectValue( - location.path, + location.context.path, location.object.content_type, location.object.translation_key, order=location.order, @@ -235,22 +241,6 @@ def create_or_update_translation(self, locale, translation=None): return translation, created -class SegmentQuerySet(models.QuerySet): - def annotate_translation(self, language): - """ - Adds a 'translation' field to the segments containing the - text content of the segment translated into the specified - language. - """ - return self.annotate( - translation=Subquery( - SegmentTranslation.objects.filter( - translation_of_id=OuterRef("pk"), language_id=pk(language) - ).values("text") - ) - ) - - class Segment(models.Model): UUID_NAMESPACE = uuid.UUID("59ed7d1c-7eb5-45fa-9c8b-7a7057ed56d7") @@ -258,8 +248,6 @@ class Segment(models.Model): text_id = models.UUIDField() text = models.TextField() - objects = SegmentQuerySet.as_manager() - @classmethod def get_text_id(cls, text): return uuid.uuid5(cls.UUID_NAMESPACE, text) @@ -284,23 +272,69 @@ class Meta: unique_together = [("language", "text_id")] +class SegmentTranslationContext(models.Model): + object = models.ForeignKey( + TranslatableObject, on_delete=models.CASCADE, related_name="+" + ) + path_id = models.UUIDField() + path = models.TextField() + + class Meta: + unique_together = [ + ("object", "path_id"), + ] + + @classmethod + def get_path_id(cls, path): + return uuid.uuid5(uuid.UUID("fcab004a-2b50-11ea-978f-2e728ce88125"), path) + + def save(self, *args, **kwargs): + if self.path and self.path_id is None: + self.path_id = self.get_path_id(self.path) + + return super().save(*args, **kwargs) + + def as_string(self): + """ + Creates a string that can be used in the "msgctxt" field of PO files. + """ + return str(self.object_id) + ":" + self.path + + @classmethod + def get_from_string(cls, msgctxt): + """ + Looks for the SegmentTranslationContext that the given string represents. + """ + object_id, path = msgctxt.split(":") + path_id = cls.get_path_id(path) + return cls.objects.get(object_id=object_id, path_id=path_id) + + class SegmentTranslation(models.Model): translation_of = models.ForeignKey( Segment, on_delete=models.CASCADE, related_name="translations" ) language = models.ForeignKey("wagtail_localize.Language", on_delete=models.CASCADE) + context = models.ForeignKey( + SegmentTranslationContext, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="translations", + ) text = models.TextField() created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) class Meta: - unique_together = [("language", "translation_of")] + unique_together = [("language", "translation_of", "context")] @classmethod - def from_text(cls, translation_of, language, text): + def from_text(cls, translation_of, language, context, text): segment, created = cls.objects.get_or_create( translation_of=translation_of, language_id=pk(language), + context_id=pk(context), defaults={"text": text}, ) @@ -333,7 +367,7 @@ def from_template_value(cls, template_value): class BaseLocation(models.Model): revision = models.ForeignKey(TranslatableRevision, on_delete=models.CASCADE) - path = models.TextField() + context = models.ForeignKey(SegmentTranslationContext, on_delete=models.PROTECT,) order = models.PositiveIntegerField() class Meta: @@ -350,7 +384,9 @@ def annotate_translation(self, language): return self.annotate( translation=Subquery( SegmentTranslation.objects.filter( - translation_of_id=OuterRef("segment_id"), language_id=pk(language) + translation_of_id=OuterRef("segment_id"), + language_id=pk(language), + context_id=OuterRef("context_id"), ).values("text") ) ) @@ -383,10 +419,13 @@ class SegmentLocation(BaseLocation): @classmethod def from_segment_value(cls, revision, language, segment_value): segment = Segment.from_text(language, segment_value.html_with_ids) + context, context_created = SegmentTranslationContext.objects.get_or_create( + object_id=revision.object_id, path=segment_value.path, + ) segment_loc, created = cls.objects.get_or_create( - revision_id=pk(revision), - path=segment_value.path, + revision=revision, + context=context, order=segment_value.order, segment=segment, html_attrs=json.dumps(segment_value.get_html_attrs()), @@ -403,10 +442,13 @@ class TemplateLocation(BaseLocation): @classmethod def from_template_value(cls, revision, template_value): template = Template.from_template_value(template_value) + context, context_created = SegmentTranslationContext.objects.get_or_create( + object_id=revision.object_id, path=template_value.path, + ) template_loc, created = cls.objects.get_or_create( - revision_id=pk(revision), - path=template_value.path, + revision=revision, + context=context, order=template_value.order, template=template, ) @@ -421,9 +463,13 @@ class RelatedObjectLocation(BaseLocation): @classmethod def from_related_object_value(cls, revision, related_object_value): + context, context_created = SegmentTranslationContext.objects.get_or_create( + object_id=revision.object_id, path=related_object_value.path, + ) + related_object_loc, created = cls.objects.get_or_create( - revision_id=pk(revision), - path=related_object_value.path, + revision=revision, + context=context, order=related_object_value.order, object=TranslatableObject.objects.get_or_create( content_type=related_object_value.content_type, diff --git a/wagtail_localize/translation_memory/utils.py b/wagtail_localize/translation_memory/utils.py index e13fe396..10e2a02a 100644 --- a/wagtail_localize/translation_memory/utils.py +++ b/wagtail_localize/translation_memory/utils.py @@ -33,18 +33,16 @@ def get_translation_progress(revision_id, language): - The number of segments that have been translated into the language """ # Get QuerySet of Segments that need to be translated - required_segments = Segment.objects.filter( - id__in=SegmentLocation.objects.filter(revision_id=revision_id).values_list( - "segment_id" - ) - ) + required_segments = SegmentLocation.objects.filter(revision_id=revision_id) # Annotate each Segment with a flag that indicates whether the segment is translated # into the specified language required_segments = required_segments.annotate( is_translated=Exists( SegmentTranslation.objects.filter( - translation_of_id=OuterRef("pk"), language=language + translation_of_id=OuterRef("segment_id"), + context_id=OuterRef("context_id"), + language=language, ) ) )