Contentful: Migrate away from legacy Compose (#12133)

* Add set of Contentful-related migrations that migrate us away from the
"Legacy Compose" pattern. This is based on
https://www.contentful.com/developers/docs/compose/upgrade-to-customizable-compose-content-model/
plus investigations by @maureenlholland.

The migrations are kept separate so they can be applied individually and
progress monitored. Also, it makes sense to 'commit' each step separately,
rather than risk locking/race issues with the Contentful API's datastore.
(This is an assumption on my part - maybe we _could_ do all this in one go.)

First one: Adds new fields to any page type that previously used Legacy Compose
(In our case that's just the pagePageResourceCenter content type.)

Second one: Populates data, drawing from the Compose:Page entry and setting it
on the pagePageResourceCenter entries.

Third one: cleanup, unpublishing and deleting the Compose:Page entries and then
removing it as a content type entirely

* Update backend Python code to match updated page Schema now we have dropped legacy Compose

Note that the Connect:Homepage approach still leaves us some niggles that
we should ideally get past soon, too

* Improve ordering of fields for pagePageResourceCenter via Contentful migration

Example of it running:

Update Content Type pagePageResourceCenter

  Move field title after field name

  Move field slug after field title

  Move field seo after field slug

Publish Content Type pagePageResourceCenter
Update Content Type pagePageResourceCenter
Migration successful

* Add ADR for how we're handling Contentful migrations

* Update README for Contentful migrations approach

* Update README and ADR with typo fixes

* Typo fixups, following code review

* Add Contentful migration to make Title and Slug on Resource Center pages required

* Add Contentful migration to force-publish all Resource Center pages

Author: stevejalim

.editorconfig

@@ -10,7 +10,7 @@ insert_final_newline = true
 trim_trailing_whitespace = true
 indent_style = space
 
-[*.{py,js,css,scss,sh,groovy}]
+[*.{py,js,cjs,css,scss,sh,groovy}]
 indent_size = 4
 
 [*.{json,html,svg,yml}]

.eslintrc.js

@@ -3,13 +3,14 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
-
+/* eslint-env es6 */
 module.exports = {
     env: {
         browser: true,
         commonjs: true
     },
     extends: ['eslint:recommended', 'plugin:json/recommended', 'prettier'],
+    ignorePatterns: ['contentful_migrations/migrations/*.cjs'],
     rules: {
         // Require strict mode directive in top level functions
         // https://eslint.org/docs/rules/strict

bedrock/contentful/api.py

@@ -17,7 +17,6 @@
 from rich_text_renderer.text_renderers import BaseInlineRenderer
 
 from bedrock.contentful.constants import (
-    COMPOSE_MAIN_PAGE_TYPE,
     CONTENT_TYPE_PAGE_GENERAL,
     CONTENT_TYPE_PAGE_RESOURCE_CENTER,
 )
@@ -479,17 +478,8 @@ def _get_preview_image_from_fields(self, fields):
                 return f"https:{preview_image_url}"
 
     def _get_info_data__slug_title_blurb(self, entry_fields, seo_fields):
-
-        if self.page.content_type.id == COMPOSE_MAIN_PAGE_TYPE:
-            # This means we're dealing with a Compose-structured setup,
-            # and the slug lives not on the Entry, nor the SEO object
-            # but just on the top-level Compose `page`
-            slug = self.page.fields().get("slug")
-        else:
-            # Non-Compose pages
-            slug = entry_fields.get("slug", "home")  # TODO: check if we can use a better fallback
-
-        title = getattr(self.page, "title", "")
+        slug = entry_fields.get("slug", "home")  # TODO: check if we can use a better fallback
+        title = entry_fields.get("title", "")
         title = entry_fields.get("preview_title", title)
         blurb = entry_fields.get("preview_blurb", "")
 
@@ -542,7 +532,6 @@ def _get_info_data__locale(self, page_type, entry_fields, entry_obj):
         return {"locale": locale}
 
     def get_info_data(self, entry_obj, seo_obj=None):
-        # TODO, need to enable connectors
         entry_fields = entry_obj.fields()
         if seo_obj:
             seo_fields = seo_obj.fields()
@@ -591,20 +580,15 @@ def get_info_data(self, entry_obj, seo_obj=None):
         return data
 
     def get_content(self):
-        # Check if it is a page or a connector, or a Compose page type
-
+        # Check if it is a page or a connector
         entry_type = self.page.content_type.id
         seo_obj = None
-        if entry_type == COMPOSE_MAIN_PAGE_TYPE:
-            # Contentful Compose page, linking to content and SEO models
-            entry_obj = self.page.content  # The page with the actual content
-            seo_obj = self.page.seo  # The SEO model
-            # Note that the slug lives on self.page, not the seo_obj.
-        elif entry_type.startswith("page"):
-            entry_obj = self.page
-        elif entry_type == "connectHomepage":
+        if entry_type == "connectHomepage":
             # Legacy - TODO: remove me once we're no longer using Connect: Homepage
             entry_obj = self.page.fields()["entry"]
+        elif entry_type.startswith("page"):  # WARNING: this requires a consistent naming of page types in Contentful, too
+            entry_obj = self.page
+            seo_obj = self.page.seo
         else:
             raise ValueError(f"{entry_type} is not a recognized page type")
 

bedrock/contentful/constants.py

@@ -2,11 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-
-# Which models do we want to sync?
-
-
-COMPOSE_MAIN_PAGE_TYPE = "page"
 MAX_MESSAGES_PER_QUEUE_POLL = 10
 
 # Specific content types we need to target in DB lookups
@@ -15,15 +10,13 @@
 CONTENT_TYPE_PAGE_GENERAL = "pageGeneral"
 
 DEFAULT_CONTENT_TYPES = ",".join(
-    # Soon, we'll only need to sync the Compose-driven `page` type, but
-    # until then we also still have homepages set with the Connect pattern
     [
-        CONTENT_TYPE_CONNECT_HOMEPAGE,  # The Connect-based approach, used for the homepage
-        COMPOSE_MAIN_PAGE_TYPE,  # General Compose Page type - the related `content` type's name is what we store in the DB
+        CONTENT_TYPE_CONNECT_HOMEPAGE,  # The Connect-based approach, currently used for the homepage
+        CONTENT_TYPE_PAGE_RESOURCE_CENTER,  # New-era Compose page with a dedicated type
     ]
 )
 
-CONTENT_CLASSIFICATION_VPN = "VPN"  # Matches string in Contenful for VPN as `product`
+CONTENT_CLASSIFICATION_VPN = "VPN"  # Matches string in Contentful for VPN as `product`
 
 ARTICLE_CATEGORY_LABEL = "category"  # for URL-param filtering
 

bedrock/contentful/management/commands/update_contentful.py

@@ -25,7 +25,6 @@
     ACTION_SAVE,
     ACTION_UNARCHIVE,
     ACTION_UNPUBLISH,
-    COMPOSE_MAIN_PAGE_TYPE,
     CONTENT_TYPE_CONNECT_HOMEPAGE,
     MAX_MESSAGES_PER_QUEUE_POLL,
 )
@@ -362,13 +361,6 @@ def _refresh_from_contentful(self) -> Tuple[int, int, int]:
                 error_count += 1
                 continue
 
-            # Compose-authored pages have a page_type of `page`
-            # but really we want the entity the Compose page references
-            if ctype == COMPOSE_MAIN_PAGE_TYPE:
-                # TODO: make this standard when we _only_ have Compose pages,
-                # because they all have a parent type of COMPOSE_MAIN_PAGE_TYPE
-                ctype = page_data["page_type"]
-
             hash = data_hash(page_data)
             _info = page_data["info"]
 

bedrock/contentful/models.py

@@ -60,7 +60,7 @@ def get_entries_by_type(
             order_by (str, optional): Sorting key for the queryset. Defaults to "last_modified".
 
         Returns:
-            QuerySet[ContentfulEntry]: the main ContenfulEntry models, not just their JSON data
+            QuerySet[ContentfulEntry]: the main ContentfulEntry models, not just their JSON data
         """
 
         kwargs = dict(

bedrock/contentful/tests/test_contentful_api.py

@@ -43,7 +43,6 @@
     get_client,
 )
 from bedrock.contentful.constants import (
-    COMPOSE_MAIN_PAGE_TYPE,
     CONTENT_TYPE_CONNECT_HOMEPAGE,
     CONTENT_TYPE_PAGE_RESOURCE_CENTER,
 )
@@ -681,7 +680,7 @@ def test__render_list():
 
 @pytest.fixture
 def basic_contentful_page(rf):
-    """Naive reusable fixutre for setting up a ContentfulPage
+    """Naive reusable fixture for setting up a ContentfulPage
     Note that it does NOTHING with set_current_request / thread-locals
     """
     with patch("bedrock.contentful.api.set_current_request"):
@@ -868,84 +867,90 @@ def test_ContentfulPage__get_info_data__locale(
 
 
 @pytest.mark.parametrize(
-    "page_title, page_type, page_fields, entry_fields, seo_fields, expected",
+    "page_type, legacy_connect_page_fields, entry_fields, seo_fields, expected",
     (
         (
-            "test page one",
-            COMPOSE_MAIN_PAGE_TYPE,
-            {"slug": "compose-main-page-slug"},
-            {},
-            {},
+            CONTENT_TYPE_PAGE_RESOURCE_CENTER,
+            {},  # Connect-type fields
             {
-                "slug": "compose-main-page-slug",
+                "slug": "vrc-main-page-slug",
+                "title": "test page one",
+            },  # fields from the page itself
+            {},  # SEO object's fields
+            {
+                "slug": "vrc-main-page-slug",
                 "title": "test page one",
                 "blurb": "",
             },
         ),
         (
-            "",
-            COMPOSE_MAIN_PAGE_TYPE,
-            {"slug": "compose-main-page-slug"},
+            CONTENT_TYPE_PAGE_RESOURCE_CENTER,
             {},
+            {
+                "slug": "vrc-main-page-slug",
+                "title": "",
+            },
             {},
             {
-                "slug": "compose-main-page-slug",
+                "slug": "vrc-main-page-slug",
                 "title": "",
                 "blurb": "",
             },
         ),
         (
-            "",
-            COMPOSE_MAIN_PAGE_TYPE,
-            {"slug": "compose-main-page-slug"},
+            CONTENT_TYPE_PAGE_RESOURCE_CENTER,
+            {},
             {
                 "preview_title": "preview title",
                 "preview_blurb": "preview blurb",
+                "slug": "vrc-main-page-slug",
+                "title": "",
             },
             {},
             {
-                "slug": "compose-main-page-slug",
+                "slug": "vrc-main-page-slug",
                 "title": "preview title",
                 "blurb": "preview blurb",
             },
         ),
         (
-            "",
-            COMPOSE_MAIN_PAGE_TYPE,
-            {"slug": "compose-main-page-slug"},
+            CONTENT_TYPE_PAGE_RESOURCE_CENTER,
+            {},
             {
                 "preview_title": "preview title",
                 "preview_blurb": "preview blurb",
+                "slug": "vrc-main-page-slug",
+                "title": "",
             },
             {"description": "seo description"},
             {
-                "slug": "compose-main-page-slug",
+                "slug": "vrc-main-page-slug",
                 "title": "preview title",
                 "blurb": "seo description",
             },
         ),
         (
-            "page title",
             CONTENT_TYPE_CONNECT_HOMEPAGE,
-            {},
             {
-                "slug": "homepage-slug",
+                "slug": "homepage-slug",  # This will be ignored
+            },
+            {
                 "preview_title": "preview title",
                 "preview_blurb": "preview blurb",
             },
             {},  # SEO fields not present for non-Compose pages
             {
-                "slug": "homepage-slug",
+                "slug": "home",  # ie, there is no way to set the slug using Connect:Homepage
                 "title": "preview title",
                 "blurb": "preview blurb",
             },
         ),
         (
-            "page title",
             CONTENT_TYPE_CONNECT_HOMEPAGE,
-            {},
             {
                 # no slug field, so will fall back to default of 'home'
+            },
+            {
                 "preview_title": "preview title",
                 "preview_blurb": "preview blurb",
             },
@@ -962,26 +967,21 @@ def test_ContentfulPage__get_info_data__locale(
         "compose page with slug, no title, no blurb",
         "compose page with slug, title from entry, blurb from entry",
         "compose page with slug, no title, blurb from seo",
-        "Non-Compose page with slug, title, blurb from entry",
+        "Non-Compose page with title, blurb from entry + PROOF SLUG IS NOT SET",
         "Non-Compose page with default slug, title, blurb from entry",
     ],
 )
 def test_ContentfulPage__get_info_data__slug_title_blurb(
     basic_contentful_page,
-    page_title,
     page_type,
-    page_fields,
+    legacy_connect_page_fields,
     entry_fields,
     seo_fields,
     expected,
 ):
     basic_contentful_page.page = Mock()
     basic_contentful_page.page.content_type.id = page_type
-    basic_contentful_page.page.fields = Mock(return_value=page_fields)
-    if page_title:
-        basic_contentful_page.page.title = page_title
-    else:
-        basic_contentful_page.page.title = ""
+    basic_contentful_page.page.fields = Mock(return_value=legacy_connect_page_fields)
 
     assert (
         basic_contentful_page._get_info_data__slug_title_blurb(
@@ -1049,6 +1049,7 @@ def test_ContentfulPage__get_info_data__category_tags_classification(
             {
                 "dummy": "seo fields",
                 "preview_image": "https://example.com/test-seo.png",
+                "description": "Test SEO description comes through",
             },
             {
                 "title": "test title",
@@ -1068,6 +1069,7 @@ def test_ContentfulPage__get_info_data__category_tags_classification(
                 "seo": {
                     "dummy": "seo fields",
                     "image": "https://example.com/test-seo.png",
+                    "description": "Test SEO description comes through",
                 },
             },
         ),
@@ -1169,11 +1171,11 @@ def test_ContentfulPage__get_info_data(
                 {
                     "dummy": "seo fields",
                     "preview_image": "https://example.com/test-seo.png",
+                    "description": "Test SEO description comes through",
                 },
             )
             in mock__get_preview_image_from_fields.call_args_list
         )
-
     else:
         assert mock__get_preview_image_from_fields.call_count == 1
         mock__get_preview_image_from_fields.assert_called_once_with(entry_obj__fields)