From 8e4a8be9cc5840275538c6e9d6a8bdfc2bcd584d Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Fri, 10 Jan 2025 10:01:07 +0530 Subject: [PATCH] feat: upstream downstream link model test: recreate_upstream_links management command test: upstream - downstream link tasks chore: fix lint issues refactor: fix import issues temp: point openedx-learning to dev branch refactor: check for upstream_version test: fix failing tests feat: create upstream links after course import refactor: rename learning classes refactor: apply review suggestions refactor: save failed status of course links docs: update docs feat: move link models from openedx-learning chore: organize imports chore: fix lint issues chore: fix dep chore: fix migrations feat: delete links for child block on delete chore: remove unused signal feat: use openedx event to create links on import temp: point openedx-events to dev branch test: fix modulestore tests test: fix weird failure in CI only refactor: update post course import signal name chore: update openedx-events dep --- cms/djangoapps/contentstore/admin.py | 67 +++++ .../commands/recreate_upstream_links.py | 94 ++++++ ...ontextlinksstatus_publishableentitylink.py | 93 ++++++ cms/djangoapps/contentstore/models.py | 178 ++++++++++++ .../contentstore/signals/handlers.py | 69 ++++- cms/djangoapps/contentstore/tasks.py | 78 ++++- .../tests/test_upstream_downstream_links.py | 274 ++++++++++++++++++ .../contentstore/tests/test_utils.py | 15 +- cms/djangoapps/contentstore/utils.py | 65 +++-- .../content_libraries/signal_handlers.py | 3 +- .../djangoapps/content_libraries/tasks.py | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- .../tests/test_mixed_modulestore.py | 13 + xmodule/modulestore/xml_importer.py | 23 +- 17 files changed, 940 insertions(+), 42 deletions(-) create mode 100644 cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py create mode 100644 cms/djangoapps/contentstore/migrations/0009_learningcontextlinksstatus_publishableentitylink.py create mode 100644 cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py index 5b58c9495f19..0b01abe05073 100644 --- a/cms/djangoapps/contentstore/admin.py +++ b/cms/djangoapps/contentstore/admin.py @@ -13,6 +13,8 @@ from cms.djangoapps.contentstore.models import ( BackfillCourseTabsConfig, CleanStaleCertificateAvailabilityDatesConfig, + LearningContextLinksStatus, + PublishableEntityLink, VideoUploadConfig ) from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate @@ -86,6 +88,71 @@ class CleanStaleCertificateAvailabilityDatesConfigAdmin(ConfigurationModelAdmin) pass +@admin.register(PublishableEntityLink) +class PublishableEntityLinkAdmin(admin.ModelAdmin): + """ + PublishableEntityLink admin. + """ + fields = ( + "uuid", + "upstream_block", + "upstream_usage_key", + "upstream_context_key", + "downstream_usage_key", + "downstream_context_key", + "version_synced", + "version_declined", + "created", + "updated", + ) + readonly_fields = fields + list_display = [ + "upstream_block", + "upstream_usage_key", + "downstream_usage_key", + "version_synced", + "updated", + ] + search_fields = [ + "upstream_usage_key", + "upstream_context_key", + "downstream_usage_key", + "downstream_context_key", + ] + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + +@admin.register(LearningContextLinksStatus) +class LearningContextLinksStatusAdmin(admin.ModelAdmin): + """ + LearningContextLinksStatus admin. + """ + fields = ( + "context_key", + "status", + "created", + "updated", + ) + readonly_fields = ("created", "updated") + list_display = ( + "context_key", + "status", + "created", + "updated", + ) + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + admin.site.register(BackfillCourseTabsConfig, ConfigurationModelAdmin) admin.site.register(VideoUploadConfig, ConfigurationModelAdmin) admin.site.register(CourseOutlineRegenerate, CourseOutlineRegenerateAdmin) diff --git a/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py b/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py new file mode 100644 index 000000000000..c1a8454cd56c --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py @@ -0,0 +1,94 @@ +""" +Management command to recreate upstream-dowstream links in PublishableEntityLink for course(s). + +This command can be run for all the courses or for given list of courses. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone + +from django.core.management.base import BaseCommand, CommandError +from django.utils.translation import gettext as _ +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey + +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview + +from ...tasks import create_or_update_upstream_links + +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Recreate links for course(s) in PublishableEntityLink table. + + Examples: + # Recreate upstream links for two courses. + $ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \ + --course course-v1:edX+DemoX.2+2015 + # Force recreate upstream links for one or more courses including processed ones. + $ ./manage.py cms recreate_upstream_links --course course-v1:edX+DemoX.1+2014 \ + --course course-v1:edX+DemoX.2+2015 --force + # Recreate upstream links for all courses. + $ ./manage.py cms recreate_upstream_links --all + # Force recreate links for all courses including completely processed ones. + $ ./manage.py cms recreate_upstream_links --all --force + # Delete all links and force recreate links for all courses + $ ./manage.py cms recreate_upstream_links --all --force --replace + """ + + def add_arguments(self, parser): + parser.add_argument( + '--course', + metavar=_('COURSE_KEY'), + action='append', + help=_('Recreate links for xblocks under given course keys. For eg. course-v1:edX+DemoX.1+2014'), + default=[], + ) + parser.add_argument( + '--all', + action='store_true', + help=_( + 'Recreate links for xblocks under all courses. NOTE: this can take long time depending' + ' on number of course and xblocks' + ), + ) + parser.add_argument( + '--force', + action='store_true', + help=_('Recreate links even for completely processed courses.'), + ) + parser.add_argument( + '--replace', + action='store_true', + help=_('Delete all and create links for given course(s).'), + ) + + def handle(self, *args, **options): + """ + Handle command + """ + courses = options['course'] + should_process_all = options['all'] + force = options['force'] + replace = options['replace'] + time_now = datetime.now(tz=timezone.utc) + if not courses and not should_process_all: + raise CommandError('Either --course or --all argument should be provided.') + + if should_process_all and courses: + raise CommandError('Only one of --course or --all argument should be provided.') + + if should_process_all: + courses = CourseOverview.get_all_course_keys() + for course in courses: + log.info(f"Start processing upstream->dowstream links in course: {course}") + try: + CourseKey.from_string(str(course)) + except InvalidKeyError: + log.error(f"Invalid course key: {course}, skipping..") + continue + create_or_update_upstream_links.delay(str(course), force=force, replace=replace, created=time_now) diff --git a/cms/djangoapps/contentstore/migrations/0009_learningcontextlinksstatus_publishableentitylink.py b/cms/djangoapps/contentstore/migrations/0009_learningcontextlinksstatus_publishableentitylink.py new file mode 100644 index 000000000000..84b80cd63359 --- /dev/null +++ b/cms/djangoapps/contentstore/migrations/0009_learningcontextlinksstatus_publishableentitylink.py @@ -0,0 +1,93 @@ +# Generated by Django 4.2.18 on 2025-02-05 05:33 + +import uuid + +import django.db.models.deletion +import opaque_keys.edx.django.models +import openedx_learning.lib.fields +import openedx_learning.lib.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('oel_publishing', '0002_alter_learningpackage_key_and_more'), + ('contentstore', '0008_cleanstalecertificateavailabilitydatesconfig'), + ] + + operations = [ + migrations.CreateModel( + name='LearningContextLinksStatus', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ( + 'context_key', + opaque_keys.edx.django.models.CourseKeyField( + help_text='Linking status for course context key', max_length=255, unique=True + ), + ), + ( + 'status', + models.CharField( + choices=[ + ('pending', 'Pending'), + ('processing', 'Processing'), + ('failed', 'Failed'), + ('completed', 'Completed'), + ], + help_text='Status of links in given learning context/course.', + max_length=20, + ), + ), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ], + options={ + 'verbose_name': 'Learning Context Links status', + 'verbose_name_plural': 'Learning Context Links status', + }, + ), + migrations.CreateModel( + name='PublishableEntityLink', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ( + 'upstream_usage_key', + opaque_keys.edx.django.models.UsageKeyField( + help_text='Upstream block usage key, this value cannot be null and useful to track upstream library blocks that do not exist yet', + max_length=255, + ), + ), + ( + 'upstream_context_key', + openedx_learning.lib.fields.MultiCollationCharField( + db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, + db_index=True, + help_text='Upstream context key i.e., learning_package/library key', + max_length=500, + ), + ), + ('downstream_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255, unique=True)), + ('downstream_context_key', opaque_keys.edx.django.models.CourseKeyField(db_index=True, max_length=255)), + ('version_synced', models.IntegerField()), + ('version_declined', models.IntegerField(blank=True, null=True)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ( + 'upstream_block', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='links', + to='oel_publishing.publishableentity', + ), + ), + ], + options={ + 'verbose_name': 'Publishable Entity Link', + 'verbose_name_plural': 'Publishable Entity Links', + }, + ), + ] diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index f3b39397cf99..6a1750b8e1c8 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -3,8 +3,20 @@ """ +from datetime import datetime, timezone + from config_models.models import ConfigurationModel +from django.db import models from django.db.models.fields import IntegerField, TextField +from django.utils.translation import gettext_lazy as _ +from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField +from opaque_keys.edx.keys import CourseKey, UsageKey +from openedx_learning.api.authoring_models import Component, PublishableEntity +from openedx_learning.lib.fields import ( + immutable_uuid_field, + key_field, + manual_date_time_field, +) class VideoUploadConfig(ConfigurationModel): @@ -63,3 +75,169 @@ class Meta: "`clean_stale_certificate_available_dates` management command.' See the management command for options." ) ) + + +class PublishableEntityLink(models.Model): + """ + This represents link between any two publishable entities or link between publishable entity and a course + xblock. It helps in tracking relationship between xblocks imported from libraries and used in different courses. + """ + uuid = immutable_uuid_field() + upstream_block = models.ForeignKey( + PublishableEntity, + on_delete=models.SET_NULL, + related_name="links", + null=True, + blank=True, + ) + upstream_usage_key = UsageKeyField( + max_length=255, + help_text=_( + "Upstream block usage key, this value cannot be null" + " and useful to track upstream library blocks that do not exist yet" + ) + ) + # Search by library/upstream context key + upstream_context_key = key_field( + help_text=_("Upstream context key i.e., learning_package/library key"), + db_index=True, + ) + # A downstream entity can only link to single upstream entity + # whereas an entity can be upstream for multiple downstream entities. + downstream_usage_key = UsageKeyField(max_length=255, unique=True) + # Search by course/downstream key + downstream_context_key = CourseKeyField(max_length=255, db_index=True) + version_synced = models.IntegerField() + version_declined = models.IntegerField(null=True, blank=True) + created = manual_date_time_field() + updated = manual_date_time_field() + + def __str__(self): + return f"{self.upstream_usage_key}->{self.downstream_usage_key}" + + class Meta: + verbose_name = _("Publishable Entity Link") + verbose_name_plural = _("Publishable Entity Links") + + @classmethod + def update_or_create( + cls, + upstream_block: Component | None, + /, + upstream_usage_key: UsageKey, + upstream_context_key: str, + downstream_usage_key: UsageKey, + downstream_context_key: CourseKey, + version_synced: int, + version_declined: int | None = None, + created: datetime | None = None, + ) -> "PublishableEntityLink": + """ + Update or create entity link. This will only update `updated` field if something has changed. + """ + if not created: + created = datetime.now(tz=timezone.utc) + new_values = { + 'upstream_usage_key': upstream_usage_key, + 'upstream_context_key': upstream_context_key, + 'downstream_usage_key': downstream_usage_key, + 'downstream_context_key': downstream_context_key, + 'version_synced': version_synced, + 'version_declined': version_declined, + } + if upstream_block: + new_values.update( + { + 'upstream_block': upstream_block.publishable_entity, + } + ) + try: + link = cls.objects.get(downstream_usage_key=downstream_usage_key) + has_changes = False + for key, value in new_values.items(): + prev = getattr(link, key) + # None != None is True, so we need to check for it specially + if prev != value and ~(prev is None and value is None): + has_changes = True + setattr(link, key, value) + if has_changes: + link.updated = created + link.save() + except cls.DoesNotExist: + link = cls(**new_values) + link.created = created + link.updated = created + link.save() + return link + + +class LearningContextLinksStatusChoices(models.TextChoices): + """ + Enumerates the states that a LearningContextLinksStatus can be in. + """ + PENDING = "pending", _("Pending") + PROCESSING = "processing", _("Processing") + FAILED = "failed", _("Failed") + COMPLETED = "completed", _("Completed") + + +class LearningContextLinksStatus(models.Model): + """ + This table stores current processing status of upstream-downstream links in PublishableEntityLink table for a + course or a learning context. + """ + context_key = CourseKeyField( + max_length=255, + # Single entry for a learning context or course + unique=True, + help_text=_("Linking status for course context key"), + ) + status = models.CharField( + max_length=20, + choices=LearningContextLinksStatusChoices.choices, + help_text=_("Status of links in given learning context/course."), + ) + created = manual_date_time_field() + updated = manual_date_time_field() + + class Meta: + verbose_name = _("Learning Context Links status") + verbose_name_plural = _("Learning Context Links status") + + def __str__(self): + return f"{self.status}|{self.context_key}" + + @classmethod + def get_or_create(cls, context_key: str, created: datetime | None = None) -> "LearningContextLinksStatus": + """ + Get or create course link status row from LearningContextLinksStatus table for given course key. + + Args: + context_key: Learning context or Course key + + Returns: + LearningContextLinksStatus object + """ + if not created: + created = datetime.now(tz=timezone.utc) + status, _ = cls.objects.get_or_create( + context_key=context_key, + defaults={ + 'status': LearningContextLinksStatusChoices.PENDING, + 'created': created, + 'updated': created, + }, + ) + return status + + def update_status( + self, + status: LearningContextLinksStatusChoices, + updated: datetime | None = None + ) -> None: + """ + Updates entity links processing status of given learning context. + """ + self.status = status + self.updated = updated or datetime.now(tz=timezone.utc) + self.save() diff --git a/cms/djangoapps/contentstore/signals/handlers.py b/cms/djangoapps/contentstore/signals/handlers.py index d756424bccaa..5f6fa2124678 100644 --- a/cms/djangoapps/contentstore/signals/handlers.py +++ b/cms/djangoapps/contentstore/signals/handlers.py @@ -12,8 +12,14 @@ from django.dispatch import receiver from edx_toggles.toggles import SettingToggle from opaque_keys.edx.keys import CourseKey -from openedx_events.content_authoring.data import CourseCatalogData, CourseScheduleData -from openedx_events.content_authoring.signals import COURSE_CATALOG_INFO_CHANGED +from openedx_events.content_authoring.data import CourseCatalogData, CourseData, CourseScheduleData, XBlockData +from openedx_events.content_authoring.signals import ( + COURSE_CATALOG_INFO_CHANGED, + COURSE_IMPORT_COMPLETED, + XBLOCK_CREATED, + XBLOCK_DELETED, + XBLOCK_UPDATED, +) from pytz import UTC from cms.djangoapps.contentstore.courseware_index import ( @@ -29,6 +35,10 @@ from openedx.core.lib.gating import api as gating_api from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import SignalHandler, modulestore +from xmodule.modulestore.exceptions import ItemNotFoundError + +from ..models import PublishableEntityLink +from ..tasks import create_or_update_upstream_links, handle_create_or_update_xblock_upstream_link from .signals import GRADING_POLICY_CHANGED log = logging.getLogger(__name__) @@ -197,12 +207,19 @@ def handle_item_deleted(**kwargs): # Strip branch info usage_key = usage_key.for_branch(None) course_key = usage_key.course_key - deleted_block = modulestore().get_item(usage_key) + try: + deleted_block = modulestore().get_item(usage_key) + except ItemNotFoundError: + return + id_list = {deleted_block.location} for block in yield_dynamic_block_descendants(deleted_block, kwargs.get('user_id')): # Remove prerequisite milestone data gating_api.remove_prerequisite(block.location) # Remove any 'requires' course content milestone relationships gating_api.set_required_content(course_key, block.location, None, None, None) + id_list.add(block.location) + + PublishableEntityLink.objects.filter(downstream_usage_key__in=id_list).delete() @receiver(GRADING_POLICY_CHANGED) @@ -224,3 +241,49 @@ def handle_grading_policy_changed(sender, **kwargs): task_id=result.task_id, kwargs=kwargs, )) + + +@receiver(XBLOCK_CREATED) +@receiver(XBLOCK_UPDATED) +def create_or_update_upstream_downstream_link_handler(**kwargs): + """ + Automatically create or update upstream->downstream link in database. + """ + xblock_info = kwargs.get("xblock_info", None) + if not xblock_info or not isinstance(xblock_info, XBlockData): + log.error("Received null or incorrect data for event") + return + + handle_create_or_update_xblock_upstream_link.delay(str(xblock_info.usage_key)) + + +@receiver(XBLOCK_DELETED) +def delete_upstream_downstream_link_handler(**kwargs): + """ + Delete upstream->downstream link from database on xblock delete. + """ + xblock_info = kwargs.get("xblock_info", None) + if not xblock_info or not isinstance(xblock_info, XBlockData): + log.error("Received null or incorrect data for event") + return + + PublishableEntityLink.objects.filter( + downstream_usage_key=xblock_info.usage_key + ).delete() + + +@receiver(COURSE_IMPORT_COMPLETED) +def handle_new_course_import(**kwargs): + """ + Automatically create upstream->downstream links for course in database on new import. + """ + course_data = kwargs.get("course", None) + if not course_data or not isinstance(course_data, CourseData): + log.error("Received null or incorrect data for event") + return + + create_or_update_upstream_links.delay( + str(course_data.course_key), + force=True, + replace=True + ) diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index 1c1b5a44abca..88f3ad77f188 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -2,17 +2,17 @@ This file contains celery tasks for contentstore views """ +import asyncio import base64 import json import os +import re import shutil import tarfile -import re -import aiohttp -import asyncio -from datetime import datetime +from datetime import datetime, timezone from tempfile import NamedTemporaryFile, mkdtemp +import aiohttp import olxcleaner import pkg_resources from ccx_keys.locator import CCXLocator @@ -28,11 +28,12 @@ set_code_owner_attribute, set_code_owner_attribute_from_module, set_custom_attribute, - set_custom_attributes_for_course_key + set_custom_attributes_for_course_key, ) from olxcleaner.exceptions import ErrorLevel from olxcleaner.reporting import report_error_summary, report_errors -from opaque_keys.edx.keys import CourseKey +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey, UsageKey from opaque_keys.edx.locator import LibraryLocator from organizations.api import add_organization_course, ensure_organization from organizations.exceptions import InvalidOrganizationException @@ -46,15 +47,16 @@ from cms.djangoapps.contentstore.courseware_index import ( CoursewareSearchIndexer, LibrarySearchIndexer, - SearchIndexingError + SearchIndexingError, ) from cms.djangoapps.contentstore.storage import course_import_export_storage from cms.djangoapps.contentstore.utils import ( IMPORTABLE_FILE_TYPES, + create_or_update_xblock_upstream_link, + delete_course, initialize_permissions, reverse_usage_url, translation_language, - delete_course ) from cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers import get_block_info from cms.djangoapps.models.settings.course_metadata import CourseMetadata @@ -70,6 +72,7 @@ from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider from openedx.core.djangoapps.discussions.tasks import update_unit_discussion_state_from_discussion_blocks from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse +from openedx.core.lib import ensure_cms from openedx.core.lib.extract_archive import safe_extractall from xmodule.contentstore.django import contentstore from xmodule.course_block import CourseFields @@ -79,6 +82,8 @@ from xmodule.modulestore.exceptions import DuplicateCourseError, InvalidProctoringProvider, ItemNotFoundError from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml from xmodule.modulestore.xml_importer import CourseImportException, import_course_from_xml, import_library_from_xml + +from .models import LearningContextLinksStatus, LearningContextLinksStatusChoices, PublishableEntityLink from .outlines import update_outline_from_modulestore from .outlines_regenerate import CourseOutlineRegenerate from .toggles import bypass_olx_failure_enabled @@ -1404,3 +1409,60 @@ def _save_broken_links_file(artifact, file_to_save): def _write_broken_links_to_file(broken_or_locked_urls, broken_links_file): with open(broken_links_file.name, 'w') as file: json.dump(broken_or_locked_urls, file, indent=4) + + +@shared_task +@set_code_owner_attribute +def handle_create_or_update_xblock_upstream_link(usage_key): + """ + Create or update upstream link for a single xblock. + """ + ensure_cms("handle_create_or_update_xblock_upstream_link may only be executed in a CMS context") + try: + xblock = modulestore().get_item(UsageKey.from_string(usage_key)) + except (ItemNotFoundError, InvalidKeyError): + LOGGER.exception(f'Could not find item for given usage_key: {usage_key}') + return + if not xblock.upstream or not xblock.upstream_version: + return + create_or_update_xblock_upstream_link(xblock, xblock.course_id) + + +@shared_task +@set_code_owner_attribute +def create_or_update_upstream_links( + course_key_str: str, + force: bool = False, + replace: bool = False, + created: datetime | None = None, +): + """ + A Celery task to create or update upstream downstream links in database from course xblock content. + """ + ensure_cms("create_or_update_upstream_links may only be executed in a CMS context") + + if not created: + created = datetime.now(timezone.utc) + course_status = LearningContextLinksStatus.get_or_create(course_key_str, created) + if course_status.status in [ + LearningContextLinksStatusChoices.COMPLETED, + LearningContextLinksStatusChoices.PROCESSING + ] and not force: + return + store = modulestore() + course_key = CourseKey.from_string(course_key_str) + course_status.update_status( + LearningContextLinksStatusChoices.PROCESSING, + updated=created, + ) + if replace: + PublishableEntityLink.objects.filter(downstream_context_key=course_key).delete() + try: + xblocks = store.get_items(course_key, settings={"upstream": lambda x: x is not None}) + except ItemNotFoundError: + LOGGER.exception(f'Could not find items for given course: {course_key}') + course_status.update_status(LearningContextLinksStatusChoices.FAILED) + return + for xblock in xblocks: + create_or_update_xblock_upstream_link(xblock, course_key_str, created) + course_status.update_status(LearningContextLinksStatusChoices.COMPLETED) diff --git a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py new file mode 100644 index 000000000000..3f0703d8b31e --- /dev/null +++ b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py @@ -0,0 +1,274 @@ +""" +Tests for upstream downstream tracking links. +""" + +from io import StringIO +from uuid import uuid4 + +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import TestCase +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryUsageLocatorV2 +from openedx_events.tests.utils import OpenEdxEventsTestMixin + +from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangolib.testing.utils import skip_unless_cms +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory + +from ..models import LearningContextLinksStatus, LearningContextLinksStatusChoices, PublishableEntityLink + + +class BaseUpstreamLinksHelpers(TestCase): + """ + Base class with helpers to create xblocks. + """ + def _set_course_data(self, course): + self.section = BlockFactory.create(parent=course, category="chapter", display_name="Section") # pylint: disable=attribute-defined-outside-init + self.sequence = BlockFactory.create(parent=self.section, category="sequential", display_name="Sequence") # pylint: disable=attribute-defined-outside-init + self.unit = BlockFactory.create(parent=self.sequence, category="vertical", display_name="Unit") # pylint: disable=attribute-defined-outside-init + + def _create_block(self, num: int, category="html"): + """ + Create xblock with random upstream key and version number. + """ + random_upstream = LibraryUsageLocatorV2.from_string( + f"lb:OpenedX:CSPROB2:{category}:{uuid4()}" + ) + return random_upstream, BlockFactory.create( + parent=self.unit, # pylint: disable=attribute-defined-outside-init + category=category, + display_name=f"An {category} Block - {num}", + upstream=str(random_upstream), + upstream_version=num, + ) + + def _create_block_and_expected_links_data(self, course_key: str | CourseKey, num_blocks: int = 3): + """ + Creates xblocks and its expected links data for given course_key + """ + data = [] + for i in range(num_blocks): + upstream, block = self._create_block(i + 1) + data.append({ + "upstream_block": None, + "downstream_context_key": course_key, + "downstream_usage_key": block.usage_key, + "upstream_usage_key": upstream, + "upstream_context_key": str(upstream.context_key), + "version_synced": i + 1, + "version_declined": None, + }) + return data + + def _compare_links(self, course_key, expected): + """ + Compares links for given course with passed expected list of dicts. + """ + links = list(PublishableEntityLink.objects.filter(downstream_context_key=course_key).values( + 'upstream_block', + 'upstream_usage_key', + 'upstream_context_key', + 'downstream_usage_key', + 'downstream_context_key', + 'version_synced', + 'version_declined', + )) + self.assertListEqual(links, expected) + + +@skip_unless_cms +class TestRecreateUpstreamLinks(ModuleStoreTestCase, OpenEdxEventsTestMixin, BaseUpstreamLinksHelpers): + """ + Test recreate_upstream_links management command. + """ + + ENABLED_SIGNALS = ['course_deleted', 'course_published'] + ENABLED_OPENEDX_EVENTS = [] + + def setUp(self): + super().setUp() + self.user = UserFactory() + self.course_1 = course_1 = CourseFactory.create(emit_signals=True) + self.course_key_1 = course_key_1 = self.course_1.id + with self.store.bulk_operations(course_key_1): + self._set_course_data(course_1) + self.expected_links_1 = self._create_block_and_expected_links_data(course_key_1) + self.course_2 = course_2 = CourseFactory.create(emit_signals=True) + self.course_key_2 = course_key_2 = self.course_2.id + with self.store.bulk_operations(course_key_2): + self._set_course_data(course_2) + self.expected_links_2 = self._create_block_and_expected_links_data(course_key_2) + self.course_3 = course_3 = CourseFactory.create(emit_signals=True) + self.course_key_3 = course_key_3 = self.course_3.id + with self.store.bulk_operations(course_key_3): + self._set_course_data(course_3) + self.expected_links_3 = self._create_block_and_expected_links_data(course_key_3) + + def call_command(self, *args, **kwargs): + """ + call command with pass args. + """ + out = StringIO() + kwargs['stdout'] = out + err = StringIO() + kwargs['stderr'] = err + call_command('recreate_upstream_links', *args, **kwargs) + return out, err + + def test_call_with_invalid_args(self): + """ + Test command with invalid args. + """ + with self.assertRaisesRegex(CommandError, 'Either --course or --all argument'): + self.call_command() + with self.assertRaisesRegex(CommandError, 'Only one of --course or --all argument'): + self.call_command('--all', '--course', str(self.course_key_1)) + + def test_call_for_single_course(self): + """ + Test command with single course argument + """ + # Pre-checks + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists() + assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_1).exists() + # Run command + self.call_command('--course', str(self.course_key_1)) + # Post verfication + assert LearningContextLinksStatus.objects.filter( + context_key=str(self.course_key_1) + ).first().status == LearningContextLinksStatusChoices.COMPLETED + self._compare_links(self.course_key_1, self.expected_links_1) + + def test_call_for_multiple_course(self): + """ + Test command with multiple course arguments + """ + # Pre-checks + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists() + assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_2).exists() + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_3)).exists() + assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_3).exists() + + # Run command + self.call_command('--course', str(self.course_key_2), '--course', str(self.course_key_3)) + + # Post verfication + assert LearningContextLinksStatus.objects.filter( + context_key=str(self.course_key_2) + ).first().status == LearningContextLinksStatusChoices.COMPLETED + assert LearningContextLinksStatus.objects.filter( + context_key=str(self.course_key_3) + ).first().status == LearningContextLinksStatusChoices.COMPLETED + self._compare_links(self.course_key_2, self.expected_links_2) + self._compare_links(self.course_key_3, self.expected_links_3) + + def test_call_for_all_courses(self): + """ + Test command with multiple course arguments + """ + # Delete all links and status just to make sure --all option works + LearningContextLinksStatus.objects.all().delete() + PublishableEntityLink.objects.all().delete() + # Pre-checks + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists() + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists() + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_3)).exists() + + # Run command + self.call_command('--all') + + # Post verfication + assert LearningContextLinksStatus.objects.filter( + context_key=str(self.course_key_1) + ).first().status == LearningContextLinksStatusChoices.COMPLETED + assert LearningContextLinksStatus.objects.filter( + context_key=str(self.course_key_2) + ).first().status == LearningContextLinksStatusChoices.COMPLETED + assert LearningContextLinksStatus.objects.filter( + context_key=str(self.course_key_3) + ).first().status == LearningContextLinksStatusChoices.COMPLETED + self._compare_links(self.course_key_1, self.expected_links_1) + self._compare_links(self.course_key_2, self.expected_links_2) + self._compare_links(self.course_key_3, self.expected_links_3) + + def test_call_for_invalid_course(self): + """ + Test recreate_upstream_links with nonexistent course + """ + course_key = "invalid-course" + with self.assertLogs(level="ERROR") as ctx: + self.call_command('--course', course_key) + self.assertEqual( + f'Invalid course key: {course_key}, skipping..', + ctx.records[0].getMessage() + ) + + def test_call_for_nonexistent_course(self): + """ + Test recreate_upstream_links with nonexistent course + """ + course_key = "course-v1:unix+ux1+2024_T2" + with self.assertLogs(level="ERROR") as ctx: + self.call_command('--course', course_key) + self.assertIn( + f'Could not find items for given course: {course_key}', + ctx.records[0].getMessage() + ) + + +@skip_unless_cms +class TestUpstreamLinksEvents(ModuleStoreTestCase, OpenEdxEventsTestMixin, BaseUpstreamLinksHelpers): + """ + Test signals related to managing upstream->downstream links. + """ + + ENABLED_SIGNALS = ['course_deleted', 'course_published'] + ENABLED_OPENEDX_EVENTS = [ + "org.openedx.content_authoring.xblock.created.v1", + "org.openedx.content_authoring.xblock.updated.v1", + "org.openedx.content_authoring.xblock.deleted.v1", + ] + + def setUp(self): + super().setUp() + self.user = UserFactory() + self.course_1 = course_1 = CourseFactory.create(emit_signals=True) + self.course_key_1 = course_key_1 = self.course_1.id + with self.store.bulk_operations(course_key_1): + self._set_course_data(course_1) + self.expected_links_1 = self._create_block_and_expected_links_data(course_key_1) + self.course_2 = course_2 = CourseFactory.create(emit_signals=True) + self.course_key_2 = course_key_2 = self.course_2.id + with self.store.bulk_operations(course_key_2): + self._set_course_data(course_2) + self.expected_links_2 = self._create_block_and_expected_links_data(course_key_2) + self.course_3 = course_3 = CourseFactory.create(emit_signals=True) + self.course_key_3 = course_key_3 = self.course_3.id + with self.store.bulk_operations(course_key_3): + self._set_course_data(course_3) + self.expected_links_3 = self._create_block_and_expected_links_data(course_key_3) + + def test_create_or_update_events(self): + """ + Test task create_or_update_upstream_links for a course + """ + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists() + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists() + assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_3)).exists() + assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_1).count() == 3 + assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_2).count() == 3 + assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_3).count() == 3 + self._compare_links(self.course_key_1, self.expected_links_1) + self._compare_links(self.course_key_2, self.expected_links_2) + self._compare_links(self.course_key_3, self.expected_links_3) + + def test_delete_handler(self): + """ + Test whether links are deleted on deletion of xblock. + """ + usage_key = self.expected_links_1[0]["downstream_usage_key"] + assert PublishableEntityLink.objects.filter(downstream_usage_key=usage_key).exists() + self.store.delete_item(usage_key, self.user.id) + assert not PublishableEntityLink.objects.filter(downstream_usage_key=usage_key).exists() diff --git a/cms/djangoapps/contentstore/tests/test_utils.py b/cms/djangoapps/contentstore/tests/test_utils.py index a91379797060..a46d9831d4e5 100644 --- a/cms/djangoapps/contentstore/tests/test_utils.py +++ b/cms/djangoapps/contentstore/tests/test_utils.py @@ -12,6 +12,7 @@ from edx_toggles.toggles.testutils import override_waffle_flag from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import CourseLocator, LibraryLocator +from openedx_events.tests.utils import OpenEdxEventsTestMixin from path import Path as path from pytz import UTC from rest_framework import status @@ -31,10 +32,13 @@ from xmodule.modulestore.tests.django_utils import ( # lint-amnesty, pylint: disable=wrong-import-order TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase, - SharedModuleStoreTestCase + SharedModuleStoreTestCase, ) -from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.partitions.partitions import Group, UserPartition # lint-amnesty, pylint: disable=wrong-import-order +from xmodule.modulestore.tests.factories import ( + BlockFactory, + CourseFactory, +) +from xmodule.partitions.partitions import Group, UserPartition class LMSLinksTestCase(TestCase): @@ -935,10 +939,13 @@ def test_update_course_details_instructor_paced(self, mock_update): @override_waffle_flag(ENABLE_NOTIFICATIONS, active=True) -class CourseUpdateNotificationTests(ModuleStoreTestCase): +class CourseUpdateNotificationTests(OpenEdxEventsTestMixin, ModuleStoreTestCase): """ Unit tests for the course_update notification. """ + ENABLED_OPENEDX_EVENTS = [ + "org.openedx.learning.course.notification.requested.v1", + ] def setUp(self): """ diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 2cbc28730c8f..79d8f757a7d1 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -2,6 +2,7 @@ Common utility functions useful throughout the contentstore """ from __future__ import annotations + import configparser import html import logging @@ -9,12 +10,12 @@ from collections import defaultdict from contextlib import contextmanager from datetime import datetime, timezone -from urllib.parse import quote_plus, urlencode, urlunparse, urlparse +from urllib.parse import quote_plus, urlencode, urlparse, urlunparse from uuid import uuid4 from bs4 import BeautifulSoup from django.conf import settings -from django.core.exceptions import ValidationError +from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.urls import reverse from django.utils import translation from django.utils.text import Truncator @@ -22,16 +23,13 @@ from eventtracking import tracker from help_tokens.core import HelpUrlExpert from lti_consumer.models import CourseAllowPIISharingInLTIFlag -from opaque_keys.edx.keys import CourseKey, UsageKey +from milestones import api as milestones_api +from opaque_keys.edx.keys import CourseKey, UsageKey, UsageKeyV2 from opaque_keys.edx.locator import LibraryLocator - -from openedx.core.lib.teams_config import CONTENT_GROUPS_FOR_TEAMS, TEAM_SCHEME from openedx_events.content_authoring.data import DuplicatedXBlockData from openedx_events.content_authoring.signals import XBLOCK_DUPLICATED from openedx_events.learning.data import CourseNotificationData from openedx_events.learning.signals import COURSE_NOTIFICATION_REQUESTED - -from milestones import api as milestones_api from pytz import UTC from xblock.fields import Scope @@ -42,13 +40,14 @@ libraries_v2_enabled, split_library_view_on_dashboard, use_new_advanced_settings_page, - use_new_course_outline_page, use_new_certificates_page, + use_new_course_outline_page, + use_new_course_team_page, + use_new_custom_pages, use_new_export_page, use_new_files_uploads_page, use_new_grading_page, use_new_group_configurations_page, - use_new_course_team_page, use_new_home_page, use_new_import_page, use_new_schedule_details_page, @@ -58,16 +57,15 @@ use_new_updates_page, use_new_video_editor, use_new_video_uploads_page, - use_new_custom_pages, ) from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.djangoapps.models.settings.course_metadata import CourseMetadata -from common.djangoapps.course_action_state.models import CourseRerunUIStateManager, CourseRerunState from common.djangoapps.course_action_state.managers import CourseActionStateItemNotFoundError +from common.djangoapps.course_action_state.models import CourseRerunState, CourseRerunUIStateManager from common.djangoapps.course_modes.models import CourseMode from common.djangoapps.edxmako.services import MakoService from common.djangoapps.student import auth -from common.djangoapps.student.auth import has_studio_read_access, has_studio_write_access, STUDIO_EDIT_ROLES +from common.djangoapps.student.auth import STUDIO_EDIT_ROLES, has_studio_read_access, has_studio_write_access from common.djangoapps.student.models import CourseEnrollment from common.djangoapps.student.roles import ( CourseInstructorRole, @@ -76,15 +74,15 @@ ) from common.djangoapps.track import contexts from common.djangoapps.util.course import get_link_for_about_page +from common.djangoapps.util.date_utils import get_default_time_display from common.djangoapps.util.milestones_helpers import ( + generate_milestone_namespace, + get_namespace_choices, is_prerequisite_courses_enabled, is_valid_course_key, remove_prerequisite_course, set_prerequisite_courses, - get_namespace_choices, - generate_milestone_namespace ) -from common.djangoapps.util.date_utils import get_default_time_display from common.djangoapps.xblock_django.api import deprecated_xblocks from common.djangoapps.xblock_django.user_service import DjangoXBlockUserService from openedx.core import toggles as core_toggles @@ -94,23 +92,28 @@ from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration from openedx.core.djangoapps.django_comment_common.models import assign_default_role from openedx.core.djangoapps.django_comment_common.utils import seed_permissions_roles +from openedx.core.djangoapps.models.course_details import CourseDetails from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers from openedx.core.djangoapps.site_configuration.models import SiteConfiguration -from openedx.core.djangoapps.models.course_details import CourseDetails +from openedx.core.djangoapps.xblock.api import get_component_from_usage_key from openedx.core.lib.courses import course_image_url from openedx.core.lib.html_to_text import html_to_text +from openedx.core.lib.teams_config import CONTENT_GROUPS_FOR_TEAMS, TEAM_SCHEME from openedx.features.content_type_gating.models import ContentTypeGatingConfig from openedx.features.content_type_gating.partitions import CONTENT_TYPE_GATING_SCHEME from openedx.features.course_experience.waffle import ENABLE_COURSE_ABOUT_SIDEBAR_HTML -from xmodule.library_tools import LegacyLibraryToolsService from xmodule.course_block import DEFAULT_START_DATE # lint-amnesty, pylint: disable=wrong-import-order from xmodule.data import CertificatesDisplayBehaviors +from xmodule.library_tools import LegacyLibraryToolsService from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.exceptions import ItemNotFoundError # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.partitions.partitions_service import get_all_partitions_for_course # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.services import SettingsService, ConfigurationService, TeamsConfigurationService +from xmodule.partitions.partitions_service import ( + get_all_partitions_for_course, # lint-amnesty, pylint: disable=wrong-import-order +) +from xmodule.services import ConfigurationService, SettingsService, TeamsConfigurationService +from .models import PublishableEntityLink IMPORTABLE_FILE_TYPES = ('.tar.gz', '.zip') log = logging.getLogger(__name__) @@ -2354,3 +2357,27 @@ def get_xblock_render_context(request, block): return str(exc) return "" + + +def create_or_update_xblock_upstream_link(xblock, course_key: str | CourseKey, created: datetime | None = None): + """ + Create or update upstream->downstream link in database for given xblock. + """ + if not xblock.upstream: + return None + upstream_usage_key = UsageKeyV2.from_string(xblock.upstream) + try: + lib_component = get_component_from_usage_key(upstream_usage_key) + except ObjectDoesNotExist: + log.error(f"Library component not found for {upstream_usage_key}") + lib_component = None + PublishableEntityLink.update_or_create( + lib_component, + upstream_usage_key=xblock.upstream, + upstream_context_key=str(upstream_usage_key.context_key), + downstream_context_key=course_key, + downstream_usage_key=xblock.usage_key, + version_synced=xblock.upstream_version, + version_declined=xblock.upstream_version_declined, + created=created, + ) diff --git a/openedx/core/djangoapps/content_libraries/signal_handlers.py b/openedx/core/djangoapps/content_libraries/signal_handlers.py index 58f45d218e95..68cc7512e540 100644 --- a/openedx/core/djangoapps/content_libraries/signal_handlers.py +++ b/openedx/core/djangoapps/content_libraries/signal_handlers.py @@ -5,7 +5,7 @@ import logging from django.conf import settings -from django.db.models.signals import post_save, post_delete, m2m_changed +from django.db.models.signals import m2m_changed, post_delete, post_save from django.dispatch import receiver from opaque_keys import InvalidKeyError @@ -28,7 +28,6 @@ from .api import library_component_usage_key from .models import ContentLibrary, LtiGradedResource - log = logging.getLogger(__name__) diff --git a/openedx/core/djangoapps/content_libraries/tasks.py b/openedx/core/djangoapps/content_libraries/tasks.py index f56b4adfe313..b76101e1c62e 100644 --- a/openedx/core/djangoapps/content_libraries/tasks.py +++ b/openedx/core/djangoapps/content_libraries/tasks.py @@ -22,11 +22,11 @@ from celery_utils.logged_task import LoggedTask from celery.utils.log import get_task_logger from edx_django_utils.monitoring import set_code_owner_attribute, set_code_owner_attribute_from_module +from opaque_keys.edx.keys import CourseKey from user_tasks.tasks import UserTask, UserTaskStatus from xblock.fields import Scope -from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import BlockUsageLocator from openedx.core.lib import ensure_cms from xmodule.capa_block import ProblemBlock diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 1b45eaf68266..fafc6ff17c8d 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -811,7 +811,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/kernel.in openedx-django-wiki==2.1.0 # via -r requirements/edx/kernel.in -openedx-events==9.15.2 +openedx-events==9.18.0 # via # -r requirements/edx/kernel.in # edx-enterprise diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 2926240aefab..318464d44812 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1361,7 +1361,7 @@ openedx-django-wiki==2.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-events==9.15.2 +openedx-events==9.18.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 0731746dc240..ddda43eb5b70 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -984,7 +984,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/base.txt openedx-django-wiki==2.1.0 # via -r requirements/edx/base.txt -openedx-events==9.15.2 +openedx-events==9.18.0 # via # -r requirements/edx/base.txt # edx-enterprise diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 6dd9f7ce23fb..6a727650fe71 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1032,7 +1032,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/base.txt openedx-django-wiki==2.1.0 # via -r requirements/edx/base.txt -openedx-events==9.15.2 +openedx-events==9.18.0 # via # -r requirements/edx/base.txt # edx-enterprise diff --git a/xmodule/modulestore/tests/test_mixed_modulestore.py b/xmodule/modulestore/tests/test_mixed_modulestore.py index 0928ab253b9c..bbc91f832f96 100644 --- a/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -164,6 +164,19 @@ def setUp(self): self.course_locations = {} self.user_id = ModuleStoreEnum.UserID.test + # mock and ignore publishable link entity related tasks to avoid unnecessary + # errors as it is tested separately + if settings.ROOT_URLCONF == 'cms.urls': + create_or_update_xblock_upstream_link_patch = patch( + 'cms.djangoapps.contentstore.signals.handlers.handle_create_or_update_xblock_upstream_link' + ) + create_or_update_xblock_upstream_link_patch.start() + self.addCleanup(create_or_update_xblock_upstream_link_patch.stop) + publishableEntityLinkPatch = patch( + 'cms.djangoapps.contentstore.signals.handlers.PublishableEntityLink' + ) + publishableEntityLinkPatch.start() + self.addCleanup(publishableEntityLinkPatch.stop) def _check_connection(self): """ diff --git a/xmodule/modulestore/xml_importer.py b/xmodule/modulestore/xml_importer.py index 5b880b4ade2f..05f0a9954ef6 100644 --- a/xmodule/modulestore/xml_importer.py +++ b/xmodule/modulestore/xml_importer.py @@ -27,6 +27,7 @@ import os import re from abc import abstractmethod +from datetime import datetime, timezone import xblock from django.core.exceptions import ObjectDoesNotExist @@ -34,12 +35,15 @@ from lxml import etree from opaque_keys.edx.keys import UsageKey from opaque_keys.edx.locator import LibraryLocator +from openedx_events.content_authoring.data import CourseData +from openedx_events.content_authoring.signals import COURSE_IMPORT_COMPLETED from path import Path as path from xblock.core import XBlockMixin from xblock.fields import Reference, ReferenceList, ReferenceValueDict, Scope from xblock.runtime import DictKeyValueStore, KvsFieldData from common.djangoapps.util.monitoring import monitor_import_failure +from openedx.core.djangoapps.content_tagging.api import import_course_tags_from_csv from xmodule.assetstore import AssetMetadata from xmodule.contentstore.content import StaticContent from xmodule.errortracker import make_error_tracker @@ -52,7 +56,6 @@ from xmodule.tabs import CourseTabList from xmodule.util.misc import escape_invalid_characters from xmodule.x_module import XModuleMixin -from openedx.core.djangoapps.content_tagging.api import import_course_tags_from_csv from .inheritance import own_metadata from .store_utilities import rewrite_nonportable_content_links @@ -548,6 +551,11 @@ def depth_first(subtree): # pylint: disable=raise-missing-from raise BlockFailedToImport(leftover.display_name, leftover.location) + def post_course_import(self, dest_id): + """ + Tasks that need to triggered after a course is imported. + """ + def run_imports(self): """ Iterate over the given directories and yield courses. @@ -589,6 +597,7 @@ def run_imports(self): logging.info(f'Course import {dest_id}: No tags.csv file present.') except ValueError as e: logging.info(f'Course import {dest_id}: {str(e)}') + self.post_course_import(dest_id) yield courselike @@ -717,6 +726,18 @@ def import_tags(self, data_path, dest_id): csv_path = path(data_path) / 'tags.csv' import_course_tags_from_csv(csv_path, dest_id) + def post_course_import(self, dest_id): + """ + Trigger celery task to create upstream links for newly imported blocks. + """ + # .. event_implemented_name: COURSE_IMPORT_COMPLETED + COURSE_IMPORT_COMPLETED.send_event( + time=datetime.now(timezone.utc), + course=CourseData( + course_key=dest_id + ) + ) + class LibraryImportManager(ImportManager): """