Merge branch 'braden/meilisearch-libraries' into rpenido/fal-3690-upd…

…ate-search-index-when-course-content-is-changed

.github/CODEOWNERS

@@ -20,7 +20,7 @@ openedx/core/djangoapps/credit                                      @openedx/2U-
 openedx/core/djangoapps/heartbeat/
 openedx/core/djangoapps/oauth_dispatch
 openedx/core/djangoapps/user_api/                                   @openedx/2U-aperture
-openedx/core/djangoapps/user_authn/
+openedx/core/djangoapps/user_authn/                                 @openedx/2U-vanguards                                 
 openedx/features/course_experience/
 xmodule/
 

.github/workflows/migrations-check.yml

@@ -19,17 +19,17 @@ jobs:
         # within the global constraint i.e. Django==4.2.8 in current case
         # because we have global constraint of Django<4.2 
         django-version: ["pinned"]
-        mongo-version: ["4"]
+        mongo-version: ["4", "7"]
         mysql-version: ["8"]
     services:
       mongo:
         image: mongo:${{ matrix.mongo-version }}
         ports:
           - 27017:27017
         # Note: Calling mongo here only works with mongo 4, in newer versions of mongo
-        # we'll have to use `mongosh`
+        # we'll have to use `mongosh`, hence the 'which mongosh mongo'.
         options: >-
-          --health-cmd "mongo --quiet --eval 'db.runCommand(\"ping\")'"
+          --health-cmd "$(which mongosh mongo) --quiet --eval 'db.runCommand(\"ping\")'"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 3

.github/workflows/static-assets-check.yml

@@ -16,6 +16,20 @@ jobs:
         python-version: [ 3.8 ]
         node-version: [ 16 ]
         npm-version: [ 8.5.x ]
+        mongo-version: ["4.4", "7.0"]
+
+    services:
+      mongo:
+        image: mongo:${{ matrix.mongo-version }}
+        ports:
+          - 27017:27017
+        # Note: Calling mongo here only works with mongo 4, in newer versions of mongo
+        # we'll have to use `mongosh`, hence the 'which mongosh mongo'.
+        options: >-
+          --health-cmd "$(which mongosh mongo) --quiet --eval 'db.runCommand(\"ping\")'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 3
 
     steps:
     - name: Checkout repo

.github/workflows/unit-tests-gh-hosted.yml

@@ -36,7 +36,10 @@ jobs:
           - "common-with-cms"
           - "xmodule-with-lms"
           - "xmodule-with-cms"
-    name: gh-hosted-python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }}
+        mongo-version:
+          - "4.4"
+          - "7.0"
+    name: gh-hosted-python-${{ matrix.python-version }},django-${{ matrix.django-version }},mongo-${{ matrix.mongo-version }}${{ matrix.shard_name }}
     steps:
       - uses: actions/checkout@v2
 
@@ -46,7 +49,7 @@ jobs:
       - name: Start MongoDB
         uses: supercharge/mongodb-github-action@1.7.0
         with:
-          mongodb-version: 4.4
+          mongodb-version: ${{ matrix.mongo-version}}
 
       - name: Setup Python
         uses: actions/setup-python@v4

.github/workflows/unit-tests.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   run-tests:
-    name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }}
+    name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},mongo-${{ matrix.mongo-version }},${{ matrix.shard_name }}
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
     runs-on: [ edx-platform-runner ]
     strategy:
@@ -36,13 +36,26 @@ jobs:
           - "common-with-cms"
           - "xmodule-with-lms"
           - "xmodule-with-cms"
+        mongo-version:
+          - "4.4"
+          - "7.0"
     # We expect Django 4.0 to fail, so don't stop when it fails.
     continue-on-error: ${{ matrix.django-version == '4.0' }}
 
     steps:
       - name: sync directory owner
         run: sudo chown runner:runner -R .*
 
+      - name: install mongo version
+        run: |
+          if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then
+            sudo apt-get purge -y "mongodb-org*"
+            sudo apt-get remove -y mongodb-org
+            wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add -
+            echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list
+            sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*"
+          fi
+
       - name: checkout repo
         uses: actions/checkout@v3
 

.github/workflows/units-test-scripts-structures-pruning.yml

@@ -0,0 +1,33 @@
+name: units-test-scripts-common
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.12' ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r scripts/structures_pruning/requirements/testing.txt
+
+      - name: Run pytest
+        run: |
+          pytest scripts/structures_pruning

.gitignore

@@ -62,6 +62,8 @@ conf/locale/fake*/LC_MESSAGES/*.mo
 # this was a mistake in i18n_tools, now fixed.
 conf/locale/messages.mo
 conf/plugins-locale/
+conf/locale/*/LC_MESSAGES/
+/*/static/js/i18n/
 /*/static/js/xblock.v1-i18n/
 
 ### Testing artifacts

Makefile

@@ -38,56 +38,33 @@ technical-docs:  ## build the technical docs
 guides:	swagger ## build the developer guide docs
 	cd docs/guides; make clean html
 
-# (IS_OPENEDX_TRANSLATIONS_WORKFLOW) is set to "yes" in the `extract-translation-source-files` GitHub actions
-# workflow on the `openedx-translations` repository. See (extract translation source files) step here:
-# https://github.com/openedx/openedx-translations/blob/main/.github/workflows/extract-translation-source-files.yml
-# Related doc: https://docs.openedx.org/en/latest/developers/how-tos/enable-translations-new-repo.html
-ifeq ($(IS_OPENEDX_TRANSLATIONS_WORKFLOW),yes)
 extract_translations: ## extract localizable strings from sources
 	i18n_tool extract --no-segment -v
 	cd conf/locale/en/LC_MESSAGES && msgcat djangojs.po underscore.po -o djangojs.po
 	cd conf/locale/en/LC_MESSAGES && msgcat django.po wiki.po edx_proctoring_proctortrack.po mako.po -o django.po
 	cd conf/locale/en/LC_MESSAGES && rm wiki.po edx_proctoring_proctortrack.po mako.po underscore.po
-else
-extract_translations: ## extract localizable strings from sources
-	i18n_tool extract -v;
-endif
-
-push_translations: ## push source strings to Transifex for translation
-	i18n_tool transifex push
 
-pull_plugin_translations:  ## Pull translations from Transifex for edx_django_utils.plugins for both lms and cms
-	rm -rf conf/plugins-locale/plugins  # Clean up existing atlas translations
-	mkdir -p conf/plugins-locale/plugins
+pull_plugin_translations:  ## Pull translations for edx_django_utils.plugins for both lms and cms
 	python manage.py lms pull_plugin_translations --verbose $(ATLAS_OPTIONS)
 	python manage.py lms compile_plugin_translations
 
 pull_xblock_translations:  ## pull xblock translations via atlas
-	rm -rf conf/plugins-locale/xblock.v1  # Clean up existing atlas translations
-	rm -rf lms/static/i18n/xblock.v1 cms/static/i18n/xblock.v1  # Clean up existing xblock compiled translations
 	python manage.py lms pull_xblock_translations --verbose $(ATLAS_OPTIONS)
 	python manage.py lms compile_xblock_translations
 	python manage.py cms compile_xblock_translations
 
-pull_translations: ## pull translations from Transifex
+pull_translations: ## pull translations via atlas
+	# Clean up the existing translations
 	git clean -fdX conf/locale
-ifeq ($(OPENEDX_ATLAS_PULL),)
-	i18n_tool transifex pull
-	i18n_tool extract
-	i18n_tool dummy
-	i18n_tool generate --verbose 1
-	git clean -fdX conf/locale/rtl
-	git clean -fdX conf/locale/eo
-	i18n_tool validate --verbose
-else
+	rm -rf conf/plugins-locale cms/static/js/i18n/ lms/static/js/i18n/ cms/static/js/xblock.v1-i18n/ lms/static/js/xblock.v1-i18n/
 	make pull_xblock_translations
 	make pull_plugin_translations
-	find conf/locale -mindepth 1 -maxdepth 1 -type d -exec rm -r {} \;
-	atlas pull $(ATLAS_OPTIONS) translations/edx-platform/conf/locale:conf/locale
-	i18n_tool generate
-endif
-	paver i18n_compilejs
-
+	atlas pull $(ATLAS_OPTIONS) \
+	    translations/edx-platform/conf/locale:conf/locale \
+	    translations/studio-frontend/src/i18n/messages:conf/plugins-locale/studio-frontend
+	python manage.py lms compilemessages
+	python manage.py lms compilejsi18n
+	python manage.py cms compilejsi18n
 
 detect_changed_source_translations: ## check if translation files are up-to-date
 	i18n_tool changed
@@ -139,7 +116,9 @@ REQ_FILES = \
 	requirements/edx/semgrep \
 	scripts/xblock/requirements \
 	scripts/user_retirement/requirements/base \
-	scripts/user_retirement/requirements/testing
+	scripts/user_retirement/requirements/testing \
+	scripts/structures_pruning/requirements/base \
+	scripts/structures_pruning/requirements/testing
 
 define COMMON_CONSTRAINTS_TEMP_COMMENT
 # This is a temporary solution to override the real common_constraints.txt\n# In edx-lint, until the pyjwt constraint in edx-lint has been removed.\n# See BOM-2721 for more details.\n# Below is the copied and edited version of common_constraints\n

README.rst

@@ -63,7 +63,7 @@ Services:
 
 * MySQL 5.7
 
-* Mongo 4.x
+* Mongo 7.x
 
 * Memcached
 

catalog-info.yaml

@@ -11,6 +11,6 @@ metadata:
       title: "Documentation"
       icon: "Web"
 spec:
-  owner: group:arch-bom
+  owner: group:2u-arch-bom
   type: 'service'
   lifecycle: 'production'

cms/djangoapps/contentstore/api/views/course_import.py

@@ -19,6 +19,7 @@
 
 from cms.djangoapps.contentstore.storage import course_import_export_storage
 from cms.djangoapps.contentstore.tasks import CourseImportTask, import_olx
+from cms.djangoapps.contentstore.utils import IMPORTABLE_FILE_TYPES
 from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes
 
 from .utils import course_author_access_required
@@ -44,8 +45,8 @@ class CourseImportView(CourseImportExportViewMixin, GenericAPIView):
     """
     **Use Case**
 
-        * Start an asynchronous task to import a course from a .tar.gz file into
-        the specified course ID, overwriting the existing course
+        * Start an asynchronous task to import a course from a .tar.gz or .zip
+        file into the specified course ID, overwriting the existing course
         * Get a status on an asynchronous task import
 
     **Example Requests**
@@ -59,7 +60,7 @@ class CourseImportView(CourseImportExportViewMixin, GenericAPIView):
 
         * course_id: (required) A string representation of a Course ID,
                                 e.g., course-v1:edX+DemoX+Demo_Course
-        * course_data: (required) The course .tar.gz file to import
+        * course_data: (required) The course .tar.gz or .zip file to import
 
     **POST Response Values**
 
@@ -83,7 +84,7 @@ class CourseImportView(CourseImportExportViewMixin, GenericAPIView):
         A GET request must include the following parameters.
 
         * task_id: (required) The UUID of the task to check, e.g. "4b357bb3-2a1e-441d-9f6c-2210cf76606f"
-        * filename: (required) The filename of the uploaded course .tar.gz
+        * filename: (required) The filename of the uploaded course .tar.gz or .zip
 
     **GET Response Values**
 
@@ -124,7 +125,7 @@ def post(self, request, course_key):
                 )
 
             filename = request.FILES['course_data'].name
-            if not filename.endswith('.tar.gz'):
+            if not filename.endswith(IMPORTABLE_FILE_TYPES):
                 raise self.api_error(
                     status_code=status.HTTP_400_BAD_REQUEST,
                     developer_message='Parameter in the wrong format',

cms/djangoapps/contentstore/errors.py

@@ -14,5 +14,5 @@
 UNKNOWN_ERROR_IN_IMPORT = _('Unknown error while importing course.')
 UNKNOWN_ERROR_IN_UNPACKING = _('An Unknown error occurred during the unpacking step.')
 UNKNOWN_USER_ID = _('Unknown User ID: {0}')
-UNSAFE_TAR_FILE = _('Unsafe tar file. Aborting import.')
+UNSAFE_ARCHIVE_FILE = _('Unsafe archive file. Aborting import.')
 USER_PERMISSION_DENIED = _('User permission denied.')