feat: collections support for containers

Author: rpenido

openedx/core/djangoapps/content/search/api.py

@@ -17,35 +17,37 @@
 from meilisearch import Client as MeilisearchClient
 from meilisearch.errors import MeilisearchApiError, MeilisearchError
 from meilisearch.models.task import TaskInfo
-from opaque_keys.edx.keys import UsageKey, OpaqueKey
-from opaque_keys.edx.locator import LibraryContainerLocator, LibraryLocatorV2, LibraryCollectionLocator
+from opaque_keys import OpaqueKey
+from opaque_keys.edx.keys import UsageKey
+from opaque_keys.edx.locator import LibraryCollectionLocator, LibraryContainerLocator, LibraryLocatorV2
 from openedx_learning.api import authoring as authoring_api
-from common.djangoapps.student.roles import GlobalStaff
 from rest_framework.request import Request
+
 from common.djangoapps.student.role_helpers import get_course_roles
+from common.djangoapps.student.roles import GlobalStaff
 from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
-from openedx.core.djangoapps.content.search.models import get_access_ids_for_request, IncrementalIndexCompleted
 from openedx.core.djangoapps.content.search.index_config import (
     INDEX_DISTINCT_ATTRIBUTE,
     INDEX_FILTERABLE_ATTRIBUTES,
-    INDEX_SEARCHABLE_ATTRIBUTES,
-    INDEX_SORTABLE_ATTRIBUTES,
     INDEX_RANKING_RULES,
+    INDEX_SEARCHABLE_ATTRIBUTES,
+    INDEX_SORTABLE_ATTRIBUTES
 )
+from openedx.core.djangoapps.content.search.models import IncrementalIndexCompleted, get_access_ids_for_request
 from openedx.core.djangoapps.content_libraries import api as lib_api
 from xmodule.modulestore.django import modulestore
 
 from .documents import (
     Fields,
     meili_id_from_opaque_key,
-    searchable_doc_for_course_block,
+    searchable_doc_collections,
     searchable_doc_for_collection,
     searchable_doc_for_container,
+    searchable_doc_for_course_block,
     searchable_doc_for_library_block,
     searchable_doc_for_key,
-    searchable_doc_collections,
     searchable_doc_tags,
-    searchable_doc_tags_for_collection,
+    searchable_doc_tags_for_collection
 )
 
 log = logging.getLogger(__name__)
@@ -487,6 +489,7 @@ def index_container_batch(batch, num_done, library_key) -> int:
                     )
                     doc = searchable_doc_for_container(container_key)
                     doc.update(searchable_doc_tags(container_key))
+                    doc.update(searchable_doc_collections(container_key))
                     docs.append(doc)
                 except Exception as err:  # pylint: disable=broad-except
                     status_cb(f"Error indexing container {container.key}: {err}")
@@ -709,15 +712,15 @@ def upsert_library_collection_index_doc(library_key: LibraryLocatorV2, collectio
     If the Collection is not found or disabled (i.e. soft-deleted), then delete it from the search index.
     """
     doc = searchable_doc_for_collection(library_key, collection_key)
-    update_components = False
+    update_items = False
 
     # Soft-deleted/disabled collections are removed from the index
     # and their components updated.
     if doc.get('_disabled'):
 
         _delete_index_doc(doc[Fields.id])
 
-        update_components = True
+        update_items = True
 
     # Hard-deleted collections are also deleted from the index,
     # but their components are automatically updated as part of the deletion process, so we don't have to.
@@ -735,10 +738,12 @@ def upsert_library_collection_index_doc(library_key: LibraryLocatorV2, collectio
         _update_index_docs([doc])
 
     # Asynchronously update the collection's components "collections" field
-    if update_components:
-        from .tasks import update_library_components_collections as update_task
+    if update_items:
+        from .tasks import update_library_components_collections as update_components_task
+        from .tasks import update_library_containers_collections as update_containers_task
 
-        update_task.delay(str(library_key), collection_key)
+        update_components_task.delay(str(library_key), collection_key)
+        update_containers_task.delay(str(library_key), collection_key)
 
 
 def update_library_components_collections(
@@ -773,6 +778,38 @@ def update_library_components_collections(
         _update_index_docs(docs)
 
 
+def update_library_containers_collections(
+    library_key: LibraryLocatorV2,
+    collection_key: str,
+    batch_size: int = 1000,
+) -> None:
+    """
+    Updates the "collections" field for all containers associated with a given Library Collection.
+
+    Because there may be a lot of containers, we send these updates to Meilisearch in batches.
+    """
+    library = lib_api.get_library(library_key)
+    containers = authoring_api.get_collection_containers(library.learning_package_id, collection_key)
+
+    paginator = Paginator(containers, batch_size)
+    for page in paginator.page_range:
+        docs = []
+
+        for container in paginator.page(page).object_list:
+            container_key = lib_api.library_container_locator(
+                library_key,
+                container,
+            )
+            doc = searchable_doc_collections(container_key)
+            docs.append(doc)
+
+        log.info(
+            f"Updating document.collections for library {library_key} containers"
+            f" page {page} / {paginator.num_pages}"
+        )
+        _update_index_docs(docs)
+
+
 def upsert_library_container_index_doc(container_key: LibraryContainerLocator) -> None:
     """
     Creates, updates, or deletes the document for the given Library Container in the search index.
@@ -819,12 +856,12 @@ def upsert_content_object_tags_index_doc(key: OpaqueKey):
     _update_index_docs([doc])
 
 
-def upsert_block_collections_index_docs(usage_key: UsageKey):
+def upsert_item_collections_index_docs(opaque_key: OpaqueKey):
     """
-    Updates the collections data in documents for the given Course/Library block
+    Updates the collections data in documents for the given Course/Library block, or Container
     """
-    doc = {Fields.id: meili_id_from_opaque_key(usage_key)}
-    doc.update(searchable_doc_collections(usage_key))
+    doc = {Fields.id: meili_id_from_opaque_key(opaque_key)}
+    doc.update(searchable_doc_collections(opaque_key))
     _update_index_docs([doc])
 
 

openedx/core/djangoapps/content/search/documents.py

@@ -309,7 +309,7 @@ def _tags_for_content_object(object_id: OpaqueKey) -> dict:
     return {Fields.tags: result}
 
 
-def _collections_for_content_object(object_id: UsageKey | LearningContextKey) -> dict:
+def _collections_for_content_object(object_id: OpaqueKey) -> dict:
     """
     Given an XBlock, course, library, etc., get the collections for its index doc.
 
@@ -340,11 +340,21 @@ def _collections_for_content_object(object_id: UsageKey | LearningContextKey) ->
     # Gather the collections associated with this object
     collections = None
     try:
-        component = lib_api.get_component_from_usage_key(object_id)
-        collections = authoring_api.get_entity_collections(
-            component.learning_package_id,
-            component.key,
-        )
+        if isinstance(object_id, UsageKey):
+            component = lib_api.get_component_from_usage_key(object_id)
+            collections = authoring_api.get_entity_collections(
+                component.learning_package_id,
+                component.key,
+            )
+        elif isinstance(object_id, LibraryContainerLocator):
+            container = lib_api.get_container_from_key(object_id)
+            collections = authoring_api.get_entity_collections(
+                container.publishable_entity.learning_package_id,
+                container.key,
+            )
+        else:
+            return result
+
     except ObjectDoesNotExist:
         log.warning(f"No component found for {object_id}")
 
@@ -438,13 +448,13 @@ def searchable_doc_tags(key: OpaqueKey) -> dict:
     return doc
 
 
-def searchable_doc_collections(usage_key: UsageKey) -> dict:
+def searchable_doc_collections(opaque_key: OpaqueKey) -> dict:
     """
     Generate a dictionary document suitable for ingestion into a search engine
     like Meilisearch or Elasticsearch, with the collections data for the given content object.
     """
-    doc = searchable_doc_for_key(usage_key)
-    doc.update(_collections_for_content_object(usage_key))
+    doc = searchable_doc_for_key(opaque_key)
+    doc.update(_collections_for_content_object(opaque_key))
 
     return doc
 

openedx/core/djangoapps/content/search/handlers.py

@@ -40,9 +40,9 @@
 
 from .api import (
     only_if_meilisearch_enabled,
-    upsert_block_collections_index_docs,
     upsert_content_object_tags_index_doc,
     upsert_collection_tags_index_docs,
+    upsert_item_collections_index_docs,
 )
 from .tasks import (
     delete_library_block_index_doc,
@@ -213,15 +213,15 @@ def content_object_associations_changed_handler(**kwargs) -> None:
 
     try:
         # Check if valid course or library block
-        usage_key = UsageKey.from_string(str(content_object.object_id))
+        opaque_key = UsageKey.from_string(str(content_object.object_id))
     except InvalidKeyError:
         try:
             # Check if valid library collection
-            usage_key = LibraryCollectionLocator.from_string(str(content_object.object_id))
+            opaque_key = LibraryCollectionLocator.from_string(str(content_object.object_id))
         except InvalidKeyError:
             try:
                 # Check if valid library container
-                usage_key = LibraryContainerLocator.from_string(str(content_object.object_id))
+                opaque_key = LibraryContainerLocator.from_string(str(content_object.object_id))
             except InvalidKeyError:
                 # Invalid content object id
                 log.error("Received invalid content object id")
@@ -230,12 +230,12 @@ def content_object_associations_changed_handler(**kwargs) -> None:
     # This event's changes may contain both "tags" and "collections", but this will happen rarely, if ever.
     # So we allow a potential double "upsert" here.
     if not content_object.changes or "tags" in content_object.changes:
-        if isinstance(usage_key, LibraryCollectionLocator):
-            upsert_collection_tags_index_docs(usage_key)
+        if isinstance(opaque_key, LibraryCollectionLocator):
+            upsert_collection_tags_index_docs(opaque_key)
         else:
-            upsert_content_object_tags_index_doc(usage_key)
+            upsert_content_object_tags_index_doc(opaque_key)
     if not content_object.changes or "collections" in content_object.changes:
-        upsert_block_collections_index_docs(usage_key)
+        upsert_item_collections_index_docs(opaque_key)
 
 
 @receiver(LIBRARY_CONTAINER_CREATED)

openedx/core/djangoapps/content/search/tasks.py

@@ -116,6 +116,19 @@ def update_library_components_collections(library_key_str: str, collection_key:
     api.update_library_components_collections(library_key, collection_key)
 
 
+@shared_task(base=LoggedTask, autoretry_for=(MeilisearchError, ConnectionError))
+@set_code_owner_attribute
+def update_library_containers_collections(library_key_str: str, collection_key: str) -> None:
+    """
+    Celery task to update the "collections" field for containers in the given content library collection.
+    """
+    library_key = LibraryLocatorV2.from_string(library_key_str)
+
+    log.info("Updating document.collections for library %s collection %s containers", library_key, collection_key)
+
+    api.update_library_containers_collections(library_key, collection_key)
+
+
 @shared_task(base=LoggedTask, autoretry_for=(MeilisearchError, ConnectionError))
 @set_code_owner_attribute
 def update_library_container_index_doc(library_key_str: str, container_key_str: str) -> None: