refactor: combine authoring apps to openedx_content

This commit makes the following major refactorings:

- All apps previously under openedx_learning/apps/authoring have now
  effectively been merged together to one openedx_content app.
- The models and logic for those apps continues to be encapsulated in
  what we're calling "applets", inside of openedx_content/applets/...
- In order to facilitate smooth database migrations, the AppConfigs and
  database migrations for the old apps continue to be preserved in
  openedx_content/backcompat/...
- A new openedx_learning.api.django.openedx_learning_apps_to_install()
  has been created to make it easier for openedx-platform to get all the
  necessary apps to install.

The rationale for this change is detailed in
docs/decisions/0020-authoring-as-one-app.rst, but the short version is
that we hope this arrangement will allow us to keep many of the benefits
of having small apps (easy to reason about), while also making it easier
to refactor internally. Refactoring is currently hindered by the
difficulty in moving models across apps.

The name change away from "authoring" also reflects that we intend to
use these APIs and models on the "learning" side of things as well, e.g.
when reading content for rendering to a student. In the longer term, we
will probably make openedx_content a top-level package in this repo, but
that would be a later step.

Most of this commit is just shuffling things around and renaming
references, but the truly complicated bits are around the sequencing of
database migrations, particularly the ones removing model state from the
old apps, and transferring that state into the new openedx_content app
without changing the actual database state. We also have some hacky
logic in openedx_content/apps.py in order to properly patch migration
dependencies so that we don't break existing openedx-platform
migrations.

This also bumps the version to 0.31.0.

Author: ormsbee

.annotation_safe_list.yml

@@ -19,43 +19,43 @@ auth.User:
   ".. pii_retirement": "consumer_api"
 contenttypes.ContentType:
   ".. no_pii:": "This model has no PII"
-oel_collections.Collection:
+openedx_content.Collection:
   ".. no_pii:": "This model has no PII"
-oel_collections.CollectionPublishableEntity:
+openedx_content.CollectionPublishableEntity:
   ".. no_pii:": "This model has no PII"
-oel_components.Component:
+openedx_content.Component:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentType:
+openedx_content.ComponentType:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentVersion:
+openedx_content.ComponentVersion:
   ".. no_pii:": "This model has no PII"
-oel_components.ComponentVersionContent:
+openedx_content.ComponentVersionContent:
   ".. no_pii:": "This model has no PII"
-oel_contents.Content:
+openedx_content.Content:
   ".. no_pii:": "This model has no PII"
-oel_contents.MediaType:
+openedx_content.MediaType:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Container:
+openedx_content.Container:
   ".. no_pii:": "This model has no PII"
-oel_publishing.ContainerVersion:
+openedx_content.ContainerVersion:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Draft:
+openedx_content.Draft:
   ".. no_pii:": "This model has no PII"
-oel_publishing.EntityList:
+openedx_content.EntityList:
   ".. no_pii:": "This model has no PII"
-oel_publishing.EntityListRow:
+openedx_content.EntityListRow:
   ".. no_pii:": "This model has no PII"
-oel_publishing.LearningPackage:
+openedx_content.LearningPackage:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishLog:
+openedx_content.PublishLog:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishLogRecord:
+openedx_content.PublishLogRecord:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishableEntity:
+openedx_content.PublishableEntity:
   ".. no_pii:": "This model has no PII"
-oel_publishing.PublishableEntityVersion:
+openedx_content.PublishableEntityVersion:
   ".. no_pii:": "This model has no PII"
-oel_publishing.Published:
+openedx_content.Published:
   ".. no_pii:": "This model has no PII"
 oel_tagging.ObjectTag:
   ".. no_pii:": "This model has no PII"
@@ -65,17 +65,17 @@ oel_tagging.TagImportTask:
   ".. no_pii:": "This model has no PII"
 oel_tagging.Taxonomy:
   ".. no_pii:": "This model has no PII"
-oel_sections.Section:
+openedx_content.Section:
   ".. no_pii:": "This model has no PII"
-oel_sections.SectionVersion:
+openedx_content.SectionVersion:
   ".. no_pii:": "This model has no PII"
-oel_subsections.Subsection:
+openedx_content.Subsection:
   ".. no_pii:": "This model has no PII"
-oel_subsections.SubsectionVersion:
+openedx_content.SubsectionVersion:
   ".. no_pii:": "This model has no PII"
-oel_units.Unit:
+openedx_content.Unit:
   ".. no_pii:": "This model has no PII"
-oel_units.UnitVersion:
+openedx_content.UnitVersion:
   ".. no_pii:": "This model has no PII"
 social_django.Association:
   ".. no_pii:": "This model has no PII"

.gitignore

@@ -75,7 +75,12 @@ venv/
 !.vscode/settings.json.example
 
 # Media files (for uploads)
-media/
+/media/
 
 # Media files generated during test runs
-test_media/
+/test_media/
+
+# uv stuff
+.lock
+CACHEDIR.TAG
+pyvenv.cfg

.importlinter

@@ -36,24 +36,24 @@ layers=
     openedx_learning.api.authoring
 
     # The "backup_restore" app handle the new export and import mechanism.
-    openedx_learning.apps.authoring.backup_restore
+    openedx_learning.apps.openedx_content.applets.backup_restore
 
     # The "components" app is responsible for storing versioned Components,
     # which is Open edX Studio terminology maps to things like individual
     # Problems, Videos, and blocks of HTML text. This is also the type we would
     # associate with a single "leaf" XBlock–one that is not a container type and
     # has no child elements.
-    openedx_learning.apps.authoring.components
+    openedx_learning.apps.openedx_content.applets.components
 
     # The "contents" app stores the simplest pieces of binary and text data,
     # without versioning information. These belong to a single Learning Package.
-    openedx_learning.apps.authoring.contents
+    openedx_learning.apps.openedx_content.applets.contents
 
     # The "collections" app stores arbitrary groupings of PublishableEntities.
     # Its only dependency should be the publishing app.
-    openedx_learning.apps.authoring.collections
+    openedx_learning.apps.openedx_content.applets.collections
 
     # The lowest layer is "publishing", which holds the basic primitives needed
     # to create Learning Packages and manage the draft and publish states for
     # various types of content.
-    openedx_learning.apps.authoring.publishing
+    openedx_learning.apps.openedx_content.applets.publishing

docs/decisions/0016-python-public-api-conventions.rst

@@ -17,10 +17,10 @@ Decision
 In addition to OEP-49, we will adopt the following practices:
 
 Learning Core Django apps will be grouped into packages.
-  Apps in ``openedx_learning`` will be grouped into broadly related packages under ``openedx_learning.apps``. The first of these groups will be "authoring" (``openedx_learning.apps.authoring``). Future packages may include "learner", "personalization", "activity", "grading", etc.
+  Apps in ``openedx_learning`` will be grouped into broadly related packages under ``openedx_learning.apps``. The first of these groups will be "authoring" (``openedx_learning.apps.openedx_content``). Future packages may include "learner", "personalization", "activity", "grading", etc.
 
 Learning Core Django apps will continue to have their own ``api`` modules.
-  So for example, ``openedx_learning.apps.authoring.components.api`` will continue to exist.
+  So for example, ``openedx_learning.apps.openedx_content.applets.components.api`` will continue to exist.
 
 Learning Core will have a top level package for its public API.
   All public APIs intended for use by consumers of Learning Core will be represented as modules in the ``openedx_learning.api`` package that corresponds to the app groupings (e.g. ``openedx_learning.api.authoring``).
@@ -35,14 +35,14 @@ App ``api`` modules will define their public functions using ``__all__``.
   This relies on the individual apps to properly set ``__all__`` to the list of functions that they are willing to publicly support.
 
 App ``api`` modules within a package of apps still import from each other.
-  So for example, ``openedx_learning.apps.authoring.components.api`` will continue to import APIs that it needs from ``..publishing.api``, instead of using the public API at ``openedx_learning.api.authoring``. These imports should not use wildcards.
+  So for example, ``openedx_learning.apps.openedx_content.applets.components.api`` will continue to import APIs that it needs from ``..publishing.api``, instead of using the public API at ``openedx_learning.api.authoring``. These imports should not use wildcards.
 
   Functions and constants that are not listed as part of a module's ``__all__`` may still be imported by other app APIs in the same package grouping. This should allow a package more flexibility to create provisional APIs that we may not want to support publicly.
 
   If a function or attribute is intended to be completely private to an app's ``api`` module (i.e. not used even by other apps in its package), it should be prefixed with an underscore.
 
 App ``api`` modules should not import directly from apps outside their package.
-  For example, ``openedx_learning.apps.personalization.api`` should import authoring API functions from ``openedx_learning.api.authoring``, **not** directly from something like ``openedx_learning.apps.authoring.components.api``. This will help to limit the impact of refactoring app package internal changes, as well as exposing shortcomings in the existing public APIs.
+  For example, ``openedx_learning.apps.personalization.api`` should import authoring API functions from ``openedx_learning.api.authoring``, **not** directly from something like ``openedx_learning.apps.openedx_content.applets.components.api``. This will help to limit the impact of refactoring app package internal changes, as well as exposing shortcomings in the existing public APIs.
 
 Public API modules may implement their own functions.
   In addition to aggregating app ``api`` modules via wildcard imports, public API modules like ``openedx_learning.api.authoring`` may implement their own functionality. This will be useful for convenience functions that invoke multiple app APIs, and for backwards compatibility shims. When possible, the bulk of the logic for these should continue to live in app-defined APIs, with the public API module acting more as a glue layer.
@@ -54,7 +54,7 @@ Rejected Alternatives
 
 Public APIs in each app package
 
-  We could have added these aggregations as ``api`` modules in each app group package, e.g. ``openedx_learning.apps.authoring.api``. We are not doing this because:
+  We could have added these aggregations as ``api`` modules in each app group package, e.g. ``openedx_learning.apps.openedx_content.api``. We are not doing this because:
 
   * It's more convenient for browsing and documentation generation to have the public API modules in the same package.
   * It's more idiomatic for Python libraries to expose their APIs in appropriately named modules (like ``authoring``), rather than all imported modules being named ``api``.

docs/decisions/0020-authoring-as-one-app.rst

@@ -0,0 +1,66 @@
+20. openedx_content as an Umbrella App of Smaller Applets
+=========================================================
+
+Context
+-------
+
+Up to this point, Learning Core has used many small apps with a narrow focus (e.g. ``components``, ``collections``, etc.) in order to make each individual app simpler to reason about. This has been useful overall, but it has made refactoring more cumbersome. For instance:
+
+#. Moving models between apps is tricky, requiring the use of Django's ``SeparateDatabaseAndState`` functionality to fake a deletion in one app and a creation in another without actually altering the database. It also requires doctoring the migration files for models in other repos that might have foreign key relations to the model being moved, so that they're pointing to the new ``app_label``.  This will be an issue when we try to extract container-related models and logic out of publishing and into a new ``containers`` app.
+#. Renaming an app is also cumbersome, because the process requires creating a new app and transitioning the models over. This came up when trying to rename the ``contents`` app to ``media``.
+
+There have also been minor inconveniences, like having a long list of ``INSTALLED_APPS`` to maintain in edx-platform over time, or not having these tables easily grouped together in the Django admin interface.
+
+Decisions
+---------
+
+1. Single openedx_content App
+~~~~~~~~~~~~~~~~~~~~~~~
+
+All existing authoring apps will be merged into one Django app (``openedx_learning.app.openedx_content``). Some consequences of this decision:
+
+- The tables will be renamed to have the ``openedx_content`` label prefix.
+- All management commands will be moved to the ``openedx_content`` app.
+
+2. Logical Separation via Applets
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We will continue to keep internal API boundaries between individual applets, and use the ``api.py`` modules. This is both to insulate applets from implementation changes in other applets, as well as to provide a set of APIs that third-party plugins can utilize. As before, we will use Import Linter to enforce dependency ordering.
+
+3. Restructuring Specifics
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In one pull request, we are going to:
+
+#. Rename the ``openedx_learning.apps.authoring`` package to be ``openedx_learning.apps.openedx_content``.
+#. Create bare shells of the existing ``authoring`` apps (``backup_restore``, ``collections``, ``components``, ``contents``, ``publishing``, ``sections``, ``subsections``, ``units``), and move them to the ``openedx_learning.apps.openedx_content.backcompat`` package. These shells will have an ``apps.py`` file and the ``migrations`` package for each existing app. This will allow for a smooth schema migration to transition the models from these individual apps to ``openedx_content``.
+#. Move the actual models files and API logic for our existing authoring apps to the ``openedx_learning.apps.openedx_content.applets`` package.
+#. Convert the top level ``openedx_learning.apps.openedx_content`` package to be a Django app. The top level ``admin.py``, ``api.py``, and ``models.py`` modules will do wildcard imports from the corresponding modules across all applet packages.
+
+In terms of model migrations, all existing apps will have a final migration that uses ``SeparateDatabaseAndState`` to remove all model state, but make no actual database changes. The initial ``openedx_content`` app migration will then also use ``SeparateDatabaseAndState`` to create the model state without doing any actual database operations. The next ``openedx_content`` app migration will rename all existing database tables to use the ``openedx_content`` prefix, for uniformity.
+
+The ordering of these migrations is important, and existing edx-platform migrations should remain unchanged. This is important to make sure that we do not introduce ordering inconsistencies for existing installations that are upgrading.
+
+Therefore, the migrations will happen in the following order:
+
+#. All ``backcompat.*`` apps migrations except for the final ones that delete model state. This takes us up to where migrations would already be before we make any changes.
+#. The ``openedx_content`` app's ``0001_intial`` migration that adds model state without changing the database. At this point, model state exists for the same models in all the old ``backcompat.*`` apps as well as the new ``openedx_content`` app.
+#. edx-platform apps that had foreign keys to old ``backcompat.*`` apps models will need to be switched to point to the new ``openedx_content`` app models. This will likewise be done without a database change, because they're still pointing to the same tables and columns.
+#. Now that edx-platform references have been updated, we can delete the model state from the old ``backcompat.*`` apps and rename the underlying tables (in either order).
+
+The tricky part is to make sure that the old ``backcompat.*`` apps models still exist when the edx-platform migrations to move over the references runs. This is problematic because the edx-platform migrations can only specify that they run *after the new openedx_content models are created*. They cannot specify that they run *before the old backcompat models are dropped*.
+
+So in order to enforce this ordering, we do the following:
+
+* The ``openedx_content`` migration ``0001_initial`` requires that all ``backcompat.*`` migrations except the last ones removing model state are run.
+* The ``openedx_content`` migration ``0002_rename_tables_to_openedx_content`` migration requires that the edx-platform migrations changing refrences over run. This is important anyway, because we want to make sure those reference changes happen before we change any table names.
+* The final ``backcompat.*`` migrations that remove model field state will list ``openedx_content`` app's ``0002_rename_tables_to_openedx_content`` as a dependency.
+
+A further complication is that ``openedx_learning`` will often run its migrations without edx-platform present (e.g. for CI or standalone dev purposes), so we can't force ``0002_rename_tables_to_openedx_content`` in the ``openedx_content`` app to have references to edx-platform migrations. To get around this, we dynamically inject those migration dependencies only if we detect those edx-platform apps exist in the currently loaded Django project. This injection happens in the ``apps.py`` initialization for the ``openedx_content`` app.
+
+The final complication is that we want these migration dependencies to be the same regardless of whether you're running edx-platform migrations with the LMS or CMS (Studio) settings, or we run the risk of getting into an inconsistent state and dropping the old models before all the edx-platform apps can run their migrations to move their references. To do this, we have to make sure that the edx-platform apps that reference Learning Core models are present in the ``INSTALLED_APPS`` for both configurations.
+
+4. The Bigger Picture
+~~~~~~~~~~~~~~~~~~~~~
+
+This practice means that the ``openedx_content`` Django app corresponds to a Subdomain in Domain Driven Design terminology, with each applet being a Bounded Context. We call these "Applets" instead of "Bounded Contexts" because we don't want it to get confused for Django's notion of Contexts and Context Processors (or Python's notion of Context Managers).

olx_importer/management/commands/load_components.py

@@ -28,9 +28,9 @@
 from django.db import transaction
 
 # Model references to remove
-from openedx_learning.apps.authoring.components import api as components_api
-from openedx_learning.apps.authoring.contents import api as contents_api
-from openedx_learning.apps.authoring.publishing import api as publishing_api
+from openedx_learning.apps.openedx_content.applets.components import api as components_api
+from openedx_learning.apps.openedx_content.applets.contents import api as contents_api
+from openedx_learning.apps.openedx_content.applets.publishing import api as publishing_api
 
 SUPPORTED_TYPES = ["problem", "video", "html"]
 logger = logging.getLogger(__name__)

openedx_learning/__init__.py

@@ -2,4 +2,4 @@
 Open edX Learning ("Learning Core").
 """
 
-__version__ = "0.30.2"
+__version__ = "0.31.0"

openedx_learning/api/authoring.py

@@ -2,21 +2,14 @@
 This is the public API for content authoring in Learning Core.
 
 This is the single ``api`` module that code outside of the
-``openedx_learning.apps.authoring.*`` package should import from. It will
+``openedx_learning.apps.openedx_content.*`` package should import from. It will
 re-export the public functions from all api.py modules of all authoring apps. It
 may also implement its own convenience APIs that wrap calls to multiple app
 APIs.
 """
 # These wildcard imports are okay because these api modules declare __all__.
 # pylint: disable=wildcard-import
-from ..apps.authoring.backup_restore.api import *
-from ..apps.authoring.collections.api import *
-from ..apps.authoring.components.api import *
-from ..apps.authoring.contents.api import *
-from ..apps.authoring.publishing.api import *
-from ..apps.authoring.sections.api import *
-from ..apps.authoring.subsections.api import *
-from ..apps.authoring.units.api import *
+from ..apps.openedx_content.api import *
 
 # This was renamed after the authoring API refactoring pushed this and other
 # app APIs into the openedx_learning.api.authoring module. Here I'm aliasing to

openedx_learning/api/authoring_models.py

@@ -7,10 +7,10 @@
 """
 # These wildcard imports are okay because these modules declare __all__.
 # pylint: disable=wildcard-import
-from ..apps.authoring.collections.models import *
-from ..apps.authoring.components.models import *
-from ..apps.authoring.contents.models import *
-from ..apps.authoring.publishing.models import *
-from ..apps.authoring.sections.models import *
-from ..apps.authoring.subsections.models import *
-from ..apps.authoring.units.models import *
+from ..apps.openedx_content.applets.collections.models import *
+from ..apps.openedx_content.applets.components.models import *
+from ..apps.openedx_content.applets.contents.models import *
+from ..apps.openedx_content.applets.publishing.models import *
+from ..apps.openedx_content.applets.sections.models import *
+from ..apps.openedx_content.applets.subsections.models import *
+from ..apps.openedx_content.applets.units.models import *