Update CI files for branch 3.105

.ci/ansible/settings.py.j2

@@ -1,3 +1,4 @@
+SECRET_KEY = "{{ django_secret }}"
 CONTENT_ORIGIN = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}"
 ANSIBLE_API_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}"
 ANSIBLE_CONTENT_HOSTNAME = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/pulp/content"

.ci/ansible/start_container.yaml

@@ -18,6 +18,8 @@
       ansible.builtin.template:
         src: "settings.py.j2"
         dest: "settings/settings.py"
+      vars:
+        django_secret: "lookup('community.general.random_string', length=50, overwrite_all='abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)')"
 
     - name: "Setup docker networking"
       community.docker.docker_network:

CHANGES/+api_root_changes.feature

@@ -0,0 +1 @@
+Added `pulpcore.app.find_url.find_api_root()` to standardize Pulp url-creation.

CHANGES/+django_secret.bugfix

@@ -0,0 +1 @@
+Stopped shipping an insecure default as DJANGO_SECRET.

pulp_file/tests/functional/api/test_domains.py

@@ -2,8 +2,8 @@
 import uuid
 
 import pytest
+from django.conf import settings
 
-from pulpcore.app import settings
 from pulpcore.tests.functional.utils import download_file, generate_iso
 
 pytestmark = pytest.mark.skipif(not settings.DOMAIN_ENABLED, reason="Domains not enabled.")

pulp_file/tests/functional/api/test_filesystem_export.py

@@ -2,8 +2,8 @@
 import uuid
 
 import pytest
+from django.conf import settings
 
-from pulpcore.app import settings
 from pulpcore.client.pulpcore.exceptions import ApiException, BadRequestException
 from pulpcore.constants import TASK_STATES
 

pulp_file/tests/functional/api/test_pulp_export.py

@@ -3,8 +3,8 @@
 from pathlib import Path
 
 import pytest
+from django.conf import settings
 
-from pulpcore.app import settings
 from pulpcore.client.pulpcore.exceptions import ApiException, BadRequestException
 from pulpcore.constants import TASK_STATES
 

pulp_file/tests/unit/test_serializers.py

@@ -2,17 +2,13 @@
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase
 
+from pulpcore.plugin.find_url import find_api_root
 from pulpcore.plugin.models import Artifact
 
 from pulp_file.app.models import FileContent
 from pulp_file.app.serializers import FileContentSerializer
 
-V3_API_ROOT = (
-    settings.V3_API_ROOT
-    if not settings.DOMAIN_ENABLED
-    else settings.V3_DOMAIN_API_ROOT.replace("<slug:pulp_domain>", "default")
-)
-
+_, V3_API_ROOT = find_api_root(domain="default")
 
 CHECKSUM_LEN = {
     "md5": 32,

pulpcore/app/access_policy.py

@@ -1,9 +1,9 @@
 from copy import deepcopy
 
+from django.conf import settings
 from rest_access_policy import AccessPolicy
 from rest_framework.exceptions import APIException
 
-from pulpcore.app import settings
 from pulpcore.app.models import AccessPolicy as AccessPolicyModel
 from pulpcore.app.util import get_view_urlpattern, get_viewset_for_model
 

pulpcore/app/authentication.py

@@ -5,6 +5,7 @@
 from gettext import gettext as _
 
 import jq
+from django.conf import settings
 from django.contrib.auth import authenticate
 from django.contrib.auth.backends import RemoteUserBackend
 from rest_framework.authentication import (
@@ -16,8 +17,6 @@
 )
 from rest_framework.exceptions import AuthenticationFailed
 
-from pulpcore.app import settings
-
 _logger = logging.getLogger(__name__)
 
 

pulpcore/app/find_url.py

@@ -0,0 +1,49 @@
+from django.conf import settings
+
+DOMAIN_SLUG = "<slug:pulp_domain>"
+REWRITE_SLUG = "/<path:api_root>/"
+
+
+# We isolate this utility-function to avoid pulling in random "Django App must be configured"
+# code segments from the rest of the .util methods/imports
+def find_api_root(version="v3", set_domain=True, domain=None, lstrip=False, rewrite_header=True):
+    """
+    Returns the tuple (api-root, <root>/api/<version>)
+
+    Args:
+        version (str): API-version desired
+        set_domain (bool): Should the domain-slug be included if DOMAIN_ENABLED?
+        domain (str): Domain-name to replace DOMAIN_SLUG
+        lstrip (bool): Should the full version have it's leading-/ stripped
+        rewrite_header (bool): Should API_ROOT_REWRITE_HEADER be honored or not
+
+    Examples:
+        find_api_root() : ("/pulp/", "/pulp/api/v3/")
+        find_api_root(), DOMAIN_ENABLED: ("/pulp/", "/pulp/<slug:pulp_domain>/api/v3/")
+        find_api_root(domain="default"), DOMAIN_ENABLED : ("/pulp/", "/pulp/default/api/v3/")
+        find_api_root(lstrip=True) : ("pulp/", "pulp/api/v3/")
+        find_api_root(), API_ROOT_REWRITE_HEADER: ("/<path:api_root>/", "/<path:api_root>/api/v3/")
+        find_api_root(version="v4", domain="foo", lstrip=True), API_ROOT_REWRITE_HEADER :
+            ("<path:api_root>/", "<path:api_root>/default/api/v4/")
+    Returns:
+        (str, str) : (API_ROOT (possibly re-written), API_ROOT/api/<version>/
+                     (with <domain> if enabled))
+    """
+    # Some current path-building wants to ignore REWRITE - make that possible
+    if rewrite_header and settings.API_ROOT_REWRITE_HEADER:
+        api_root = REWRITE_SLUG
+    else:
+        api_root = settings.API_ROOT
+
+    # Some current path-building wants to ignore DOMAIN - make that possible
+    if set_domain and settings.DOMAIN_ENABLED:
+        if domain:
+            path = f"{api_root}{domain}/api/{version}/"
+        else:
+            path = f"{api_root}{DOMAIN_SLUG}/api/{version}/"
+    else:
+        path = f"{api_root}api/{version}/"
+    if lstrip:
+        return api_root.lstrip("/"), path.lstrip("/")
+    else:
+        return api_root, path

pulpcore/app/redis_connection.py

@@ -1,8 +1,7 @@
+from django.conf import settings
 from redis import Redis
 from redis.asyncio import Redis as aRedis
 
-from pulpcore.app.settings import settings
-
 _conn = None
 _a_conn = None
 

pulpcore/app/serializers/exporter.py

@@ -2,9 +2,10 @@
 import re
 from gettext import gettext as _
 
+from django.conf import settings
 from rest_framework import serializers
 
-from pulpcore.app import models, settings
+from pulpcore.app import models
 from pulpcore.app.serializers import (
     DetailIdentityField,
     DetailRelatedField,

pulpcore/app/serializers/importer.py

@@ -1,10 +1,11 @@
 import os
 from gettext import gettext as _
 
+from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 from rest_framework import serializers
 
-from pulpcore.app import models, settings
+from pulpcore.app import models
 from pulpcore.app.serializers import (
     DetailIdentityField,
     DomainUniqueValidator,

pulpcore/app/serializers/repository.py

@@ -2,10 +2,11 @@
 from gettext import gettext as _
 from urllib.parse import urlparse
 
+from django.conf import settings
 from rest_framework import fields, serializers
 from rest_framework_nested.serializers import NestedHyperlinkedModelSerializer
 
-from pulpcore.app import models, settings
+from pulpcore.app import models
 from pulpcore.app.serializers import (
     DetailIdentityField,
     DetailRelatedField,

pulpcore/app/settings.py

@@ -88,9 +88,6 @@
 # List of upload handler classes to be applied in order.
 FILE_UPLOAD_HANDLERS = ("pulpcore.app.files.HashingFileUploadHandler",)
 
-# SECURITY WARNING: this should be set to a unique, unpredictable value
-SECRET_KEY = "SECRET"
-
 # Key used to encrypt fields in the database
 DB_ENCRYPTION_KEY = "/etc/pulp/certs/database_fields.symmetric.key"
 
@@ -586,6 +583,9 @@ def otel_middleware_hook(settings):
     api_root = "/<path:api_root>/"
 else:
     api_root = settings.API_ROOT
+# protocol://host:port/{API_ROOT}{domain}/api/{version}/
+# All of the below are DEPRECATED, and should be replaced by calling
+# pulpcore.plugin.find_url.find_api_root() (q.v.)
 settings.set("V3_API_ROOT", api_root + "api/v3/")  # Not user configurable
 settings.set("V3_DOMAIN_API_ROOT", api_root + "<slug:pulp_domain>/api/v3/")
 settings.set("V3_API_ROOT_NO_FRONT_SLASH", settings.V3_API_ROOT.lstrip("/"))

pulpcore/app/templatetags/pulp_urls.py

@@ -1,8 +1,9 @@
-from django.conf import settings
 from django.template.defaultfilters import register, stringfilter
 from django.template.defaultfilters import urlize as orig_urlize
 from django.utils.safestring import SafeData, mark_safe
 
+from pulpcore.app.find_url import find_api_root
+
 
 @register.filter(needs_autoescape=True)
 @stringfilter
@@ -12,13 +13,19 @@ def urlize(text, autoescape=True):
 
     This filter overwrites the django default implementation to also handle pulp api hrefs.
     """
+    #    urlize() will turn strings into links of they're of the form protocol://site/path.
+    #    We force it to recognize Pulp-api-links by replacing strings in the incoming text of
+    #    the form "API_ROOT/api/v3/whatever", forcing them to look like
+    #    "http://SENTINEL.org/(string)", urlize()ing the result, then undoing the replace to
+    #    lose the http://SENTINEL.org's.
+    _, current_path = find_api_root(set_domain=False, rewrite_header=False)
     safe_input = isinstance(text, SafeData)
-    text = text.replace(settings.V3_API_ROOT, "http://SENTINEL.org" + settings.V3_API_ROOT)
+    text = text.replace(current_path, "http://SENTINEL.org" + current_path)
     if safe_input:
         text = mark_safe(text)
     text = orig_urlize(text, autoescape=autoescape)
     safe_input = isinstance(text, SafeData)
-    text = text.replace("http://SENTINEL.org" + settings.V3_API_ROOT, settings.V3_API_ROOT)
+    text = text.replace("http://SENTINEL.org" + current_path, current_path)
     if safe_input:
         text = mark_safe(text)
     return text

pulpcore/app/urls.py

@@ -27,15 +27,13 @@
     OrphansCleanupViewset,
     ReclaimSpaceViewSet,
 )
+from pulpcore.plugin.find_url import find_api_root
 
-if settings.DOMAIN_ENABLED:
-    API_ROOT = settings.V3_DOMAIN_API_ROOT_NO_FRONT_SLASH
-else:
-    API_ROOT = settings.V3_API_ROOT_NO_FRONT_SLASH
-if settings.API_ROOT_REWRITE_HEADER:
-    V3_API_ROOT = settings.V3_API_ROOT.replace("/<path:api_root>/", settings.API_ROOT)
-else:
-    V3_API_ROOT = settings.V3_API_ROOT
+_, PATH_DOMAIN_REWRITE_NOFRONT = find_api_root(lstrip=True, set_domain=True, rewrite_header=True)
+_, PATH_NODOMAIN_NOREWRITE_NOFRONT = find_api_root(
+    lstrip=True, set_domain=False, rewrite_header=False
+)
+_, PATH_NODOMAIN_REWRITE_NOFRONT = find_api_root(lstrip=True, set_domain=False, rewrite_header=True)
 
 
 class ViewSetNode:
@@ -199,7 +197,7 @@ class PulpDefaultRouter(routers.DefaultRouter):
             SpectacularRedocView.as_view(
                 authentication_classes=[],
                 permission_classes=[],
-                url=f"{V3_API_ROOT}docs/api.json?include_html=1&pk_path=1",
+                url=f"/{PATH_NODOMAIN_NOREWRITE_NOFRONT}docs/api.json?include_html=1&pk_path=1",
             )
         ),
         name="schema-redoc",
@@ -210,17 +208,18 @@ class PulpDefaultRouter(routers.DefaultRouter):
             SpectacularSwaggerView.as_view(
                 authentication_classes=[],
                 permission_classes=[],
-                url=f"{V3_API_ROOT}docs/api.json?include_html=1&pk_path=1",
+                url=f"/{PATH_NODOMAIN_NOREWRITE_NOFRONT}docs/api.json?include_html=1&pk_path=1",
             )
         ),
         name="schema-swagger",
     ),
 ]
 
 urlpatterns = [
-    path(API_ROOT, include(special_views)),
+    path(PATH_DOMAIN_REWRITE_NOFRONT, include(special_views)),
     path("auth/", include("rest_framework.urls")),
-    path(settings.V3_API_ROOT_NO_FRONT_SLASH, include(docs_and_status)),
+    # docs/status aren't "inside" a domain
+    path(PATH_NODOMAIN_REWRITE_NOFRONT, include(docs_and_status)),
 ]
 
 if settings.DOMAIN_ENABLED:
@@ -235,7 +234,7 @@ class NoSchema(p.callback.cls):
         view = NoSchema.as_view(**p.callback.initkwargs)
         name = p.name + "-domains" if p.name else None
         docs_and_status_no_schema.append(path(str(p.pattern), view, name=name))
-    urlpatterns.insert(-1, path(API_ROOT, include(docs_and_status_no_schema)))
+    urlpatterns.insert(-1, path(PATH_DOMAIN_REWRITE_NOFRONT, include(docs_and_status_no_schema)))
 
 if "social_django" in settings.INSTALLED_APPS:
     urlpatterns.append(
@@ -247,7 +246,7 @@ class NoSchema(p.callback.cls):
 
 all_routers = [root_router] + vs_tree.register_with(root_router)
 for router in all_routers:
-    urlpatterns.append(path(API_ROOT, include(router.urls)))
+    urlpatterns.append(path(PATH_DOMAIN_REWRITE_NOFRONT, include(router.urls)))
 
 # If plugins define a urls.py, include them into the root namespace.
 for plugin_pattern in plugin_patterns:

pulpcore/app/util.py

@@ -31,7 +31,6 @@
 
 # a little cache so viewset_for_model doesn't have to iterate over every app every time
 _model_viewset_cache = {}
-STRIPPED_API_ROOT = settings.API_ROOT.strip("/")
 
 
 def reverse(viewname, args=None, kwargs=None, request=None, relative_url=True, **extra):
@@ -45,7 +44,7 @@ def reverse(viewname, args=None, kwargs=None, request=None, relative_url=True, *
     if settings.DOMAIN_ENABLED:
         kwargs.setdefault("pulp_domain", get_domain().name)
     if settings.API_ROOT_REWRITE_HEADER:
-        kwargs.setdefault("api_root", getattr(request, "api_root", STRIPPED_API_ROOT))
+        kwargs.setdefault("api_root", getattr(request, "api_root", settings.API_ROOT.strip("/")))
     if relative_url:
         request = None
     return drf_reverse(viewname, args=args, kwargs=kwargs, request=request, **extra)
@@ -79,7 +78,6 @@ def get_url(model, domain=None, request=None):
         kwargs["pk"] = model.pk
     else:
         view_action = "list"
-
     return reverse(get_view_name_for_model(model, view_action), kwargs=kwargs, request=request)
 
 
@@ -301,7 +299,6 @@ def get_view_name_for_model(model_obj, view_action):
     if isinstance(model_obj, models.MasterModel):
         model_obj = model_obj.cast()
     viewset = get_viewset_for_model(model_obj)
-
     # return the complete view name, joining the registered viewset base name with
     # the requested view method.
     for router in all_routers:

pulpcore/app/views/importer.py

@@ -2,11 +2,11 @@
 import os
 from gettext import gettext as _
 
+from django.conf import settings
 from drf_spectacular.utils import extend_schema
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from pulpcore.app import settings
 from pulpcore.app.serializers import PulpImportCheckResponseSerializer, PulpImportCheckSerializer
 
 

pulpcore/cache/cache.py

@@ -5,6 +5,7 @@
 
 from aiohttp.web import FileResponse, HTTPSuccessful, Request, Response, StreamResponse
 from aiohttp.web_exceptions import HTTPFound
+from django.conf import settings
 from django.http import FileResponse as ApiFileResponse
 from django.http import HttpResponse, HttpResponseRedirect
 from redis import ConnectionError
@@ -16,7 +17,6 @@
     get_async_redis_connection,
     get_redis_connection,
 )
-from pulpcore.app.settings import settings
 from pulpcore.metrics import artifacts_size_counter
 from pulpcore.responses import ArtifactResponse