Update CI files

Author: pulpbot

.ci/assets/ci_constraints.txt

@@ -1,5 +1,6 @@
 # Pulpcore versions without the openapi command do no longer work in the CI
-pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*
+# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70
+pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.*
 
 
 tablib!=3.6.0
@@ -8,3 +9,11 @@ tablib!=3.6.0
 
 multidict!=6.3.0
 # This release failed the lower bounds test for some case sensitivity in CIMultiDict.
+
+
+azure-storage-blob!=12.28.*
+# Apparently does not work with current azurite.
+
+
+pycares<5
+# older aiodns versions don't pin pycares UB, and are broken by pycares>=5

.ci/scripts/check_release.py

@@ -1,11 +1,22 @@
 #!/usr/bin/env python
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "gitpython>=3.1.46,<3.2.0",
+#     "packaging>=26.0,<26.1",
+#     "pyyaml>=6.0.3,<6.1.0",
+# ]
+# ///
 
 import argparse
 import re
 import os
+import sys
 import tomllib
-import yaml
+import typing as t
 from pathlib import Path
+
+import yaml
 from packaging.version import Version
 from git import Repo
 
@@ -14,7 +25,7 @@
 Z_CHANGELOG_EXTS = [".bugfix", ".misc"]
 
 
-def options():
+def options() -> argparse.Namespace:
     """Check which branches need a release."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -33,13 +44,13 @@ def options():
     return parser.parse_args()
 
 
-def template_config():
+def template_config() -> dict[str, t.Any]:
     # Assume this script lies in .ci/scripts
     path = Path(__file__).absolute().parent.parent.parent / "template_config.yml"
     return yaml.safe_load(path.read_text())
 
 
-def current_version(repo, commitish):
+def current_version(repo: Repo, commitish: str) -> Version:
     try:
         pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml"))
         try:
@@ -53,7 +64,7 @@ def current_version(repo, commitish):
     return Version(current_version)
 
 
-def check_pyproject_dependencies(repo, from_commit, to_commit):
+def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]:
     try:
         new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml"))
         try:
@@ -74,8 +85,8 @@ def check_pyproject_dependencies(repo, from_commit, to_commit):
         return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."]
 
 
-def main(options, template_config):
-    DEFAULT_BRANCH = template_config["plugin_default_branch"]
+def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int:
+    DEFAULT_BRANCH: str = template_config["plugin_default_branch"]
 
     repo = Repo()
 
@@ -88,7 +99,7 @@ def main(options, template_config):
 
     # Warning: This will not work if branch names contain "/" but we don't really care here.
     heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")]
-    available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
+    available_branches = [h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)]
     available_branches.sort(key=lambda ver: Version(ver))
     available_branches.append(DEFAULT_BRANCH)
 
@@ -105,7 +116,10 @@ def main(options, template_config):
 
     if diff := branches - set(available_branches):
         print(f"Supplied branches contains non-existent branches! {diff}")
-        exit(1)
+        return 1
+
+    branches = [branch for branch in available_branches if branch in branches]
+    branches.reverse()
 
     print(f"Checking for releases on branches: {branches}")
 
@@ -170,6 +184,8 @@ def main(options, template_config):
     if len(releases) == 0:
         print("No new releases to perform.")
 
+    return 0
+
 
 if __name__ == "__main__":
-    main(options(), template_config())
+    sys.exit(main(options(), template_config()))

.ci/scripts/clean_gh_release_notes.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# This script is running with elevated privileges from the main branch against pull requests.
+#
+# It cleans the input from artifacts which are used by the pulp documentation internally,
+# but clutter for GitHub releases
+
+import sys
+
+NOTE = """
+> [!NOTE]
+> Official changes are available on [Pulp docs]({docs_url})\
+"""
+
+
+def main():
+    plugin_name = sys.argv[1]
+    version_str = sys.argv[2]
+    docs_url = f"https://pulpproject.org/{plugin_name}/changes/#{version_str}"
+    note_added = False
+    for line in sys.stdin:
+        if line.endswith("\n"):
+            line = line[:-1]
+        if line.startswith("#"):
+            print(line.split(" {: #")[0])
+            if not note_added and version_str in line:
+                print(NOTE.format(docs_url=docs_url))
+                note_added = True
+        else:
+            print(line)
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/collect_changes.py

@@ -1,4 +1,12 @@
 #!/bin/env python3
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "gitpython>=3.1.46,<3.2.0",
+#     "packaging>=26.0,<26.1",
+# ]
+# ///
+
 # WARNING: DO NOT EDIT!
 #
 # This file was generated by plugin_template, and is managed by it. Please use

.github/pull_request_template.md

@@ -0,0 +1,15 @@
+<!---
+Thank you for submitting a PR to the Pulp Project!
+
+If this is your first time contributing, please read the Pull Request Walkthrough documentation
+(https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/).
+-->
+
+### 📜 Checklist
+
+- [ ] Commits are cleanly separated with meaningful messages (simple features and bug fixes should be [squashed](https://pulpproject.org/pulpcore/docs/dev/guides/git/#rebasing-and-squashing) to one commit)
+- [ ] A [changelog entry](https://pulpproject.org/pulpcore/docs/dev/guides/git/#changelog-update) or entries has been added for any significant changes
+- [ ] Follows the [Pulp policy on AI Usage](https://pulpproject.org/help/more/governance/ai_policy/)
+- [ ] (For new features) - User documentation and test coverage has been added
+
+See: [Pull Request Walkthrough](https://pulpproject.org/pulpcore/docs/dev/guides/pull-request-walkthrough/)

.github/workflows/ci.yml

@@ -94,13 +94,13 @@ jobs:
       run_docs: ${{ needs.check-changes.outputs.run_docs }}
 
   lint:
-    needs:
-      - "check-changes"
-    if: needs.check-changes.outputs.run_tests == '1'
     uses: "./.github/workflows/lint.yml"
 
   build:
-    needs: "lint"
+    needs:
+      - "check-changes"
+      - "lint"
+    if: needs.check-changes.outputs.run_tests == '1'
     uses: "./.github/workflows/build.yml"
 
   test:

.github/workflows/nightly.yml

@@ -61,11 +61,20 @@ jobs:
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v6
+        id: "create_pr_changelog"
         with:
           token: ${{ secrets.RELEASE_TOKEN }}
           title: "Update Changelog"
           body: ""
           branch: "changelog/update"
           delete-branch: true
           path: "pulp_container"
+      - name: "Mark PR automerge"
+        working-directory: "pulp_container"
+        run: |
+          gh pr merge --rebase --auto "${{ steps.create_pr_changelog.outputs.pull-request-number }}"
+        if: "steps.create_pr_changelog.outputs.pull-request-number"
+        env:
+          GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}"
+        continue-on-error: true
 ...

.github/workflows/publish.yml

@@ -136,7 +136,7 @@ jobs:
         run: |
           # The last commit before the release commit contains the release CHANGES fragments
           git checkout "${TAG_NAME}~"
-          NOTES=$(towncrier build --draft --version $TAG_NAME)
+          NOTES=$(towncrier build --draft --version $TAG_NAME | .ci/scripts/clean_gh_release_notes.py pulp_container $TAG_NAME)
           echo "body<<EOF" >> $GITHUB_OUTPUT
           echo "$NOTES" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT

.github/workflows/scripts/before_install.sh

@@ -7,66 +7,124 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+# This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file.
+#
+#   It requires the following environment:
+#     TEST - The name of the scenario to prepare.
+#
+#   It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario.
+
+set -eu -o pipefail
+
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 
-set -mveuo pipefail
+if [ -f .github/workflows/scripts/pre_before_install.sh ]; then
+  source .github/workflows/scripts/pre_before_install.sh
+fi
 
-if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ]
-then
-  BRANCH_BUILD=0
-else
-  BRANCH_BUILD=1
-  BRANCH="${GITHUB_REF##refs/heads/}"
+COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
+COMPONENT_SOURCE="./pulp_container/dist/pulp_container-${COMPONENT_VERSION}-py3-none-any.whl"
+if [ "$TEST" = "s3" ]; then
+  COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3]"
 fi
-if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ]
-then
-  TAG_BUILD=0
-else
-  TAG_BUILD=1
-  BRANCH="${GITHUB_REF##refs/tags/}"
+if [ "$TEST" = "azure" ]; then
+  COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]"
 fi
 
-COMMIT_MSG=$(git log --format=%B --no-merges -1)
-export COMMIT_MSG
+if [[ "$TEST" = "pulp" ]]; then
+  python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt
+fi
+if [[ "$TEST" = "lowerbounds" ]]; then
+  python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt
+fi
+export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/")
 
-COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py)
+echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
 
-mkdir .ci/ansible/vars || true
-echo "---" > .ci/ansible/vars/main.yaml
-echo "legacy_component_name: pulp_container" >> .ci/ansible/vars/main.yaml
-echo "component_name: container" >> .ci/ansible/vars/main.yaml
-echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml
+# Compose the scenario definition.
+mkdir -p .ci/ansible/vars
 
-export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh
-export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh
+cat > .ci/ansible/vars/main.yaml << VARSYAML
+---
+scenario: "${TEST}"
+legacy_component_name: "pulp_container"
+component_name: "container"
+component_version: "${COMPONENT_VERSION}"
+pulp_env: {}
+pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]}
+pulp_scheme: "https"
+pulp_default_container: "ghcr.io/pulp/pulp-ci-centos:latest"
+api_root: "${PULP_API_ROOT}"
+image:
+  name: "pulp"
+  tag: "ci_build"
+plugins:
+  - name: "pulp_container"
+    source: "${COMPONENT_SOURCE}"
+    ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
+    upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
+    lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
+services:
+  - name: "pulp"
+    image: "pulp:ci_build"
+    volumes:
+      - "./settings:/etc/pulp"
+      - "./ssh:/keys/"
+      - "~/.config:/var/lib/pulp/.config"
+      - "../../../pulp-openapi-generator:/root/pulp-openapi-generator"
+    env:
+      PULP_WORKERS: "4"
+      PULP_HTTPS: "true"
+VARSYAML
 
-if [ -f $PRE_BEFORE_INSTALL ]; then
-  source $PRE_BEFORE_INSTALL
+if [ "$TEST" = "s3" ]; then
+  MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
+  MINIO_SECRET_KEY=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
+  cat >> .ci/ansible/vars/main.yaml << VARSYAML
+  - name: "minio"
+    image: "minio/minio"
+    env:
+      MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}"
+      MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}"
+    command: "server /data"
+s3_test: true
+minio_access_key: "${MINIO_ACCESS_KEY}"
+minio_secret_key: "${MINIO_SECRET_KEY}"
+pulp_scenario_settings: {"AWS_ACCESS_KEY_ID": "AKIAIT2Z5TDYPX3ARJBA", "AWS_DEFAULT_ACL": "@none None", "AWS_S3_ADDRESSING_STYLE": "path", "AWS_S3_ENDPOINT_URL": "http://minio:9000", "AWS_S3_REGION_NAME": "eu-central-1", "AWS_S3_SIGNATURE_VERSION": "s3v4", "AWS_SECRET_ACCESS_KEY": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "AWS_STORAGE_BUCKET_NAME": "pulp3", "DEFAULT_FILE_STORAGE": "storages.backends.s3boto3.S3Boto3Storage", "MEDIA_ROOT": ""}
+pulp_scenario_env: {}
+VARSYAML
 fi
 
-if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ]
-then
-  echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml
+if [ "$TEST" = "azure" ]; then
+  cat >> .ci/ansible/vars/main.yaml << VARSYAML
+  - name: "ci-azurite"
+    image: "mcr.microsoft.com/azure-storage/azurite"
+    command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"
+azure_test: true
+pulp_scenario_settings: {"AZURE_ACCOUNT_KEY": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "AZURE_ACCOUNT_NAME": "devstoreaccount1", "AZURE_CONNECTION_STRING": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "AZURE_CONTAINER": "pulp-test", "AZURE_LOCATION": "pulp3", "AZURE_OVERWRITE_FILES": true, "AZURE_URL_EXPIRATION_SECS": 120, "DEFAULT_FILE_STORAGE": "storages.backends.azure_storage.AzureStorage", "MEDIA_ROOT": ""}
+pulp_scenario_env: {}
+VARSYAML
 fi
 
-for i in {1..3}
-do
-  ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
-done
-if [[ $s -gt 0 ]]
-then
-  echo "Failed to install amazon.aws"
-  exit $s
+if [ "$TEST" = "gcp" ]; then
+  cat >> .ci/ansible/vars/main.yaml << VARSYAML
+  - name: "ci-gcp"
+    image: "fsouza/fake-gcs-server"
+    volumes:
+      - "storage_data:/etc/pulp"
+    command: " -scheme http"
+gcp_test: true
+pulp_scenario_settings: null
+pulp_scenario_env: {}
+VARSYAML
 fi
 
-if [[ "$TEST" = "pulp" ]]; then
-  python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt
-fi
-if [[ "$TEST" = "lowerbounds" ]]; then
-  python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt
-fi
+cat >> .ci/ansible/vars/main.yaml << VARSYAML
+...
+VARSYAML
+cat .ci/ansible/vars/main.yaml
 
-if [ -f $POST_BEFORE_INSTALL ]; then
-  source $POST_BEFORE_INSTALL
+if [ -f .github/workflows/scripts/post_before_install.sh ]; then
+  source .github/workflows/scripts/post_before_install.sh
 fi