Update CI files

Author: pulpbot

.ci/scripts/check_release.py

@@ -11,7 +11,9 @@
 import argparse
 import re
 import os
+import sys
 import tomllib
+import typing as t
 from pathlib import Path
 
 import yaml
@@ -23,7 +25,7 @@
 Z_CHANGELOG_EXTS = [".bugfix", ".misc"]
 
 
-def options():
+def options() -> argparse.Namespace:
     """Check which branches need a release."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -42,13 +44,13 @@ def options():
     return parser.parse_args()
 
 
-def template_config():
+def template_config() -> dict[str, t.Any]:
     # Assume this script lies in .ci/scripts
     path = Path(__file__).absolute().parent.parent.parent / "template_config.yml"
     return yaml.safe_load(path.read_text())
 
 
-def current_version(repo, commitish):
+def current_version(repo: Repo, commitish: str) -> Version:
     try:
         pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml"))
         try:
@@ -62,7 +64,7 @@ def current_version(repo, commitish):
     return Version(current_version)
 
 
-def check_pyproject_dependencies(repo, from_commit, to_commit):
+def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]:
     try:
         new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml"))
         try:
@@ -83,8 +85,8 @@ def check_pyproject_dependencies(repo, from_commit, to_commit):
         return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."]
 
 
-def main(options, template_config):
-    DEFAULT_BRANCH = template_config["plugin_default_branch"]
+def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int:
+    DEFAULT_BRANCH: str = template_config["plugin_default_branch"]
 
     repo = Repo()
 
@@ -97,7 +99,7 @@ def main(options, template_config):
 
     # Warning: This will not work if branch names contain "/" but we don't really care here.
     heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")]
-    available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
+    available_branches = [h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)]
     available_branches.sort(key=lambda ver: Version(ver))
     available_branches.append(DEFAULT_BRANCH)
 
@@ -114,7 +116,10 @@ def main(options, template_config):
 
     if diff := branches - set(available_branches):
         print(f"Supplied branches contains non-existent branches! {diff}")
-        exit(1)
+        return 1
+
+    branches = [branch for branch in available_branches if branch in branches]
+    branches.reverse()
 
     print(f"Checking for releases on branches: {branches}")
 
@@ -179,6 +184,8 @@ def main(options, template_config):
     if len(releases) == 0:
         print("No new releases to perform.")
 
+    return 0
+
 
 if __name__ == "__main__":
-    main(options(), template_config())
+    sys.exit(main(options(), template_config()))

.ci/scripts/clean_gh_release_notes.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# This script is running with elevated privileges from the main branch against pull requests.
+#
+# It cleans the input from artifacts which are used by the pulp documentation internally,
+# but clutter for GitHub releases
+
+import sys
+
+NOTE = """
+> [!NOTE]
+> Official changes are available on [Pulp docs]({docs_url})\
+"""
+
+
+def main():
+    plugin_name = sys.argv[1]
+    version_str = sys.argv[2]
+    docs_url = f"https://pulpproject.org/{plugin_name}/changes/#{version_str}"
+    note_added = False
+    for line in sys.stdin:
+        if line.endswith("\n"):
+            line = line[:-1]
+        if line.startswith("#"):
+            print(line.split(" {: #")[0])
+            if not note_added and version_str in line:
+                print(NOTE.format(docs_url=docs_url))
+                note_added = True
+        else:
+            print(line)
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/collect_changes.py

@@ -1,4 +1,12 @@
 #!/bin/env python3
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "gitpython>=3.1.46,<3.2.0",
+#     "packaging>=26.0,<26.1",
+# ]
+# ///
+
 # WARNING: DO NOT EDIT!
 #
 # This file was generated by plugin_template, and is managed by it. Please use

.github/workflows/nightly.yml

@@ -61,11 +61,20 @@ jobs:
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v6
+        id: "create_pr_changelog"
         with:
           token: ${{ secrets.RELEASE_TOKEN }}
           title: "Update Changelog"
           body: ""
           branch: "changelog/update"
           delete-branch: true
           path: "pulpcore"
+      - name: "Mark PR automerge"
+        working-directory: "pulpcore"
+        run: |
+          gh pr merge --rebase --auto "${{ steps.create_pr_changelog.outputs.pull-request-number }}"
+        if: "steps.create_pr_changelog.outputs.pull-request-number"
+        env:
+          GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}"
+        continue-on-error: true
 ...

.github/workflows/publish.yml

@@ -142,7 +142,7 @@ jobs:
         run: |
           # The last commit before the release commit contains the release CHANGES fragments
           git checkout "${TAG_NAME}~"
-          NOTES=$(towncrier build --draft --version $TAG_NAME)
+          NOTES=$(towncrier build --draft --version $TAG_NAME | .ci/scripts/clean_gh_release_notes.py pulpcore $TAG_NAME)
           echo "body<<EOF" >> $GITHUB_OUTPUT
           echo "$NOTES" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT

.github/workflows/scripts/before_install.sh

@@ -7,66 +7,128 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+# This script prepares the scenario definition in the .ci/ansible/vars/main.yaml file.
+#
+#   It requires the following environment:
+#     TEST - The name of the scenario to prepare.
+#
+#   It may also dump the {lower,upper}bounds_constraints.txt for the specific scenario.
+
+set -eu -o pipefail
+
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 
-set -mveuo pipefail
+if [ -f .github/workflows/scripts/pre_before_install.sh ]; then
+  source .github/workflows/scripts/pre_before_install.sh
+fi
 
-if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ]
-then
-  BRANCH_BUILD=0
-else
-  BRANCH_BUILD=1
-  BRANCH="${GITHUB_REF##refs/heads/}"
+COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
+COMPONENT_SOURCE="./pulpcore/dist/pulpcore-${COMPONENT_VERSION}-py3-none-any.whl"
+if [ "$TEST" = "s3" ]; then
+  COMPONENT_SOURCE="${COMPONENT_SOURCE}[s3]"
 fi
-if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ]
-then
-  TAG_BUILD=0
-else
-  TAG_BUILD=1
-  BRANCH="${GITHUB_REF##refs/tags/}"
+if [ "$TEST" = "azure" ]; then
+  COMPONENT_SOURCE="${COMPONENT_SOURCE}[azure]"
 fi
 
-COMMIT_MSG=$(git log --format=%B --no-merges -1)
-export COMMIT_MSG
+if [[ "$TEST" = "pulp" ]]; then
+  python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt
+fi
+if [[ "$TEST" = "lowerbounds" ]]; then
+  python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt
+fi
+export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/")
 
-COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py)
+echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
 
-mkdir .ci/ansible/vars || true
-echo "---" > .ci/ansible/vars/main.yaml
-echo "legacy_component_name: pulpcore" >> .ci/ansible/vars/main.yaml
-echo "component_name: core" >> .ci/ansible/vars/main.yaml
-echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml
+# Compose the scenario definition.
+mkdir -p .ci/ansible/vars
 
-export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh
-export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh
+cat > .ci/ansible/vars/main.yaml << VARSYAML
+---
+scenario: "${TEST}"
+legacy_component_name: "pulpcore"
+component_name: "core"
+component_version: "${COMPONENT_VERSION}"
+pulp_env: {"PULP_CA_BUNDLE": "/etc/pulp/certs/pulp_webserver.crt"}
+pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "content_path_prefix": "/somewhere/else/", "orphan_protection_time": 0, "task_protection_time": 10, "tmpfile_protection_time": 10, "upload_protection_time": 10}
+pulp_scheme: "https"
+pulp_default_container: "ghcr.io/pulp/pulp-ci-centos9:latest"
+api_root: "${PULP_API_ROOT}"
+image:
+  name: "pulp"
+  tag: "ci_build"
+plugins:
+  - name: "pulpcore"
+    source: "${COMPONENT_SOURCE}"
+    ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
+    upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
+    lowerounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
+services:
+  - name: "pulp"
+    image: "pulp:ci_build"
+    volumes:
+      - "./settings:/etc/pulp"
+      - "./ssh:/keys/"
+      - "~/.config:/var/lib/pulp/.config"
+      - "../../../pulp-openapi-generator:/root/pulp-openapi-generator"
+    env:
+      PULP_WORKERS: "4"
+      PULP_HTTPS: "true"
+  - name: "pulp-fixtures"
+    image: "docker.io/pulp/pulp-fixtures:latest"
+    env:
+      BASE_URL: "http://pulp-fixtures:8080"
+VARSYAML
 
-if [ -f $PRE_BEFORE_INSTALL ]; then
-  source $PRE_BEFORE_INSTALL
+if [ "$TEST" = "s3" ]; then
+  MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
+  MINIO_SECRET_KEY=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
+  cat >> .ci/ansible/vars/main.yaml << VARSYAML
+  - name: "minio"
+    image: "minio/minio"
+    env:
+      MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}"
+      MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}"
+    command: "server /data"
+s3_test: true
+minio_access_key: "${MINIO_ACCESS_KEY}"
+minio_secret_key: "${MINIO_SECRET_KEY}"
+pulp_scenario_settings: {"DISABLED_authentication_backends": "@merge django.contrib.auth.backends.RemoteUserBackend", "DISABLED_authentication_json_header": "HTTP_X_RH_IDENTITY", "DISABLED_authentication_json_header_jq_filter": ".identity.user.username", "DISABLED_authentication_json_header_openapi_security_scheme": {"description": "External OAuth integration", "flows": {"clientCredentials": {"scopes": {"api.console": "grant_access_to_pulp"}, "tokenUrl": "https://your-identity-provider/token/issuer"}}, "type": "oauth2"}, "DISABLED_rest_framework__default_authentication_classes": "@merge pulpcore.app.authentication.JSONHeaderRemoteAuthentication", "MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true, "hide_guarded_distributions": true}
+pulp_scenario_env: {}
+VARSYAML
 fi
 
-if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ]
-then
-  echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml
+if [ "$TEST" = "azure" ]; then
+  cat >> .ci/ansible/vars/main.yaml << VARSYAML
+  - name: "ci-azurite"
+    image: "mcr.microsoft.com/azure-storage/azurite"
+    command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"
+azure_test: true
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "api_root_rewrite_header": "X-API-Root", "domain_enabled": true}
+pulp_scenario_env: {}
+VARSYAML
 fi
 
-for i in {1..3}
-do
-  ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
-done
-if [[ $s -gt 0 ]]
-then
-  echo "Failed to install amazon.aws"
-  exit $s
+if [ "$TEST" = "gcp" ]; then
+  cat >> .ci/ansible/vars/main.yaml << VARSYAML
+  - name: "ci-gcp"
+    image: "fsouza/fake-gcs-server"
+    volumes:
+      - "storage_data:/etc/pulp"
+    command: " -scheme http"
+gcp_test: true
+pulp_scenario_settings: null
+pulp_scenario_env: {}
+VARSYAML
 fi
 
-if [[ "$TEST" = "pulp" ]]; then
-  python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt
-fi
-if [[ "$TEST" = "lowerbounds" ]]; then
-  python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt
-fi
+cat >> .ci/ansible/vars/main.yaml << VARSYAML
+...
+VARSYAML
+cat .ci/ansible/vars/main.yaml
 
-if [ -f $POST_BEFORE_INSTALL ]; then
-  source $POST_BEFORE_INSTALL
+if [ -f .github/workflows/scripts/post_before_install.sh ]; then
+  source .github/workflows/scripts/post_before_install.sh
 fi