WIP Experiment with CI cleanup

Author: mdellweg

.ci/scripts/check_release.py

@@ -11,7 +11,9 @@
 import argparse
 import re
 import os
+import sys
 import tomllib
+import typing as t
 from pathlib import Path
 
 import yaml
@@ -23,7 +25,7 @@
 Z_CHANGELOG_EXTS = [".bugfix", ".misc"]
 
 
-def options():
+def options() -> argparse.Namespace:
     """Check which branches need a release."""
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -42,13 +44,13 @@ def options():
     return parser.parse_args()
 
 
-def template_config():
+def template_config() -> dict[str, t.Any]:
     # Assume this script lies in .ci/scripts
     path = Path(__file__).absolute().parent.parent.parent / "template_config.yml"
     return yaml.safe_load(path.read_text())
 
 
-def current_version(repo, commitish):
+def current_version(repo: Repo, commitish: str) -> Version:
     try:
         pyproject_toml = tomllib.loads(repo.git.show(f"{commitish}:pyproject.toml"))
         try:
@@ -62,7 +64,7 @@ def current_version(repo, commitish):
     return Version(current_version)
 
 
-def check_pyproject_dependencies(repo, from_commit, to_commit):
+def check_pyproject_dependencies(repo: Repo, from_commit: str, to_commit: str) -> list[str]:
     try:
         new_pyproject = tomllib.loads(repo.git.show(f"{to_commit}:pyproject.toml"))
         try:
@@ -83,8 +85,8 @@ def check_pyproject_dependencies(repo, from_commit, to_commit):
         return ["pyproject.toml changed somehow (PLEASE check if dependencies are affected)."]
 
 
-def main(options, template_config):
-    DEFAULT_BRANCH = template_config["plugin_default_branch"]
+def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int:
+    DEFAULT_BRANCH: str = template_config["plugin_default_branch"]
 
     repo = Repo()
 
@@ -97,7 +99,7 @@ def main(options, template_config):
 
     # Warning: This will not work if branch names contain "/" but we don't really care here.
     heads = [h.split("/")[-1] for h in repo.git.branch("--remote").split("\n")]
-    available_branches = [h for h in heads if re.search(RELEASE_BRANCH_REGEX, h)]
+    available_branches = [h for h in heads if re.fullmatch(RELEASE_BRANCH_REGEX, h)]
     available_branches.sort(key=lambda ver: Version(ver))
     available_branches.append(DEFAULT_BRANCH)
 
@@ -114,7 +116,10 @@ def main(options, template_config):
 
     if diff := branches - set(available_branches):
         print(f"Supplied branches contains non-existent branches! {diff}")
-        exit(1)
+        return 1
+
+    branches = [branch for branch in available_branches if branch in branches]
+    branches.reverse()
 
     print(f"Checking for releases on branches: {branches}")
 
@@ -179,6 +184,8 @@ def main(options, template_config):
     if len(releases) == 0:
         print("No new releases to perform.")
 
+    return 0
+
 
 if __name__ == "__main__":
-    main(options(), template_config())
+    sys.exit(main(options(), template_config()))

.ci/scripts/collect_changes.py

@@ -1,4 +1,12 @@
 #!/bin/env python3
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#     "gitpython>=3.1.46,<3.2.0",
+#     "packaging>=26.0,<26.1",
+# ]
+# ///
+
 # WARNING: DO NOT EDIT!
 #
 # This file was generated by plugin_template, and is managed by it. Please use

.github/workflows/nightly.yml

@@ -61,11 +61,20 @@ jobs:
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v6
+        id: "create_pr_changelog"
         with:
           token: ${{ secrets.RELEASE_TOKEN }}
           title: "Update Changelog"
           body: ""
           branch: "changelog/update"
           delete-branch: true
           path: "pulp_gem"
+      - name: "Mark PR automerge"
+        working-directory: "pulp_gem"
+        run: |
+          gh pr merge --rebase --auto "${{ steps.create_pr_changelog.outputs.pull-request-number }}"
+        if: "steps.create_pr_changelog.outputs.pull-request-number"
+        env:
+          GH_TOKEN: "${{ secrets.RELEASE_TOKEN }}"
+        continue-on-error: true
 ...

.github/workflows/scripts/before_install.sh

@@ -7,57 +7,25 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+set -euo pipefail
+
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 
-set -mveuo pipefail
-
-if [ "${GITHUB_REF##refs/heads/}" = "${GITHUB_REF}" ]
-then
-  BRANCH_BUILD=0
-else
-  BRANCH_BUILD=1
-  BRANCH="${GITHUB_REF##refs/heads/}"
-fi
-if [ "${GITHUB_REF##refs/tags/}" = "${GITHUB_REF}" ]
-then
-  TAG_BUILD=0
-else
-  TAG_BUILD=1
-  BRANCH="${GITHUB_REF##refs/tags/}"
-fi
-
-COMMIT_MSG=$(git log --format=%B --no-merges -1)
-export COMMIT_MSG
-
-COMPONENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
-
-mkdir .ci/ansible/vars || true
-echo "---" > .ci/ansible/vars/main.yaml
-echo "legacy_component_name: pulp_gem" >> .ci/ansible/vars/main.yaml
-echo "component_name: gem" >> .ci/ansible/vars/main.yaml
-echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml
-
 export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh
 export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh
 
 if [ -f $PRE_BEFORE_INSTALL ]; then
   source $PRE_BEFORE_INSTALL
 fi
 
-if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ]
-then
-  echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml
+COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
+COMPONENT_SOURCE="./pulp_gem/dist/pulp_gem-${COMPONENT_VERSION}-py3-none-any.whl"
+if [ "$TEST" = "s3" ]; then
+  COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3]"
 fi
-
-for i in {1..3}
-do
-  ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
-done
-if [[ $s -gt 0 ]]
-then
-  echo "Failed to install amazon.aws"
-  exit $s
+if [ "$TEST" = "azure" ]; then
+  COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]"
 fi
 
 if [[ "$TEST" = "pulp" ]]; then
@@ -67,6 +35,24 @@ if [[ "$TEST" = "lowerbounds" ]]; then
   python3 .ci/scripts/calc_constraints.py pyproject.toml > lowerbounds_constraints.txt
 fi
 
+# Compose ansible container definition.
+mkdir -p .ci/ansible/vars
+
+cat > .ci/ansible/vars/main.yaml << VARSYAML
+---
+legacy_component_name: "pulp_gem"
+component_name: "gem"
+component_version: "${COMPONENT_VERSION}"
+image:
+  name: "pulp"
+  tag: "ci_build"
+plugins:
+  - name: "pulp_gem"
+    source: "${COMPONENT_SOURCE}"
+VARSYAML
+
+cat .ci/ansible/vars/main.yaml
+
 if [ -f $POST_BEFORE_INSTALL ]; then
   source $POST_BEFORE_INSTALL
 fi

.github/workflows/scripts/before_script.sh

@@ -7,6 +7,8 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+# This script dumps some files to help understand the setup of the test scenario.
+
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 
@@ -22,33 +24,35 @@ if [[ -f $PRE_BEFORE_SCRIPT ]]; then
 fi
 
 # Developers should be able to reproduce the containers with this config
-echo "CI vars:"
+echo "# CI vars:"
 tail -v -n +1 .ci/ansible/vars/main.yaml
 
 # Developers often want to know the final pulp config
-echo "PULP CONFIG:"
+echo
+echo "# Pulp config:"
 tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
 
-echo "Containerfile:"
+echo
+echo "# Containerfile:"
 tail -v -n +1 .ci/ansible/Containerfile
 
-echo "Constraints Files:"
-# The need not even exist.
+echo
+echo "# Constraints Files:"
+# They need not even exist.
 tail -v -n +1  ../*/*constraints.txt || true
 
-# Needed for some functional tests
-cmd_prefix bash -c "echo '%wheel        ALL=(ALL)       NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
-cmd_prefix bash -c "usermod -a -G wheel pulp"
+echo
+echo "# pip list outside the container"
 
-if [[ "${REDIS_DISABLED:-false}" == true ]]; then
-  cmd_prefix bash -c "s6-rc -d change redis"
-  echo "The Redis service was disabled for $TEST"
-fi
+echo
+echo "# pip list inside the container"
+cmd_prefix bash -c "pip3 list"
+
+echo
+echo "# State of the containers"
+docker ps -a
 
 if [[ -f $POST_BEFORE_SCRIPT ]]; then
   source $POST_BEFORE_SCRIPT
 fi
 
-# Lots of plugins try to use this path, and throw warnings if they cannot access it.
-cmd_prefix mkdir /.pytest_cache
-cmd_prefix chown pulp:pulp /.pytest_cache

.github/workflows/scripts/install.sh

@@ -7,44 +7,39 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+set -euv
+
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 REPO_ROOT="$PWD"
 
-set -euv
-
 source .github/workflows/scripts/utils.sh
 
-PLUGIN_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
-PLUGIN_SOURCE="./pulp_gem/dist/pulp_gem-${PLUGIN_VERSION}-py3-none-any.whl"
-
 export PULP_API_ROOT="/pulp/"
 
 PIP_REQUIREMENTS=("pulp-cli-gem")
 
 # This must be the **only** call to "pip install" on the test runner.
 pip install ${PIP_REQUIREMENTS[*]}
 
+if [[ "$TEST" = "s3" ]]; then
+for i in {1..3}
+do
+  ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
+done
+if [[ $s -gt 0 ]]
+then
+  echo "Failed to install amazon.aws"
+  exit $s
+fi
+fi
+
 # Check out the pulp-cli-gem branch matching the installed version.
 PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli-gem==//p')"
 git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli-gem.git ../pulp-cli-gem
 
 cd .ci/ansible/
-if [ "$TEST" = "s3" ]; then
-  PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[s3]"
-fi
-if [ "$TEST" = "azure" ]; then
-  PLUGIN_SOURCE="${PLUGIN_SOURCE} pulpcore[azure]"
-fi
 
-cat >> vars/main.yaml << VARSYAML
-image:
-  name: pulp
-  tag: "ci_build"
-plugins:
-  - name: pulp_gem
-    source: "${PLUGIN_SOURCE}"
-VARSYAML
 if [[ -f ../../ci_requirements.txt ]]; then
   cat >> vars/main.yaml << VARSYAML
     ci_requirements: true
@@ -155,6 +150,16 @@ if [[ "$TEST" = "azure" ]]; then
   az storage container create --name pulp-test --connection-string $AZURE_STORAGE_CONNECTION_STRING
 fi
 
-echo ::group::PIP_LIST
-cmd_prefix bash -c "pip3 list"
-echo ::endgroup::
+# Needed for some functional tests
+cmd_prefix bash -c "echo '%wheel        ALL=(ALL)       NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
+cmd_prefix bash -c "usermod -a -G wheel pulp"
+
+# In some scenarios we want to simulate a failed redis cache.
+if [[ " s3 " =~ " ${TEST} " ]]; then
+  cmd_prefix bash -c "s6-rc -d change redis"
+  echo "The Redis service was disabled for $TEST"
+fi
+
+# Lots of plugins try to use this path, and throw warnings if they cannot access it.
+cmd_prefix mkdir /.pytest_cache
+cmd_prefix chown pulp:pulp /.pytest_cache

.github/workflows/test.yml

@@ -99,7 +99,7 @@ jobs:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
 
-      - name: "Before Script"
+      - name: "Dump CI Metadata"
         run: |
           .github/workflows/scripts/before_script.sh
         shell: "bash"
@@ -108,7 +108,6 @@ jobs:
           ANSIBLE_FORCE_COLOR: "1"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
-          REDIS_DISABLED: "${{ contains('', matrix.env.TEST) }}"
 
       - name: "Script"
         run: |

MANIFEST.in

@@ -9,3 +9,4 @@ include test_requirements.txt
 include unittest_requirements.txt
 include pyproject.toml
 exclude releasing.md
+exclude Makefile

Makefile

@@ -0,0 +1,21 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --ci pulp_gem' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+.PHONY: format
+format:
+	black .
+
+.PHONY: lint
+lint:
+	yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows
+	bump-my-version bump --dry-run release
+	black --check --diff .
+	flake8
+	check-manifest
+	python .ci/scripts/check_requirements.py
+	sh .ci/scripts/check_pulpcore_imports.sh
+	sh .ci/scripts/check_gettext.sh