|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +# |
| 3 | +# Copyright (C) GrimoireLab Developers |
| 4 | +# |
| 5 | +# This program is free software; you can redistribute it and/or modify |
| 6 | +# it under the terms of the GNU General Public License as published by |
| 7 | +# the Free Software Foundation; either version 3 of the License, or |
| 8 | +# (at your option) any later version. |
| 9 | +# |
| 10 | +# This program is distributed in the hope that it will be useful, |
| 11 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | +# GNU General Public License for more details. |
| 14 | +# |
| 15 | +# You should have received a copy of the GNU General Public License |
| 16 | +# along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 17 | +# |
| 18 | + |
| 19 | +from __future__ import annotations |
| 20 | + |
| 21 | +import concurrent.futures |
| 22 | +import os |
| 23 | +import random |
| 24 | +import shutil |
| 25 | +import subprocess |
| 26 | +import tempfile |
| 27 | +import time |
| 28 | + |
| 29 | +import dulwich.repo |
| 30 | +import pytest |
| 31 | + |
| 32 | +from testcontainers.core.waiting_utils import wait_for_logs |
| 33 | +from testcontainers.mysql import MySqlContainer |
| 34 | +from testcontainers.opensearch import OpenSearchContainer |
| 35 | +from testcontainers.redis import RedisContainer |
| 36 | + |
| 37 | + |
| 38 | +mysql = MySqlContainer("mariadb:latest", root_password="root").with_exposed_ports(3306) |
| 39 | +redis = RedisContainer().with_exposed_ports(6379) |
| 40 | +opensearch = OpenSearchContainer().with_exposed_ports(9200) |
| 41 | + |
| 42 | + |
| 43 | +def generate_repository(repo_tmpdir, max_commits): |
| 44 | + """Create a temporary git repository. |
| 45 | +
|
| 46 | + Each repository is created with a random number of commits. The parameter |
| 47 | + 'max_commits' defines the maximum number of commits a repository can have. |
| 48 | + The last commit includes a 'summary.txt' file with the number of commits |
| 49 | + of the repository. |
| 50 | +
|
| 51 | + :param max_commits: Maximum number of commits per repository. |
| 52 | +
|
| 53 | + :return: filepath to the temporary directory |
| 54 | + """ |
| 55 | + repo = dulwich.repo.Repo.init(repo_tmpdir, mkdir=True) |
| 56 | + |
| 57 | + num_commits = random.randint(1, max_commits) |
| 58 | + |
| 59 | + # Create empty commits |
| 60 | + for j in range(num_commits - 1): |
| 61 | + repo.do_commit( |
| 62 | + message=f"Commit #{j}".encode(), |
| 63 | + author=b"John Smith <[email protected]>", |
| 64 | + committer=b"John Smith <[email protected]>" |
| 65 | + ) |
| 66 | + |
| 67 | + # Create the last commit with a summary file |
| 68 | + summary_file = os.path.join(repo_tmpdir, "summary.txt") |
| 69 | + with open(summary_file, 'w') as fd: |
| 70 | + fd.write(f"{str(num_commits)}\n") |
| 71 | + repo.stage("summary.txt") |
| 72 | + |
| 73 | + repo.do_commit( |
| 74 | + message=f"Commit #{num_commits - 1} - Summary".encode(), |
| 75 | + author=b"John Smith <[email protected]>", |
| 76 | + committer=b"John Smith <[email protected]>" |
| 77 | + ) |
| 78 | + |
| 79 | + |
| 80 | +@pytest.fixture |
| 81 | +def setup_git_repositories(request, num_repositories, max_commits): |
| 82 | + """Create a number of temporary git repositories. |
| 83 | +
|
| 84 | + This fixture uses threads to speed up the creation of the |
| 85 | + testing repositories. |
| 86 | +
|
| 87 | + :param num_repositories: Number of repositories to create. |
| 88 | + :param max_commits: Maximum number of commits per repository. |
| 89 | +
|
| 90 | + :return: filepath to the temporary directory |
| 91 | + """ |
| 92 | + tmpdir = tempfile.mkdtemp(prefix="grimoirelab_") |
| 93 | + |
| 94 | + # Add a finalizer to remove the temporary directory |
| 95 | + def remove_tmpdir(): |
| 96 | + shutil.rmtree(tmpdir) |
| 97 | + |
| 98 | + request.addfinalizer(remove_tmpdir) |
| 99 | + |
| 100 | + random.seed() |
| 101 | + |
| 102 | + processed = 0 |
| 103 | + max_threads = 25 |
| 104 | + |
| 105 | + while processed < num_repositories: |
| 106 | + to_process = min(max_threads, num_repositories - processed) |
| 107 | + |
| 108 | + with concurrent.futures.ThreadPoolExecutor() as executor: |
| 109 | + for i in range(to_process): |
| 110 | + repo_tmpdir = tempfile.mktemp(dir=tmpdir) |
| 111 | + executor.submit(generate_repository, repo_tmpdir, max_commits) |
| 112 | + |
| 113 | + processed += to_process |
| 114 | + |
| 115 | + return tmpdir |
| 116 | + |
| 117 | + |
| 118 | +@pytest.fixture(scope="module") |
| 119 | +def setup_mysql(request): |
| 120 | + mysql.start() |
| 121 | + |
| 122 | + def remove_container(): |
| 123 | + mysql.stop() |
| 124 | + |
| 125 | + request.addfinalizer(remove_container) |
| 126 | + |
| 127 | + |
| 128 | +@pytest.fixture(scope="module") |
| 129 | +def setup_redis(request): |
| 130 | + redis.start() |
| 131 | + |
| 132 | + def remove_container(): |
| 133 | + redis.stop() |
| 134 | + |
| 135 | + request.addfinalizer(remove_container) |
| 136 | + |
| 137 | + |
| 138 | +@pytest.fixture(scope="module") |
| 139 | +def setup_opensearch(request): |
| 140 | + opensearch.start() |
| 141 | + |
| 142 | + def remove_container(): |
| 143 | + opensearch.stop() |
| 144 | + |
| 145 | + request.addfinalizer(remove_container) |
| 146 | + wait_for_logs(opensearch, ".*recovered .* indices into cluster_state.*") |
| 147 | + |
| 148 | + |
| 149 | +@pytest.fixture(scope="module", autouse=True) |
| 150 | +def setup_containers(setup_opensearch, setup_redis, setup_mysql): |
| 151 | + yield |
| 152 | + |
| 153 | + |
| 154 | +@pytest.fixture(scope="module", autouse=True) |
| 155 | +def setup_grimoirelab(request): |
| 156 | + os.environ["DJANGO_SETTINGS_MODULE"] = "grimoirelab.core.config.settings" |
| 157 | + os.environ["GRIMOIRELAB_REDIS_PORT"] = redis.get_exposed_port(6379) |
| 158 | + os.environ["GRIMOIRELAB_DB_PORT"] = mysql.get_exposed_port(3306) |
| 159 | + os.environ["GRIMOIRELAB_DB_PASSWORD"] = mysql.root_password |
| 160 | + os.environ["GRIMOIRELAB_ARCHIVIST_STORAGE_URL"] = f"http://localhost:{opensearch.get_exposed_port(9200)}" |
| 161 | + os.environ["GRIMOIRELAB_USER_PASSWORD"] = "admin" |
| 162 | + os.environ["GRIMOIRELAB_ARCHIVIST_BLOCK_TIMEOUT"] = "1000" |
| 163 | + |
| 164 | + subprocess.run(["grimoirelab", "admin", "setup"]) |
| 165 | + subprocess.run(["grimoirelab", "admin", "create-user", "--username", "admin", "--no-interactive"]) |
| 166 | + |
| 167 | + grimoirelab = subprocess.Popen( |
| 168 | + ["grimoirelab", "run", "server", "--dev"], |
| 169 | + stdout=subprocess.DEVNULL, |
| 170 | + stderr=subprocess.DEVNULL, |
| 171 | + start_new_session=True, |
| 172 | + ) |
| 173 | + eventizers = subprocess.Popen( |
| 174 | + ["grimoirelab", "run", "eventizers", "--workers", "10"], |
| 175 | + stdout=subprocess.DEVNULL, |
| 176 | + stderr=subprocess.DEVNULL, |
| 177 | + start_new_session=True, |
| 178 | + ) |
| 179 | + archivists = subprocess.Popen( |
| 180 | + ["grimoirelab", "run", "archivists", "--workers", "10"], |
| 181 | + stdout=subprocess.DEVNULL, |
| 182 | + stderr=subprocess.DEVNULL, |
| 183 | + start_new_session=True, |
| 184 | + ) |
| 185 | + |
| 186 | + def stop_grimoirelab(): |
| 187 | + # GrimoireLab uses uWSGI. It won't stop with SIGTERM, |
| 188 | + # so we need to use SIGKILL |
| 189 | + grimoirelab.kill() |
| 190 | + |
| 191 | + archivists.terminate() |
| 192 | + eventizers.terminate() |
| 193 | + |
| 194 | + # Wait for process to be done |
| 195 | + grimoirelab.wait() |
| 196 | + archivists.wait() |
| 197 | + eventizers.wait() |
| 198 | + |
| 199 | + request.addfinalizer(stop_grimoirelab) |
| 200 | + |
| 201 | + time.sleep(10) |
0 commit comments