diff --git a/conan/cli/commands/cache.py b/conan/cli/commands/cache.py index 117eea9a3d9..cd3fa5eca2a 100644 --- a/conan/cli/commands/cache.py +++ b/conan/cli/commands/cache.py @@ -185,6 +185,9 @@ def cache_save(conan_api: ConanAPI, parser, subparser, *args): else: ref_pattern = ListPattern(args.pattern) package_list = conan_api.list.select(ref_pattern) + if args.file and not args.file.endswith(".tgz"): + ConanOutput().warning("Compression using other than .tgz is experimental. Use .tzx or " + ".tzst (Python>=3.14 only) extensions for other formats") tgz_path = make_abs_path(args.file or "conan_cache_save.tgz") conan_api.cache.save(package_list, tgz_path, args.no_source) return {"results": {"Local Cache": package_list.serialize()}} diff --git a/conan/internal/api/uploader.py b/conan/internal/api/uploader.py index 4a8f90aca91..902bb4b09a8 100644 --- a/conan/internal/api/uploader.py +++ b/conan/internal/api/uploader.py @@ -2,6 +2,7 @@ import gzip import os import shutil +import sys import tarfile import time @@ -10,8 +11,8 @@ from conan.internal.source import retrieve_exports_sources from conan.internal.errors import NotFoundException from conan.errors import ConanException -from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME, - EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO) +from conan.internal.paths import CONAN_MANIFEST, CONANFILE, CONANINFO, COMPRESSIONS, \ + EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME from conan.internal.util.files import (clean_dirty, is_dirty, gather_files, set_dirty_context_manager, mkdir, human_size) @@ -85,6 +86,26 @@ def __init__(self, app: ConanApp, global_conf): self._app = app self._global_conf = global_conf + # No compressed file exists, need to compress + compressformat = self._global_conf.get("core.upload:compression_format", + default="gz", choices=COMPRESSIONS) + if compressformat in ("xz", "zst"): + ConanOutput().warning(f"The {compressformat} compression is highly experimental, " + f"use it at your own risk and expect issues. Feedback welcome, " + f"please report it as Github tickets", + warn_tag="risk") + if compressformat == "zst" and sys.version_info.minor < 14: + raise ConanException("The 'core.upload:compression_format=zst' is only for Python>=3.14") + compresslevel = self._global_conf.get("core:compresslevel", check_type=int) + if compresslevel is None and compressformat == "gz": + compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) + if compresslevel is not None: + ConanOutput().warning("core.gzip:compresslevel is deprecated, " + "use core.compresslevel instead", warn_tag="deprecated") + + self._compressformat = compressformat + self._compresslevel = compresslevel + def prepare(self, pkg_list, enabled_remotes): local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"]) for ref, packages in pkg_list.items(): @@ -128,14 +149,6 @@ def _prepare_recipe(self, ref, ref_bundle, conanfile, remotes): def _compress_recipe_files(self, layout, ref): download_export_folder = layout.download_export() - output = ConanOutput(scope=str(ref)) - for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME): - tgz_path = os.path.join(download_export_folder, f) - if is_dirty(tgz_path): - output.warning("Removing %s, marked as dirty" % f) - os.remove(tgz_path) - clean_dirty(tgz_path) - export_folder = layout.export() files, symlinked_folders = gather_files(export_folder) files.update(symlinked_folders) @@ -159,18 +172,13 @@ def _compress_recipe_files(self, layout, ref): files.pop(CONANFILE) files.pop(CONAN_MANIFEST) - def add_tgz(tgz_name, tgz_files): - tgz = os.path.join(download_export_folder, tgz_name) - if os.path.isfile(tgz): - result[tgz_name] = tgz - elif tgz_files: - compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz = compress_files(tgz_files, tgz_name, download_export_folder, - compresslevel=compresslevel, ref=ref) - result[tgz_name] = tgz - - add_tgz(EXPORT_TGZ_NAME, files) - add_tgz(EXPORT_SOURCES_TGZ_NAME, src_files) + if files: + comp = self._compressed_file(EXPORT_FILE_NAME, files, download_export_folder, ref) + result[comp] = os.path.join(download_export_folder, comp) + if src_files: + comp = self._compressed_file(EXPORT_SOURCES_FILE_NAME, src_files, + download_export_folder, ref) + result[comp] = os.path.join(download_export_folder, comp) return result def _prepare_package(self, pref, prev_bundle): @@ -181,14 +189,35 @@ def _prepare_package(self, pref, prev_bundle): cache_files = self._compress_package_files(pkg_layout, pref) prev_bundle["files"] = cache_files + def _compressed_file(self, filename, files, download_folder, ref): + output = ConanOutput(scope=str(ref)) + + # Check if there is some existing compressed file already + matches = [] + for extension in COMPRESSIONS: + file_name = filename + extension + package_file = os.path.join(download_folder, file_name) + if is_dirty(package_file): + output.warning(f"Removing {file_name}, marked as dirty") + os.remove(package_file) + clean_dirty(package_file) + if os.path.isfile(package_file): + matches.append(file_name) + if len(matches) > 1: + raise ConanException(f"{ref}: Multiple package files found for {filename}: {matches}") + if len(matches) == 1: + return matches[0] + + file_name = filename + self._compressformat + package_file = os.path.join(download_folder, file_name) + compressed_path = compress_files(files, file_name, download_folder, + compresslevel=self._compresslevel, ref=ref) + assert compressed_path == package_file + assert os.path.exists(package_file) + return file_name + def _compress_package_files(self, layout, pref): - output = ConanOutput(scope=str(pref)) download_pkg_folder = layout.download_package() - package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME) - if is_dirty(package_tgz): - output.warning("Removing %s, marked as dirty" % PACKAGE_TGZ_NAME) - os.remove(package_tgz) - clean_dirty(package_tgz) # Get all the files in that directory # existing package @@ -209,15 +238,8 @@ def _compress_package_files(self, layout, pref): files.pop(CONANINFO) files.pop(CONAN_MANIFEST) - if not os.path.isfile(package_tgz): - tgz_files = {f: path for f, path in files.items()} - compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int) - tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder, - compresslevel=compresslevel, ref=pref) - assert tgz_path == package_tgz - assert os.path.exists(package_tgz) - - return {PACKAGE_TGZ_NAME: package_tgz, + compressed_file = self._compressed_file(PACKAGE_FILE_NAME, files, download_pkg_folder, pref) + return {compressed_file: os.path.join(download_pkg_folder, compressed_file), CONANINFO: os.path.join(download_pkg_folder, CONANINFO), CONAN_MANIFEST: os.path.join(download_pkg_folder, CONAN_MANIFEST)} @@ -288,6 +310,19 @@ def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursiv tgz_path = os.path.join(dest_dir, name) if ref: ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}") + + if name.endswith("zst"): + with tarfile.open(tgz_path, "w:zst", level=compresslevel) as tar: # noqa Py314 only + for filename, abs_path in sorted(files.items()): + tar.add(abs_path, filename, recursive=recursive) + return tgz_path + + if name.endswith("xz"): + with tarfile.open(tgz_path, "w:xz", preset=compresslevel, format=tarfile.PAX_FORMAT) as tar: + for filename, abs_path in sorted(files.items()): + tar.add(abs_path, filename, recursive=recursive) + return tgz_path + with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel) for filename, abs_path in sorted(files.items()): diff --git a/conan/internal/model/conf.py b/conan/internal/model/conf.py index 42c662e5063..ab59e754d23 100644 --- a/conan/internal/model/conf.py +++ b/conan/internal/model/conf.py @@ -60,8 +60,11 @@ "core.net.http:cacert_path": "Path containing a custom Cacert file", "core.net.http:client_cert": "Path or tuple of files containing a client cert (and key)", "core.net.http:clean_system_proxy": "If defined, the proxies system env-vars will be discarded", - # Gzip compression + # Compression for `conan upload` + "core.upload:compression_format": "The compression format used when uploading Conan packages. " + "Possible values: 'zst', 'xz', 'gz' (default=gz)", "core.gzip:compresslevel": "The Gzip compression level for Conan artifacts (default=9)", + "core:compresslevel": "The compression level for Conan artifacts (default zstd=3, gz=9)", # Excluded from revision_mode = "scm" dirty and Git().is_dirty() checks "core.scm:excluded": "List of excluded patterns for builtin git dirty checks", "core.scm:local_url": "By default allows to store local folders as remote url, but not upload them. Use 'allow' for allowing upload and 'block' to completely forbid it", diff --git a/conan/internal/model/manifest.py b/conan/internal/model/manifest.py index 849f40339cc..3e598c8e3b2 100644 --- a/conan/internal/model/manifest.py +++ b/conan/internal/model/manifest.py @@ -1,7 +1,8 @@ import os from collections import defaultdict -from conan.internal.paths import CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME +from conan.internal.paths import CONAN_MANIFEST, COMPRESSIONS, PACKAGE_FILE_NAME, EXPORT_FILE_NAME, \ + EXPORT_SOURCES_FILE_NAME from conan.internal.util.dates import timestamp_now, timestamp_to_str from conan.internal.util.files import load, md5, md5sum, save, gather_files @@ -91,8 +92,10 @@ def create(cls, folder, exports_sources_folder=None): """ files, _ = gather_files(folder) # The folders symlinks are discarded for the manifest - for f in (PACKAGE_TGZ_NAME, EXPORT_TGZ_NAME, CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME): - files.pop(f, None) + for f in (PACKAGE_FILE_NAME, EXPORT_FILE_NAME, EXPORT_SOURCES_FILE_NAME): + for e in COMPRESSIONS: + files.pop(f + e, None) + files.pop(CONAN_MANIFEST, None) file_dict = {} for name, filepath in files.items(): diff --git a/conan/internal/paths.py b/conan/internal/paths.py index 084c35dfd84..adc7d0dfa6f 100644 --- a/conan/internal/paths.py +++ b/conan/internal/paths.py @@ -86,7 +86,8 @@ def _user_home_from_conanrc_file(): CONANFILE_TXT = "conanfile.txt" CONAN_MANIFEST = "conanmanifest.txt" CONANINFO = "conaninfo.txt" -PACKAGE_TGZ_NAME = "conan_package.tgz" -EXPORT_TGZ_NAME = "conan_export.tgz" -EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz" +PACKAGE_FILE_NAME = "conan_package.t" +EXPORT_FILE_NAME = "conan_export.t" +EXPORT_SOURCES_FILE_NAME = "conan_sources.t" +COMPRESSIONS = "gz", "xz", "zst" DATA_YML = "conandata.yml" diff --git a/conan/internal/rest/remote_manager.py b/conan/internal/rest/remote_manager.py index bfe49099c4f..e130a701a3e 100644 --- a/conan/internal/rest/remote_manager.py +++ b/conan/internal/rest/remote_manager.py @@ -1,11 +1,14 @@ import os import shutil +import sys + from collections import namedtuple from typing import List from requests.exceptions import ConnectionError from conan.api.model import LOCAL_RECIPES_INDEX +from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME from conan.internal.rest.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex from conan.api.model import Remote from conan.api.output import ConanOutput @@ -17,7 +20,6 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference from conan.internal.util.files import rmdir, human_size -from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME from conan.internal.util.files import mkdir, tar_extract @@ -86,7 +88,8 @@ def get_recipe(self, ref, remote, metadata=None): self._cache.remove_recipe_layout(layout) raise export_folder = layout.export() - tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) + export_file = next((f for f in zipped_files if EXPORT_FILE_NAME in f), None) + tgz_file = zipped_files.pop(export_file, None) if tgz_file: uncompress_file(tgz_file, export_folder, scope=str(ref)) @@ -132,7 +135,8 @@ def get_recipe_sources(self, ref, layout, remote): return self._signer.verify(ref, download_folder, files=zipped_files) - tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] + # Only 1 file is guaranteed + tgz_file = next(iter(zipped_files.values())) uncompress_file(tgz_file, export_sources_folder, scope=str(ref)) def get_package(self, pref, remote, metadata=None): @@ -178,12 +182,15 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata): metadata, only_metadata=False) zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)} # quick server package integrity check: - for f in ("conaninfo.txt", "conanmanifest.txt", "conan_package.tgz"): + for f in (CONANINFO, CONAN_MANIFEST): if f not in zipped_files: raise ConanException(f"Corrupted {pref} in '{remote.name}' remote: no {f}") + + # This is guaranteed to exists, otherwise RestClient would have raised already + package_file = next(f for f in zipped_files if PACKAGE_FILE_NAME in f) self._signer.verify(pref, download_pkg_folder, zipped_files) - tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) + tgz_file = zipped_files.pop(package_file) package_folder = layout.package() uncompress_file(tgz_file, package_folder, scope=str(pref.ref)) mkdir(package_folder) # Just in case it doesn't exist, because uncompress did nothing @@ -337,6 +344,9 @@ def _call_remote(self, remote, method, *args, **kwargs): def uncompress_file(src_path, dest_folder, scope=None): + if sys.version_info.minor < 14 and src_path.endswith(".tzst"): + raise ConanException(f"File {os.path.basename(src_path)} compressed with 'zst', " + f"unsupported for Python<3.14 ") try: filesize = os.path.getsize(src_path) big_file = filesize > 10000000 # 10 MB diff --git a/conan/internal/rest/rest_client_v2.py b/conan/internal/rest/rest_client_v2.py index b8d23cd68a4..762cbd5fd57 100644 --- a/conan/internal/rest/rest_client_v2.py +++ b/conan/internal/rest/rest_client_v2.py @@ -9,7 +9,8 @@ from uuid import getnode as get_mac from conan.api.output import ConanOutput - +from conan.internal.paths import EXPORT_SOURCES_FILE_NAME, CONANINFO, CONAN_MANIFEST, \ + EXPORT_FILE_NAME, PACKAGE_FILE_NAME from conan.internal.rest.caching_file_downloader import ConanInternalCacheDownloader from conan.internal.rest import response_to_str from conan.internal.rest.client_routes import ClientV2Router @@ -18,7 +19,6 @@ RecipeNotFoundException, PackageNotFoundException, EXCEPTION_CODE_MAPPING from conan.errors import ConanException from conan.api.model import PkgReference -from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME from conan.api.model import RecipeReference from conan.internal.util.dates import from_iso8601_to_timestamp @@ -239,9 +239,11 @@ def get_recipe(self, ref, dest_folder, metadata, only_metadata): result = {} if not only_metadata: - accepted_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt", - "metadata/sign"] + accepted_files = ["conanfile.py", CONAN_MANIFEST, "metadata/sign"] files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)] + export_file = self._find_compressed_file(ref, server_files, EXPORT_FILE_NAME) + if export_file is not None: + files.append(export_file) # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.recipe_file(ref, fn) for fn in files} self._download_and_save_files(urls, dest_folder, files, parallel=True) @@ -260,9 +262,10 @@ def get_recipe_sources(self, ref, dest_folder): url = self.router.recipe_snapshot(ref) data = self._get_file_list_json(url) files = data["files"] - if EXPORT_SOURCES_TGZ_NAME not in files: + src_file = self._find_compressed_file(ref, files, EXPORT_SOURCES_FILE_NAME) + if src_file is None: return None - files = [EXPORT_SOURCES_TGZ_NAME, ] + files = [src_file, ] # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.recipe_file(ref, fn) for fn in files} @@ -270,6 +273,19 @@ def get_recipe_sources(self, ref, dest_folder): ret = {fn: os.path.join(dest_folder, fn) for fn in files} return ret + @staticmethod + def _find_compressed_file(ref, server_files, artifact, exists=False): + pkg_files = [f for f in server_files if f.startswith(artifact)] + if len(pkg_files) > 1: + raise ConanException(f"{ref} is corrupted in the server, it contains " + f"more than one compressed file: {sorted(pkg_files)}") + if not pkg_files: + if not exists: + return None + raise ConanException(f"Recipe {ref} is corrupted in the server, it doesn't contain " + f"a {artifact} file") + return pkg_files[0] + def get_package(self, pref, dest_folder, metadata, only_metadata): url = self.router.package_snapshot(pref) data = self._get_file_list_json(url) @@ -277,8 +293,8 @@ def get_package(self, pref, dest_folder, metadata, only_metadata): result = {} # Download only known files, but not metadata (except sign) if not only_metadata: # Retrieve package first, then metadata - accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt", - "metadata/sign"] + pkg_file = self._find_compressed_file(pref, server_files, PACKAGE_FILE_NAME, exists=True) + accepted_files = [CONANINFO, pkg_file, CONAN_MANIFEST, "metadata/sign"] files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)] # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.package_file(pref, fn) for fn in files} diff --git a/conan/test/utils/test_files.py b/conan/test/utils/test_files.py index 7b3f011116d..7b099b064c0 100644 --- a/conan/test/utils/test_files.py +++ b/conan/test/utils/test_files.py @@ -12,7 +12,6 @@ from conan.tools.files.files import untargz from conan.internal.subsystems import get_cased_path from conan.errors import ConanException -from conan.internal.paths import PACKAGE_TGZ_NAME def wait_until_removed(folder): @@ -59,6 +58,7 @@ def uncompress_packaged_files(paths, pref): pref.revision = prev package_path = paths.package(pref) + PACKAGE_TGZ_NAME = "conan_package.tgz" if not(os.path.exists(os.path.join(package_path, PACKAGE_TGZ_NAME))): raise ConanException("%s not found in %s" % (PACKAGE_TGZ_NAME, package_path)) tmp = temp_folder() diff --git a/test/integration/command/cache/test_cache_save_restore.py b/test/integration/command/cache/test_cache_save_restore.py index a6bc91b495d..005c608e49f 100644 --- a/test/integration/command/cache/test_cache_save_restore.py +++ b/test/integration/command/cache/test_cache_save_restore.py @@ -2,6 +2,7 @@ import os import platform import shutil +import sys import tarfile import time @@ -14,7 +15,6 @@ from conan.internal.util.files import save, load - def test_cache_save_restore(): c = TestClient() c.save({"conanfile.py": GenConanfile().with_settings("os")}) @@ -199,7 +199,8 @@ def test_cache_save_restore_metadata(): # FIXME: check the timestamps of the conan cache restore -@pytest.mark.skipif(platform.system() == "Windows", reason="Fails in windows in ci because of the low precission of the clock") +@pytest.mark.skipif(platform.system() == "Windows", + reason="Fails in windows in ci because of the low precission of the clock") def test_cache_save_restore_multiple_revisions(): c = TestClient() c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) @@ -214,7 +215,6 @@ def test_cache_save_restore_multiple_revisions(): c.run("create .") rrev3 = c.exported_recipe_revision() - def check_ordered_revisions(client): client.run("list *#* --format=json") revisions = json.loads(client.stdout)["Local Cache"]["pkg/0.1"]["revisions"] @@ -293,3 +293,27 @@ def test_cache_save_restore_custom_storage_path(src_store, dst_store): c2.run("cache restore conan_cache_save.tgz") c2.run("list *:*") assert "pkg/1.0" in c2.out + + +@pytest.mark.parametrize("compress", ["gz", "xz", "zst"]) +def test_cache_save_restore_compressions(compress): + """ we accept different compressions formats""" + if compress == "zst" and sys.version_info.minor < 14: + pytest.skip("Skipping zst compression tests") + + conan_file = GenConanfile() \ + .with_settings("os") \ + .with_package_file("bin/file.txt", "content!!") + + client = TestClient() + client.save({"conanfile.py": conan_file}) + client.run("create . --name=pkg --version=1.0 -s os=Linux") + client.run(f"cache save pkg/*:* --file=mysave.t{compress}") + cache_path = os.path.join(client.current_folder, f"mysave.t{compress}") + assert os.path.exists(cache_path) + + c2 = TestClient() + shutil.copy2(cache_path, c2.current_folder) + c2.run(f"cache restore mysave.t{compress}") + c2.run("list *:*#*") + assert "pkg/1.0" in c2.out diff --git a/test/integration/command/upload/upload_test.py b/test/integration/command/upload/upload_test.py index d9eb6888caf..30de88a034d 100644 --- a/test/integration/command/upload/upload_test.py +++ b/test/integration/command/upload/upload_test.py @@ -12,7 +12,6 @@ from conan.errors import ConanException from conan.api.model import PkgReference from conan.internal.api.uploader import gzopen_without_timestamps -from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, PACKAGE_TGZ_NAME from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer, \ GenConanfile, TestRequester, TestingResponse from conan.internal.util.files import is_dirty, save, set_dirty, sha1sum @@ -129,7 +128,7 @@ def gzopen_patched(name, mode="r", fileobj=None, **kwargs): # noqa export_download_folder = layout.download_export() - tgz = os.path.join(export_download_folder, EXPORT_SOURCES_TGZ_NAME) + tgz = os.path.join(export_download_folder, "conan_sources.tgz") assert os.path.exists(tgz) assert is_dirty(tgz) @@ -147,7 +146,7 @@ def test_broken_package_tgz(self): pref = client.created_layout().reference def gzopen_patched(name, fileobj, compresslevel=None): # noqa - if name == PACKAGE_TGZ_NAME: + if name == "conan_package.tgz": raise ConanException("Error gzopen %s" % name) return gzopen_without_timestamps(name, fileobj) with patch('conan.internal.api.uploader.gzopen_without_timestamps', new=gzopen_patched): @@ -155,7 +154,7 @@ def gzopen_patched(name, fileobj, compresslevel=None): # noqa assert "Error gzopen conan_package.tgz" in client.out download_folder = client.get_latest_pkg_layout(pref).download_package() - tgz = os.path.join(download_folder, PACKAGE_TGZ_NAME) + tgz = os.path.join(download_folder, "conan_package.tgz") assert os.path.exists(tgz) assert is_dirty(tgz) diff --git a/test/integration/remote/rest_api_test.py b/test/integration/remote/rest_api_test.py index c9f8beb75c9..4d9f7d198dc 100644 --- a/test/integration/remote/rest_api_test.py +++ b/test/integration/remote/rest_api_test.py @@ -169,7 +169,8 @@ def _upload_package(self, package_reference, base_files=None): files = {"conanfile.py": GenConanfile("3").with_requires("1", "12").with_exports("*"), "hello.cpp": "hello", - "conanmanifest.txt": ""} + "conanmanifest.txt": "", + "conan_package.tgz": ""} if base_files: files.update(base_files) diff --git a/test/integration/symlinks/symlinks_test.py b/test/integration/symlinks/symlinks_test.py index 7bd09064c86..009cb783f65 100644 --- a/test/integration/symlinks/symlinks_test.py +++ b/test/integration/symlinks/symlinks_test.py @@ -6,7 +6,6 @@ from conan.api.model import PkgReference from conan.api.model import RecipeReference -from conan.internal.paths import PACKAGE_TGZ_NAME from conan.test.assets.genconanfile import GenConanfile from conan.test.utils.test_files import temp_folder from conan.test.utils.tools import TestClient, TestServer @@ -306,7 +305,7 @@ def package(self): client.run("upload * -r=default -c") # We can uncompress it without warns - tgz = os.path.join(p_folder, PACKAGE_TGZ_NAME) + tgz = os.path.join(p_folder, "conan_package.tgz") client.run_command('gzip -d "{}"'.format(tgz)) client.run_command('tar tvf "{}"'.format(os.path.join(p_folder, "conan_package.tar"))) lines = str(client.out).splitlines() diff --git a/test/integration/test_compressions.py b/test/integration/test_compressions.py new file mode 100644 index 00000000000..4813c074efd --- /dev/null +++ b/test/integration/test_compressions.py @@ -0,0 +1,144 @@ +import json +import os +import sys +import textwrap + +import pytest + +from conan.api.model import RecipeReference, PkgReference +from conan.internal.util import load +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +@pytest.mark.parametrize("compress", ["gz", "xz", "zst"]) +def test_xz(compress): + if compress == "zst" and sys.version_info.minor < 14: + pytest.skip("Skipping zst compression tests") + + c = TestClient(default_server_user=True) + c.save_home({"global.conf": f"core.upload:compression_format={compress}"}) + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import copy + + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + exports_sources = "*.h" + exports = "*.yml" + def package(self): + copy(self, "*.h", self.source_folder, self.package_folder) + """) + + c.save({"conanfile.py": conanfile, + "header.h": "myheader", + "myfile.yml": "myyml"}) + c.run("create -tf=") + c.run("upload * -r=default -c --format=json") + + # Verify the uploaded files are all txz + upload_json = json.loads(c.stdout) + rrev = upload_json["default"]["pkg/0.1"]["revisions"]["4e81a0b14da7ae918cf3dba3a07578d6"] + rfiles = rrev["files"] + assert f"conan_export.t{compress}" in rfiles + assert f"conan_sources.t{compress}" in rfiles + prevs = rrev["packages"]["da39a3ee5e6b4b0d3255bfef95601890afd80709"]["revisions"] + prev = prevs["13eb72928af98144fa7bf104b69663bc"] + pfiles = prev["files"] + assert f"conan_package.t{compress}" in pfiles + + # decompress should work anyway + c.save_home({"global.conf": ""}) + c.run("remove * -c") + c.run("install --requires=pkg/0.1") + + # checking the recipe + ref = RecipeReference.loads("pkg/0.1") + rlayout = c.get_latest_ref_layout(ref) + downloaded_files = os.listdir(rlayout.download_export()) + assert f"conan_export.t{compress}" in downloaded_files + assert f"conan_sources.t{compress}" not in downloaded_files + assert "myyml" == load(os.path.join(rlayout.export(), "myfile.yml")) + + # checking the package + pref = PkgReference(rlayout.reference, "da39a3ee5e6b4b0d3255bfef95601890afd80709") + playout = c.get_latest_pkg_layout(pref) + downloaded_files = os.listdir(playout.download_package()) + assert f"conan_package.t{compress}" in downloaded_files + assert "myheader" == load(os.path.join(playout.package(), "header.h")) + + # Force the build from source + c.run("install --requires=pkg/0.1 --build=*") + downloaded_files = os.listdir(rlayout.download_export()) + assert f"conan_export.t{compress}" in downloaded_files + assert f"conan_sources.t{compress}" in downloaded_files + + +@pytest.mark.skipif(sys.version_info.minor >= 14, reason="validate zstd error in python<314") +def test_unsupported_zstd(): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_package_file("myfile.h", "contents")}) + c.run("create") + playout = c.created_layout() + c.run("upload * -r=default -c -cc core.upload:compression_format=zst", assert_error=True) + assert "ERROR: The 'core.upload:compression_format=zst' is only for Python>=3.14" in c.out + + # Lets cheat, creating a fake zstd to test download + c.run("upload * -r=default -c --dry-run") + os.rename(os.path.join(playout.download_package(), "conan_package.tgz"), + os.path.join(playout.download_package(), "conan_package.tzst")) + c.run("upload * -r=default -c") + c.run("remove * -c") + c.run("install --requires=pkg/0.1", assert_error=True) + assert ("ERROR: File conan_package.tzst compressed with 'zst', unsupported " + "for Python<3.14") in c.out + + +class TestDuplicatedInServerErrors: + + def test_duplicated_export(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1"), + "conandata.yml": ""}) + c.run("export") + c.run("upload * -r=default -c") + c.run("remove * -c") + c.run("export") + c.run("upload * -r=default -c -cc core.upload:compression_format=xz --force") + assert ("WARN: risk: The xz compression is highly experimental, use it at your " + "own risk and expect issues" in c.out) + + c.run("remove * -c") + c.run("install --requires=pkg/0.1", assert_error=True) + assert ("it contains more than one compressed file: " + "['conan_export.tgz', 'conan_export.txz']") in c.out + + def test_duplicated_source(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_exports_sources("*.h"), + "myheader.h": "content"}) + c.run("export") + c.run("upload * -r=default -c") + c.run("remove * -c") + c.run("export") + c.run("upload * -r=default -c -cc core.upload:compression_format=xz --force") + + c.run("remove * -c") + c.run("install --requires=pkg/0.1 --build=missing", assert_error=True) + assert ("it contains more than one compressed file: " + "['conan_sources.tgz', 'conan_sources.txz']") in c.out + + def test_duplicated_package(self): + c = TestClient(default_server_user=True) + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create") + c.run("upload * -r=default -c") + c.run("remove * -c") + c.run("create") + c.run("upload * -r=default -c -cc core.upload:compression_format=xz --force") + + c.run("remove * -c") + c.run("install --requires=pkg/0.1", assert_error=True) + assert ("it contains more than one compressed file: " + "['conan_package.tgz', 'conan_package.txz']") in c.out diff --git a/test/integration/tgz_macos_dot_files_test.py b/test/integration/tgz_macos_dot_files_test.py index 8621be5890e..44340726c8f 100644 --- a/test/integration/tgz_macos_dot_files_test.py +++ b/test/integration/tgz_macos_dot_files_test.py @@ -9,7 +9,6 @@ from conan.internal.rest.remote_manager import uncompress_file from conan.api.model import RecipeReference -from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME from conan.test.utils.tools import TestClient, NO_SETTINGS_PACKAGE_ID @@ -94,7 +93,7 @@ def _add_macos_metadata_to_file(filepath): # 3) In the upload process, the metadata is lost again export_download_folder = t.get_latest_ref_layout(pref.ref).download_export() - tgz = os.path.join(export_download_folder, EXPORT_SOURCES_TGZ_NAME) + tgz = os.path.join(export_download_folder, "conan_sources.tgz") assert not os.path.exists(tgz) t.run("upload lib/version@user/channel -r default --only-recipe") self._test_for_metadata_in_zip_file(tgz, 'file.txt', dot_file_expected=False) diff --git a/test/unittests/client/remote_manager_test.py b/test/unittests/client/remote_manager_test.py index cbad3cba042..60f01384ea9 100644 --- a/test/unittests/client/remote_manager_test.py +++ b/test/unittests/client/remote_manager_test.py @@ -1,15 +1,32 @@ import os +import sys + import pytest from conan.internal.api.uploader import compress_files -from conan.internal.paths import PACKAGE_TGZ_NAME +from conan.internal.rest.remote_manager import uncompress_file from conan.test.utils.test_files import temp_folder from conan.internal.util.files import save class TestRemoteManager: - def test_compress_files(self): + def test_compress_files_tgz(self): + folder = temp_folder() + save(os.path.join(folder, "one_file.txt"), "The contents") + save(os.path.join(folder, "Two_file.txt"), "Two contents") + + files = { + "one_file.txt": os.path.join(folder, "one_file.txt"), + "Two_file.txt": os.path.join(folder, "Two_file.txt"), + } + + path = compress_files(files, "conan_package.tgz", dest_dir=folder) + assert os.path.exists(path) + expected_path = os.path.join(folder, "conan_package.tgz") + assert path == expected_path + + def test_compress_and_uncompress_xz_files(self): folder = temp_folder() save(os.path.join(folder, "one_file.txt"), "The contents") save(os.path.join(folder, "Two_file.txt"), "Two contents") @@ -19,7 +36,47 @@ def test_compress_files(self): "Two_file.txt": os.path.join(folder, "Two_file.txt"), } - path = compress_files(files, PACKAGE_TGZ_NAME, dest_dir=folder) + path = compress_files(files, "conan_package.txz", dest_dir=folder) assert os.path.exists(path) - expected_path = os.path.join(folder, PACKAGE_TGZ_NAME) + expected_path = os.path.join(folder, "conan_package.txz") assert path == expected_path + + extract_dir = os.path.join(folder, "extracted") + uncompress_file(path, extract_dir) + + extract_files = list(sorted(os.listdir(extract_dir))) + expected_files = sorted(files.keys()) + assert extract_files == expected_files + + for name, path in files.items(): + extract_path = os.path.join(extract_dir, name) + with open(path, "r") as f1, open(extract_path, "r") as f2: + assert f1.read() == f2.read() + + @pytest.mark.skipif(sys.version_info.minor < 14, reason="zstd needs Python >= 3.14") + def test_compress_and_uncompress_zst_files(self): + folder = temp_folder() + save(os.path.join(folder, "one_file.txt"), "The contents") + save(os.path.join(folder, "Two_file.txt"), "Two contents") + + files = { + "one_file.txt": os.path.join(folder, "one_file.txt"), + "Two_file.txt": os.path.join(folder, "Two_file.txt"), + } + + path = compress_files(files, "conan_package.tzst", dest_dir=folder) + assert os.path.exists(path) + expected_path = os.path.join(folder, "conan_package.tzst") + assert path == expected_path + + extract_dir = os.path.join(folder, "extracted") + uncompress_file(path, extract_dir) + + extract_files = list(sorted(os.listdir(extract_dir))) + expected_files = sorted(files.keys()) + assert extract_files == expected_files + + for name, path in sorted(files.items()): + extract_path = os.path.join(extract_dir, name) + with open(path, "r") as f1, open(extract_path, "r") as f2: + assert f1.read() == f2.read()