Feature/builtin compression

conan/internal/api/uploader.py

@@ -2,6 +2,7 @@
 import gzip
 import os
 import shutil
+import sys
 import tarfile
 import time
 
@@ -10,8 +11,8 @@
 from conan.internal.source import retrieve_exports_sources
 from conan.internal.errors import NotFoundException
 from conan.errors import ConanException
-from conan.internal.paths import (CONAN_MANIFEST, CONANFILE, EXPORT_SOURCES_TGZ_NAME,
-                                  EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME, CONANINFO)
+from conan.internal.paths import CONAN_MANIFEST, CONANFILE, CONANINFO, COMPRESSIONS, \
+    EXPORT_SOURCES_FILE_NAME, EXPORT_FILE_NAME, PACKAGE_FILE_NAME
 from conan.internal.util.files import (clean_dirty, is_dirty, gather_files,
                                        set_dirty_context_manager, mkdir, human_size)
 
@@ -85,6 +86,21 @@ def __init__(self, app: ConanApp, global_conf):
         self._app = app
         self._global_conf = global_conf
 
+        # No compressed file exists, need to compress
+        compressformat = self._global_conf.get("core.upload:compression_format",
+                                               default="gz", choices=COMPRESSIONS)
+        if compressformat == "zst" and sys.version_info.minor < 14:
+            raise ConanException("The 'core.upload:compression_format=zst' is only for Python>=3.14")
+        compresslevel = self._global_conf.get("core:compresslevel", check_type=int)
+        if compresslevel is None and compressformat == "gz":
+            compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
+            if compresslevel is not None:
+                ConanOutput().warning("core.gzip:compresslevel is deprecated, "
+                                      "use core.compresslevel instead", warn_tag="deprecated")
+
+        self._compressformat = compressformat
+        self._compresslevel = compresslevel
+
     def prepare(self, pkg_list, enabled_remotes):
         local_url = self._global_conf.get("core.scm:local_url", choices=["allow", "block"])
         for ref, packages in pkg_list.items():
@@ -128,14 +144,6 @@ def _prepare_recipe(self, ref, ref_bundle, conanfile, remotes):
     def _compress_recipe_files(self, layout, ref):
         download_export_folder = layout.download_export()
 
-        output = ConanOutput(scope=str(ref))
-        for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
-            tgz_path = os.path.join(download_export_folder, f)
-            if is_dirty(tgz_path):
-                output.warning("Removing %s, marked as dirty" % f)
-                os.remove(tgz_path)
-                clean_dirty(tgz_path)
-
         export_folder = layout.export()
         files, symlinked_folders = gather_files(export_folder)
         files.update(symlinked_folders)
@@ -159,18 +167,13 @@ def _compress_recipe_files(self, layout, ref):
         files.pop(CONANFILE)
         files.pop(CONAN_MANIFEST)
 
-        def add_tgz(tgz_name, tgz_files):
-            tgz = os.path.join(download_export_folder, tgz_name)
-            if os.path.isfile(tgz):
-                result[tgz_name] = tgz
-            elif tgz_files:
-                compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
-                tgz = compress_files(tgz_files, tgz_name, download_export_folder,
-                                     compresslevel=compresslevel, ref=ref)
-                result[tgz_name] = tgz
-
-        add_tgz(EXPORT_TGZ_NAME, files)
-        add_tgz(EXPORT_SOURCES_TGZ_NAME, src_files)
+        if files:
+            comp = self._compressed_file(EXPORT_FILE_NAME, files, download_export_folder, ref)
+            result[comp] = os.path.join(download_export_folder, comp)
+        if src_files:
+            comp = self._compressed_file(EXPORT_SOURCES_FILE_NAME, src_files,
+                                         download_export_folder, ref)
+            result[comp] = os.path.join(download_export_folder, comp)
         return result
 
     def _prepare_package(self, pref, prev_bundle):
@@ -181,14 +184,36 @@ def _prepare_package(self, pref, prev_bundle):
         cache_files = self._compress_package_files(pkg_layout, pref)
         prev_bundle["files"] = cache_files
 
+    def _compressed_file(self, filename, files, download_folder, ref):
+        output = ConanOutput(scope=str(ref))
+
+        # Check if there is some existing compressed file already
+        matches = []
+        for extension in COMPRESSIONS:
+            file_name = filename + extension
+            package_file = os.path.join(download_folder, file_name)
+            if is_dirty(package_file):
+                output.warning(f"Removing {file_name}, marked as dirty")
+                os.remove(package_file)
+                clean_dirty(package_file)
+            if os.path.isfile(package_file):
+                matches.append(file_name)
+        if len(matches) > 1:
+            raise ConanException(f"{ref}: Multiple package files found for {filename}: {matches}")
+        if len(matches) == 1:
+            return matches[0]
+
+        file_name = filename + self._compressformat
+        package_file = os.path.join(download_folder, file_name)
+        compressed_path = compress_files(files, file_name, download_folder,
+                                         compresslevel=self._compresslevel,
+                                         compressformat=self._compressformat, ref=ref)
+        assert compressed_path == package_file
+        assert os.path.exists(package_file)
+        return file_name
+
     def _compress_package_files(self, layout, pref):
-        output = ConanOutput(scope=str(pref))
         download_pkg_folder = layout.download_package()
-        package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME)
-        if is_dirty(package_tgz):
-            output.warning("Removing %s, marked as dirty" % PACKAGE_TGZ_NAME)
-            os.remove(package_tgz)
-            clean_dirty(package_tgz)
 
         # Get all the files in that directory
         # existing package
@@ -209,15 +234,8 @@ def _compress_package_files(self, layout, pref):
         files.pop(CONANINFO)
         files.pop(CONAN_MANIFEST)
 
-        if not os.path.isfile(package_tgz):
-            tgz_files = {f: path for f, path in files.items()}
-            compresslevel = self._global_conf.get("core.gzip:compresslevel", check_type=int)
-            tgz_path = compress_files(tgz_files, PACKAGE_TGZ_NAME, download_pkg_folder,
-                                      compresslevel=compresslevel, ref=pref)
-            assert tgz_path == package_tgz
-            assert os.path.exists(package_tgz)
-
-        return {PACKAGE_TGZ_NAME: package_tgz,
+        compressed_file = self._compressed_file(PACKAGE_FILE_NAME, files, download_pkg_folder, pref)
+        return {compressed_file: os.path.join(download_pkg_folder, compressed_file),
                 CONANINFO: os.path.join(download_pkg_folder, CONANINFO),
                 CONAN_MANIFEST: os.path.join(download_pkg_folder, CONAN_MANIFEST)}
 
@@ -282,12 +300,26 @@ def gzopen_without_timestamps(name, fileobj, compresslevel=None):
     return t
 
 
-def compress_files(files, name, dest_dir, compresslevel=None, ref=None, recursive=False):
+def compress_files(files, name, dest_dir, compressformat=None, compresslevel=None, ref=None,
+                   recursive=False):
     t1 = time.time()
     # FIXME, better write to disk sequentially and not keep tgz contents in memory
     tgz_path = os.path.join(dest_dir, name)
     if ref:
         ConanOutput(scope=str(ref) if ref else None).info(f"Compressing {name}")
+
+    if compressformat == "zst":
+        with tarfile.open(tgz_path, "w:zst", level=compresslevel) as tar:  # noqa Py314 only
+            for filename, abs_path in sorted(files.items()):
+                tar.add(abs_path, filename, recursive=recursive)
+        return tgz_path
+
+    if compressformat == "xz":
+        with tarfile.open(tgz_path, "w:xz", preset=compresslevel, format=tarfile.PAX_FORMAT) as tar:
+            for filename, abs_path in sorted(files.items()):
+                tar.add(abs_path, filename, recursive=recursive)
+        return tgz_path
+
     with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle:
         tgz = gzopen_without_timestamps(name, fileobj=tgz_handle, compresslevel=compresslevel)
         for filename, abs_path in sorted(files.items()):

conan/internal/model/conf.py

@@ -60,8 +60,11 @@
     "core.net.http:cacert_path": "Path containing a custom Cacert file",
     "core.net.http:client_cert": "Path or tuple of files containing a client cert (and key)",
     "core.net.http:clean_system_proxy": "If defined, the proxies system env-vars will be discarded",
-    # Gzip compression
+    # Compression for `conan upload`
+    "core.upload:compression_format": "The compression format used when uploading Conan packages. "
+                                      "Possible values: 'zst', 'xz', 'gz' (default=gz)",
     "core.gzip:compresslevel": "The Gzip compression level for Conan artifacts (default=9)",
+    "core:compresslevel": "The compression level for Conan artifacts (default zstd=3, gz=9)",
     # Excluded from revision_mode = "scm" dirty and Git().is_dirty() checks
     "core.scm:excluded": "List of excluded patterns for builtin git dirty checks",
     "core.scm:local_url": "By default allows to store local folders as remote url, but not upload them. Use 'allow' for allowing upload and 'block' to completely forbid it",

conan/internal/model/manifest.py

@@ -1,7 +1,8 @@
 import os
 from collections import defaultdict
 
-from conan.internal.paths import CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
+from conan.internal.paths import CONAN_MANIFEST, COMPRESSIONS, PACKAGE_FILE_NAME, EXPORT_FILE_NAME, \
+    EXPORT_SOURCES_FILE_NAME
 from conan.internal.util.dates import timestamp_now, timestamp_to_str
 from conan.internal.util.files import load, md5, md5sum, save, gather_files
 
@@ -91,8 +92,10 @@ def create(cls, folder, exports_sources_folder=None):
         """
         files, _ = gather_files(folder)
         # The folders symlinks are discarded for the manifest
-        for f in (PACKAGE_TGZ_NAME, EXPORT_TGZ_NAME, CONAN_MANIFEST, EXPORT_SOURCES_TGZ_NAME):
-            files.pop(f, None)
+        for f in (PACKAGE_FILE_NAME, EXPORT_FILE_NAME, EXPORT_SOURCES_FILE_NAME):
+            for e in COMPRESSIONS:
+                files.pop(f + e, None)
+        files.pop(CONAN_MANIFEST, None)
 
         file_dict = {}
         for name, filepath in files.items():

conan/internal/paths.py

@@ -86,7 +86,8 @@ def _user_home_from_conanrc_file():
 CONANFILE_TXT = "conanfile.txt"
 CONAN_MANIFEST = "conanmanifest.txt"
 CONANINFO = "conaninfo.txt"
-PACKAGE_TGZ_NAME = "conan_package.tgz"
-EXPORT_TGZ_NAME = "conan_export.tgz"
-EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz"
+PACKAGE_FILE_NAME = "conan_package.t"
+EXPORT_FILE_NAME = "conan_export.t"
+EXPORT_SOURCES_FILE_NAME = "conan_sources.t"
+COMPRESSIONS = "gz", "xz", "zst"
 DATA_YML = "conandata.yml"

conan/internal/rest/remote_manager.py

@@ -1,11 +1,14 @@
 import os
 import shutil
+import sys
+
 from collections import namedtuple
 from typing import List
 
 from requests.exceptions import ConnectionError
 
 from conan.api.model import LOCAL_RECIPES_INDEX
+from conan.internal.paths import CONANINFO, CONAN_MANIFEST, PACKAGE_FILE_NAME, EXPORT_FILE_NAME
 from conan.internal.rest.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex
 from conan.api.model import Remote
 from conan.api.output import ConanOutput
@@ -17,7 +20,6 @@
 from conan.api.model import PkgReference
 from conan.api.model import RecipeReference
 from conan.internal.util.files import rmdir, human_size
-from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME
 from conan.internal.util.files import mkdir, tar_extract
 
 
@@ -86,7 +88,8 @@ def get_recipe(self, ref, remote, metadata=None):
             self._cache.remove_recipe_layout(layout)
             raise
         export_folder = layout.export()
-        tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
+        export_file = next((f for f in zipped_files if EXPORT_FILE_NAME in f), None)
+        tgz_file = zipped_files.pop(export_file, None)
 
         if tgz_file:
             uncompress_file(tgz_file, export_folder, scope=str(ref))
@@ -132,7 +135,8 @@ def get_recipe_sources(self, ref, layout, remote):
             return
 
         self._signer.verify(ref, download_folder, files=zipped_files)
-        tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME]
+        # Only 1 file is guaranteed
+        tgz_file = next(iter(zipped_files.values()))
         uncompress_file(tgz_file, export_sources_folder, scope=str(ref))
 
     def get_package(self, pref, remote, metadata=None):
@@ -178,12 +182,17 @@ def _get_package(self, layout, pref, remote, scoped_output, metadata):
                                              metadata, only_metadata=False)
             zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)}
             # quick server package integrity check:
-            for f in ("conaninfo.txt", "conanmanifest.txt", "conan_package.tgz"):
+            for f in (CONANINFO, CONAN_MANIFEST):
                 if f not in zipped_files:
                     raise ConanException(f"Corrupted {pref} in '{remote.name}' remote: no {f}")
+
+            package_file = next((f for f in zipped_files if PACKAGE_FILE_NAME in f), None)
+            if not package_file:
+                raise ConanException(f"Corrupted {pref} in '{remote.name}' remote: "
+                                     f"no conan_package found")
             self._signer.verify(pref, download_pkg_folder, zipped_files)
 
-            tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None)
+            tgz_file = zipped_files.pop(package_file)
             package_folder = layout.package()
             uncompress_file(tgz_file, package_folder, scope=str(pref.ref))
             mkdir(package_folder)  # Just in case it doesn't exist, because uncompress did nothing
@@ -337,6 +346,9 @@ def _call_remote(self, remote, method, *args, **kwargs):
 
 
 def uncompress_file(src_path, dest_folder, scope=None):
+    if sys.version_info.major < 14 and src_path.endswith(".tzst"):
+        raise ConanException(f"File {os.path.basename(src_path)} compressed with 'zst', "
+                             f"unsupported for Python<3.14 ")
     try:
         filesize = os.path.getsize(src_path)
         big_file = filesize > 10000000  # 10 MB

conan/internal/rest/rest_client_v2.py

@@ -9,7 +9,8 @@
 from uuid import getnode as get_mac
 
 from conan.api.output import ConanOutput
-
+from conan.internal.paths import EXPORT_SOURCES_FILE_NAME, CONANINFO, CONAN_MANIFEST, \
+    EXPORT_FILE_NAME
 from conan.internal.rest.caching_file_downloader import ConanInternalCacheDownloader
 from conan.internal.rest import response_to_str
 from conan.internal.rest.client_routes import ClientV2Router
@@ -18,7 +19,6 @@
     RecipeNotFoundException, PackageNotFoundException, EXCEPTION_CODE_MAPPING
 from conan.errors import ConanException
 from conan.api.model import PkgReference
-from conan.internal.paths import EXPORT_SOURCES_TGZ_NAME
 from conan.api.model import RecipeReference
 from conan.internal.util.dates import from_iso8601_to_timestamp
 
@@ -239,8 +239,7 @@ def get_recipe(self, ref, dest_folder, metadata, only_metadata):
         result = {}
 
         if not only_metadata:
-            accepted_files = ["conanfile.py", "conan_export.tgz", "conanmanifest.txt",
-                              "metadata/sign"]
+            accepted_files = ["conanfile.py", EXPORT_FILE_NAME, CONAN_MANIFEST,  "metadata/sign"]
             files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)]
             # If we didn't indicated reference, server got the latest, use absolute now, it's safer
             urls = {fn: self.router.recipe_file(ref, fn) for fn in files}
@@ -260,9 +259,10 @@ def get_recipe_sources(self, ref, dest_folder):
         url = self.router.recipe_snapshot(ref)
         data = self._get_file_list_json(url)
         files = data["files"]
-        if EXPORT_SOURCES_TGZ_NAME not in files:
+        sources_file = next((f for f in files if EXPORT_SOURCES_FILE_NAME in f), None)
+        if sources_file is None:
             return None
-        files = [EXPORT_SOURCES_TGZ_NAME, ]
+        files = [sources_file, ]
 
         # If we didn't indicated reference, server got the latest, use absolute now, it's safer
         urls = {fn: self.router.recipe_file(ref, fn) for fn in files}
@@ -275,10 +275,15 @@ def get_package(self, pref, dest_folder, metadata, only_metadata):
         data = self._get_file_list_json(url)
         server_files = data["files"]
         result = {}
+        pkg_files = [f for f in server_files if f.startswith("conan_package.")]
+        if len(pkg_files) > 1:
+            raise ConanException(f"Package {pref} is corrupted in the server, it contains "
+                                 f"more than one package file: {pkg_files}")
         # Download only known files, but not metadata (except sign)
         if not only_metadata:  # Retrieve package first, then metadata
-            accepted_files = ["conaninfo.txt", "conan_package.tgz", "conanmanifest.txt",
-                              "metadata/sign"]
+            accepted_files = [CONANINFO, CONAN_MANIFEST, "metadata/sign"]
+            if len(pkg_files) == 1:
+                accepted_files.append(pkg_files[0])
             files = [f for f in server_files if any(f.startswith(m) for m in accepted_files)]
             # If we didn't indicated reference, server got the latest, use absolute now, it's safer
             urls = {fn: self.router.package_file(pref, fn) for fn in files}

conan/test/utils/test_files.py

@@ -12,7 +12,6 @@
 from conan.tools.files.files import untargz
 from conan.internal.subsystems import get_cased_path
 from conan.errors import ConanException
-from conan.internal.paths import PACKAGE_TGZ_NAME
 
 
 def wait_until_removed(folder):
@@ -59,6 +58,7 @@ def uncompress_packaged_files(paths, pref):
     pref.revision = prev
 
     package_path = paths.package(pref)
+    PACKAGE_TGZ_NAME = "conan_package.tgz"
     if not(os.path.exists(os.path.join(package_path, PACKAGE_TGZ_NAME))):
         raise ConanException("%s not found in %s" % (PACKAGE_TGZ_NAME, package_path))
     tmp = temp_folder()