create --paths-from-shell-command, fixes #5968

This adds the `--paths-from-shell-command` option to the `create` command, enabling the use of shell-specific features like pipes and redirection when specifying input paths. Includes related test coverage.

Author: ThomasWaldmann

src/borg/archiver/create_cmd.py

@@ -91,13 +91,24 @@ def create_inner(archive, cache, fso):
                 else:
                     status = "+"  # included
                 self.print_file_status(status, path)
-            elif args.paths_from_command or args.paths_from_stdin:
+            elif args.paths_from_command or args.paths_from_shell_command or args.paths_from_stdin:
                 paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else "\n"
-                if args.paths_from_command:
+                if args.paths_from_command or args.paths_from_shell_command:
                     try:
                         env = prepare_subprocess_env(system=True)
+                        if args.paths_from_shell_command:
+                            # Use shell=True to support pipes, redirection, etc.
+                            shell = True
+                            cmd = " ".join(args.paths)
+                        else:
+                            shell = False
+                            cmd = args.paths
                         proc = subprocess.Popen(  # nosec B603
-                            args.paths, stdout=subprocess.PIPE, env=env, preexec_fn=None if is_win32 else ignore_sigint
+                            cmd,
+                            stdout=subprocess.PIPE,
+                            env=env,
+                            shell=shell,
+                            preexec_fn=None if is_win32 else ignore_sigint,
                         )
                     except (FileNotFoundError, PermissionError) as e:
                         raise CommandError(f"Failed to execute command: {e}")
@@ -131,7 +142,7 @@ def create_inner(archive, cache, fso):
                     self.print_file_status(status, path)
                     if not dry_run and status is not None:
                         fso.stats.files_stats[status] += 1
-                if args.paths_from_command:
+                if args.paths_from_command or args.paths_from_shell_command:
                     rc = proc.wait()
                     if rc != 0:
                         raise CommandError(f"Command {args.paths[0]!r} exited with status {rc}")
@@ -844,6 +855,11 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
             action="store_true",
             help="interpret PATH as command and treat its output as ``--paths-from-stdin``",
         )
+        subparser.add_argument(
+            "--paths-from-shell-command",
+            action="store_true",
+            help="interpret PATH as a shell command (be careful!) and treat its output as ``--paths-from-stdin``",
+        )
         subparser.add_argument(
             "--paths-delimiter",
             action=Highlander,

src/borg/testsuite/archiver/create_cmd_test.py

@@ -418,6 +418,21 @@ def test_create_paths_from_command_missing_command(archivers, request):
     assert output.endswith("No command given." + os.linesep)
 
 
+@pytest.mark.skipif(is_win32, reason="shell patterns not supported on Windows")
+def test_create_paths_from_shell_command(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    create_regular_file(archiver.input_path, "file1", size=1024 * 80)
+    create_regular_file(archiver.input_path, "file2", size=1024 * 80)
+    create_regular_file(archiver.input_path, "file3", size=1024 * 80)
+    input_data = "input/file1\ninput/file2\ninput/file3"
+    # Use a shell pipe to test that shell=True works correctly.
+    cmd(archiver, "create", "--paths-from-shell-command", "test", "--", f"echo '{input_data}' | head -n 2")
+    archive_list = cmd(archiver, "list", "test", "--json-lines")
+    paths = [json.loads(line)["path"] for line in archive_list.split("\n") if line]
+    assert paths == ["input/file1", "input/file2"]
+
+
 def test_create_without_root(archivers, request):
     """test create without a root"""
     archiver = request.getfixturevalue(archivers)