publish async again...

IsaacCalligeros95 · IsaacCalligeros95 · commit bb10d2d210e3 · 2025-05-27T09:35:32.000+09:30
diff --git a/build/Build.cs b/build/Build.cs
@@ -272,8 +272,21 @@ async Task PublishCalamariProjects(List<Project> projects)
                                 .Distinct(t => new { t.Project?.Name, t.Architecture, t.Framework });
 
             var packagesToPublish = crossPlatformPackages.Concat(netFxPackages).ToArray();
+            
+            var publishTasks = packagesToPublish
+                               .Select(package => PublishPackageAsync(package))
+                               .ToList();
 
-            packagesToPublish.ForEach(PublishPackage);
+            var outputPaths = await Task.WhenAll(publishTasks);
+
+            var signTasks = outputPaths
+                            .Where(output => output != null && !output.ToString().Contains("Tests"))
+                            .Select(output => Task.Run(() => SignDirectory(output!)))
+                            .ToList();
+
+            await Task.WhenAll(signTasks);
+            
+            
             await Task.WhenAll(SignDirectoriesTasks);
 
             StageLegacyCalamariAssemblies(packagesToPublish);
@@ -282,11 +295,39 @@ async Task PublishCalamariProjects(List<Project> projects)
             await Task.WhenAll(ProjectCompressionTasks);
         }
 
+async Task<AbsolutePath?> PublishPackageAsync(CalamariPackageMetadata calamariPackageMetadata)
+{
+    if (!OperatingSystem.IsWindows() && !calamariPackageMetadata.IsCrossPlatform)
+    {
+        Log.Warning($"Not publishing {calamariPackageMetadata.Framework}: can only publish netfx on a Windows OS");
+        return null;
+    }
+
+    Log.Information($"Publishing {calamariPackageMetadata.Project?.Name} for framework '{calamariPackageMetadata.Framework}' and arch '{calamariPackageMetadata.Architecture}'");
+
+    var project = calamariPackageMetadata.Project;
+    var outputDirectory = PublishDirectory / project?.Name / (calamariPackageMetadata.IsCrossPlatform ? calamariPackageMetadata.Architecture : "netfx");
+
+    await Task.Run(() =>
+        DotNetPublish(s =>
+            s.SetConfiguration(Configuration)
+             .SetProject(project)
+             .SetFramework(calamariPackageMetadata.Framework)
+             .SetRuntime(calamariPackageMetadata.Architecture)
+             .EnableDisableParallel()
+             .EnableSelfContained()
+             .SetOutput(outputDirectory)));
+
+    File.Copy(RootDirectory / "global.json", outputDirectory / "global.json");
+
+    return outputDirectory;
+}
+
         void PublishPackage(CalamariPackageMetadata calamariPackageMetadata)
         {
             if (!OperatingSystem.IsWindows() && !calamariPackageMetadata.IsCrossPlatform)
             {
-                Log.Warning($"Not building {calamariPackageMetadata.Framework}: can only build netfx on a Windows OS");
+                Log.Warning($"Not publishing {calamariPackageMetadata.Framework}: can only build netfx on a Windows OS");
                 return;
             }
 
@@ -300,6 +341,7 @@ void PublishPackage(CalamariPackageMetadata calamariPackageMetadata)
                                .SetProject(project)
                                .SetFramework(calamariPackageMetadata.Framework)
                                .SetRuntime(calamariPackageMetadata.Architecture)
+                               .EnableDisableParallel()
                                //explicitly mark all of these as self-contained (because they use a specific runtime)
                                .EnableSelfContained()
                                .SetOutput(outputDirectory)
@@ -597,16 +639,8 @@ void CompressCalamariProject(Project project)
 
         async Task RunPackActions(List<Action> actions)
         {
-            if (PackInParallel)
-            {
                 var tasks = actions.Select(Task.Run).ToList();
                 await Task.WhenAll(tasks);
-            }
-            else
-            {
-                foreach (var action in actions)
-                    action();
-            }
         }
 
         AbsolutePath DoPublish(string project, string framework, string version, string? runtimeId = null)
diff --git a/source/Calamari.Common/Features/Scripting/Bash/Bootstrap.sh b/source/Calamari.Common/Features/Scripting/Bash/Bootstrap.sh
@@ -265,6 +265,24 @@ function log_environment_information
 
 log_environment_information
 
+function hex_to_bin_fd() {
+    local hex="$1"
+    local fd="$2"
+    
+    # Process hex in reasonable chunks
+    for ((i=0; i<${#hex}; i+=40)); do
+        local chunk="${hex:$i:40}"
+        local cmd="printf '"
+        
+        for ((j=0; j<${#chunk}; j+=2)); do
+            cmd+="\\x${chunk:$j:2}"
+        done
+        
+        cmd+="'"
+        eval "$cmd" >&$fd
+    done
+}
+
 function decrypt_and_parse_variables {
     local encrypted="$1"
     local iv="$2"
@@ -287,7 +305,9 @@ function decrypt_and_parse_variables {
     local concatenated_hex
     concatenated_hex=$(printf "%s" "${hex_parts[@]}")
 
-    exec 3< <(echo -n "$concatenated_hex" | xxd -r -p)
+    exec 3<> <(:)
+    hex_to_bin_fd "$concatenated_hex" 3
+    exec 3<&3
 
     local idx
     for idx in "${!key_byte_lengths[@]}"; do
diff --git a/source/Calamari.Tests/Fixtures/Bash/BashFixture.cs b/source/Calamari.Tests/Fixtures/Bash/BashFixture.cs
@@ -2,12 +2,15 @@
 using System.Collections.Generic;
 using System.IO;
 using Calamari.Common.Features.Processes;
+using System.Diagnostics;
+using System.Linq;
 using Calamari.Common.FeatureToggles;
 using Calamari.Common.Plumbing;
 using Calamari.Common.Plumbing.Variables;
 using Calamari.Deployment;
 using Calamari.Testing.Requirements;
 using Calamari.Tests.Helpers;
+using FluentAssertions;
 using NUnit.Framework;
 
 namespace Calamari.Tests.Fixtures.Bash
@@ -34,7 +37,7 @@ public void ShouldPrintEncodedVariable(FeatureToggle? featureToggle)
         [RequiresBashDotExeIfOnWindows]
         public void ShouldPrintSensitiveVariable(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("print-sensitive-variable.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("print-sensitive-variable.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -48,7 +51,7 @@ public void ShouldPrintSensitiveVariable(FeatureToggle? featureToggle)
         [RequiresBashDotExeIfOnWindows]
         public void ShouldCreateArtifact(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("create-artifact.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("create-artifact.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -62,7 +65,7 @@ public void ShouldCreateArtifact(FeatureToggle? featureToggle)
         [RequiresBashDotExeIfOnWindows]
         public void ShouldUpdateProgress(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("update-progress.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("update-progress.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -160,7 +163,7 @@ public void ShouldConsumeParametersWithQuotes(FeatureToggle? featureToggle)
         {
             var (output, _) = RunScript("parameters.sh",
                                         new Dictionary<string, string>()
-                                            { [SpecialVariables.Action.Script.ScriptParameters] = "\"Para meter0\" 'Para meter1'" }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                            { [SpecialVariables.Action.Script.ScriptParameters] = "\"Para meter0\" 'Para meter1'" }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -174,8 +177,10 @@ public void ShouldConsumeParametersWithQuotes(FeatureToggle? featureToggle)
         [RequiresBashDotExeIfOnWindows]
         public void ShouldNotReceiveParametersIfNoneProvided(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("parameters.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }), sensitiveVariablesPassword:
-            "5XETGOgqYR2bRhlfhDruEg==");
+            var (output, _) = RunScript("parameters.sh",
+                                        new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }),
+                                        sensitiveVariablesPassword:
+                                        "5XETGOgqYR2bRhlfhDruEg==");
 
             Assert.Multiple(() =>
                             {
@@ -197,7 +202,7 @@ public void ShouldCallHello(FeatureToggle? featureToggle)
                                             ["Variable3"] = "GHI",
                                             ["Foo_bar"] = "Hello",
                                             ["Host"] = "Never",
-                                        }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                        }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -213,8 +218,9 @@ public void ShouldCallHelloWithSensitiveVariable(FeatureToggle? featureToggle)
         {
             var (output, _) = RunScript("hello.sh",
                                         new Dictionary<string, string>()
-                                            { ["Name"] = "NameToEncrypt" }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }), sensitiveVariablesPassword:
-            "5XETGOgqYR2bRhlfhDruEg==");
+                                            { ["Name"] = "NameToEncrypt" }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }),
+                                        sensitiveVariablesPassword:
+                                        "5XETGOgqYR2bRhlfhDruEg==");
 
             Assert.Multiple(() =>
                             {
@@ -230,7 +236,7 @@ public void ShouldCallHelloWithNullVariable(FeatureToggle? featureToggle)
         {
             var (output, _) = RunScript("hello.sh",
                                         new Dictionary<string, string>()
-                                            { ["Name"] = null }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                            { ["Name"] = null }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -246,8 +252,9 @@ public void ShouldCallHelloWithNullSensitiveVariable(FeatureToggle? featureToggl
         {
             var (output, _) = RunScript("hello.sh",
                                         new Dictionary<string, string>()
-                                            { ["Name"] = null }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }), sensitiveVariablesPassword:
-            "5XETGOgqYR2bRhlfhDruEg==");
+                                            { ["Name"] = null }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }),
+                                        sensitiveVariablesPassword:
+                                        "5XETGOgqYR2bRhlfhDruEg==");
 
             Assert.Multiple(() =>
                             {
@@ -262,7 +269,7 @@ public void ShouldCallHelloWithNullSensitiveVariable(FeatureToggle? featureToggl
         public void ShouldNotFailOnStdErr(FeatureToggle? featureToggle)
         {
             var (output, _) = RunScript("stderr.sh",
-                                        new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                        new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -278,7 +285,7 @@ public void ShouldFailOnStdErrWithTreatScriptWarningsAsErrors(FeatureToggle? fea
         {
             var (output, _) = RunScript("stderr.sh",
                                         new Dictionary<string, string>()
-                                            { [SpecialVariables.Action.FailScriptOnErrorOutput] = "True" }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                            { [SpecialVariables.Action.FailScriptOnErrorOutput] = "True" }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -294,7 +301,7 @@ public void ShouldNotFailOnStdErrFromServiceMessagesWithTreatScriptWarningsAsErr
         {
             var (output, _) = RunScript("hello.sh",
                                         new Dictionary<string, string>()
-                                            { [SpecialVariables.Action.FailScriptOnErrorOutput] = "True" }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                            { [SpecialVariables.Action.FailScriptOnErrorOutput] = "True" }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             output.AssertSuccess();
         }
@@ -304,7 +311,7 @@ public void ShouldNotFailOnStdErrFromServiceMessagesWithTreatScriptWarningsAsErr
         [TestCase(null)]
         public void ShouldSupportStrictVariableUnset(FeatureToggle? featureToggle)
         {
-            var (output, _) = RunScript("strict-mode.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+            var (output, _) = RunScript("strict-mode.sh", new Dictionary<string, string>().AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             Assert.Multiple(() =>
                             {
@@ -313,7 +320,7 @@ public void ShouldSupportStrictVariableUnset(FeatureToggle? featureToggle)
                             });
         }
 
-        static string specialCharacters => "! \" # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \\ ] ^ _ ` { | } ~  \n\u00b1 \u00d7 \u00f7 \u2211 \u220f \u2202 \u221e \u222b \u2248 \u2260 \u2264 \u2265 \u221a \u221b \u2206 \u2207 \u221d  \n$ \u00a2 \u00a3 \u00a5 \u20ac \u20b9 \u20a9 \u20b1 \u20aa \u20bf  \n• ‣ … ′ ″ ‘ ’ “ ” ‽ ¡ ¿ – — ―  \n( ) [ ] { } ⟨ ⟩ « » ‘ ’ “ ”  \n\u2190 \u2191 \u2192 \u2193 \u2194 \u2195 \u2196 \u2197 \u2198 \u2199 \u2b05 \u2b06 \u2b07 \u27a1 \u27f3  \nα β γ δ ε ζ η θ ι κ λ μ ν ξ ο π ρ σ τ υ φ χ ψ ω  \n\u00a9 \u00ae \u2122 § ¶ † ‡ µ #\n";
+        static string specialCharacters => "! \" # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \\ ] ^ _ ` { | } ~  \n\u00b1 \u00d7 \u00f7 \u2211 \u220f \u2202 \u221e \u222b \u2248 \u2260 \u2264 \u2265 \u221a \u221b \u2206 \u2207 \u221d  \n$ \u00a2 \u00a3 \u00a5 \u20ac \u20b9 \u20a9 \u20b1 \u20aa \u20bf  \n• ‣ … ′ ″ ‘ ’ “ ” ‽ ¡ ¿ – — ―  \n( ) [ ] { } ⟨ ⟩ « » ‘ ’ “ ”  \n\u2190 \u2191 \u2192 \u2193 \u2194 \u2195 \u2196 \u2197 \u2198 \u2199 \u2b05 \u2b06 \u2b07 \u27a1 \u27f3  \nα β γ δ ε ζ η θ ι κ λ μ ν ξ ο π ρ σ τ υ φ χ ψ ω  \n\u00a9 \u00ae \u2122 § ¶ † ‡ µ #";
 
         [TestCase(FeatureToggle.BashParametersArrayFeatureToggle)]
         [TestCase(null)]
@@ -336,8 +343,9 @@ public void ShouldBeAbleToEnumerateVariableValues(FeatureToggle? featureToggle)
                                             ["VariableName \n 11"] = "Value \n 11",
                                             ["VariableName.prop.anotherprop 12"] = "Value.prop.12",
                                             ["VariableName`prop`anotherprop` 13"] = "Value`prop`13",
+                                            ["VariableName 14 😭🙈👀"] = "Value 14 😭🙈👀",
                                             [specialCharacters] = specialCharacters
-                                        }.AddFeatureToggleToDictionary(new List<FeatureToggle?>{ featureToggle }));
+                                        }.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
 
             output.AssertSuccess();
             if (featureToggle == FeatureToggle.BashParametersArrayFeatureToggle)
@@ -366,9 +374,70 @@ public void ShouldBeAbleToEnumerateVariableValues(FeatureToggle? featureToggle)
 
                 output.AssertOutput("Key: VariableName.prop.anotherprop 12, Value: Value.prop.12");
                 output.AssertOutput("Key: VariableName`prop`anotherprop` 13, Value: Value`prop`13");
+                output.AssertOutput("Key: VariableName 14 😭🙈👀, Value: Value 14 😭🙈👀");
                 output.AssertOutput($"Key: {specialCharacters}, Value: {specialCharacters}");
             }
         }
+    [TestCase(FeatureToggle.BashParametersArrayFeatureToggle)]
+    [TestCase(null)]
+    [RequiresBashDotExeIfOnWindows]
+    public void ShouldBeAbleToEnumerateLargeVariableSetsEfficiently(FeatureToggle? featureToggle)
+    {
+        // Create a dictionary with 10,000 variables with diverse characters
+        var variables = new Dictionary<string, string>();
+        var random = new Random(42); // Seed for reproducibility
+    
+        // Generate 10,000 unique variables with diverse content
+        for (int i = 0; i < 10000; i++)
+        {
+            string key = $"Key{i}_{Guid.NewGuid().ToString("N")}";
+            string value = $"Value{i}_{Convert.ToBase64String(Guid.NewGuid().ToByteArray())}";
+    
+            // Mix in some random Unicode characters
+            if (random.Next(5) == 0)
+            {
+                key += (char)random.Next(0x1F600, 0x1F64F); // Emoji range
+                value += Environment.NewLine + (char)random.Next(0x2600, 0x26FF); // Unicode symbols
+            }
+    
+            variables[key] = value;
+        }
+        
+        var sw = Stopwatch.StartNew();
+        // Run the script with all these variables
+        var (output, _) = RunScript("enumerate-variables.sh",
+                                    variables.AddFeatureToggleToDictionary(new List<FeatureToggle?> { featureToggle }));
+        sw.Stop();
+        sw.Elapsed.TotalMilliseconds.Should().BeLessThan(2000);
+        
+        output.AssertSuccess();
+        if (featureToggle == FeatureToggle.BashParametersArrayFeatureToggle)
+        {
+            var fullOutput = string.Join(Environment.NewLine, output.CapturedOutput.Infos);
+            if (fullOutput.Contains("Bash version 4.2 or later is required to use octopus_parameters"))
+            {
+                output.AssertOutput("Still ran this script");
+                return;
+            }
+    
+            // Get all output lines that start with "Key: "
+            var outputLines = output.CapturedOutput.Infos
+                                    .Where(line => line.StartsWith("Key: "))
+                                    .ToList();
+    
+            // Verify count matches
+            Assert.That(outputLines.Count, Is.EqualTo(variables.Count), 
+                        "Not all variables were processed");
+    
+            // For each variable, construct the expected output format and verify it exists
+            foreach (var kvp in variables)
+            {
+                string expectedOutput = $"Key: {kvp.Key}, Value: {kvp.Value}";
+                Assert.That(outputLines.Contains(expectedOutput), 
+                            $"Expected output line not found: '{expectedOutput}'");
+            }
+        }
+    } 
     }
 
     public static class AdditionalVariablesExtensions
