Add --show-all-tests option to GenerateTestSummary tool

.github/workflows/specialized-test-runner.yml

@@ -28,6 +28,11 @@ on:
         required: false
         type: boolean
         default: false
+      # Extra arguments to pass to GenerateTestSummary tool
+      extraSummaryArgs:
+        required: false
+        type: string
+        description: 'Extra arguments to pass to GenerateTestSummary (e.g., --show-all-tests)'
 
 jobs:
 
@@ -179,7 +184,7 @@ jobs:
           --project ${{ github.workspace }}/tools/GenerateTestSummary/GenerateTestSummary.csproj
           --
           ${{ github.workspace }}/testresults
-          --combined
+          ${{ inputs.extraSummaryArgs || '--combined' }}
 
       - name: Fail if any dependency failed
         # 'skipped' can be when a transitive dependency fails and the dependent job gets 'skipped'.

.github/workflows/tests-outerloop.yml

@@ -39,3 +39,4 @@ jobs:
       extraRunSheetBuilderArgs: "-p:RunOuterloopTests=true"
       extraTestArgs: "--filter-trait outerloop=true"
       enablePlaywrightInstall: true
+      extraSummaryArgs: "--show-all-tests"

.github/workflows/tests-quarantine.yml

@@ -41,3 +41,4 @@ jobs:
       extraTestArgs: "--filter-trait quarantined=true"
       enablePlaywrightInstall: true
       ignoreTestFailures: true
+      extraSummaryArgs: "--show-all-tests"

tools/GenerateTestSummary/Program.cs

@@ -5,21 +5,23 @@
 using System.CommandLine;
 using Aspire.TestTools;
 
-// Usage: dotnet tools run GenerateTestSummary --dirPathOrTrxFilePath <path> [--output <output>] [--combined]
+// Usage: dotnet tools run GenerateTestSummary --dirPathOrTrxFilePath <path> [--output <output>] [--combined] [--show-all-tests]
 // Generate a summary report from trx files.
 // And write to $GITHUB_STEP_SUMMARY if running in GitHub Actions.
 
 var dirPathOrTrxFilePathArgument = new Argument<string>("dirPathOrTrxFilePath");
 var outputOption = new Option<string>("--output", "-o") { Description = "Output file path" };
 var combinedSummaryOption = new Option<bool>("--combined", "-c") { Description = "Generate combined summary report" };
 var urlOption = new Option<string>("--url", "-u") { Description = "URL for test links" };
+var showAllTestsOption = new Option<bool>("--show-all-tests", "-a") { Description = "Show all tests (passing, failing, skipped) in a single list ordered by name" };
 
 var rootCommand = new RootCommand
 {
     dirPathOrTrxFilePathArgument,
     outputOption,
     combinedSummaryOption,
-    urlOption
+    urlOption,
+    showAllTestsOption
 };
 
 rootCommand.SetAction(result =>
@@ -33,15 +35,32 @@
 
     var combinedSummary = result.GetValue<bool>(combinedSummaryOption);
     var url = result.GetValue<string>(urlOption);
+    var showAllTests = result.GetValue<bool>(showAllTestsOption);
 
     if (combinedSummary && !string.IsNullOrEmpty(url))
     {
         Console.WriteLine("Error: --url option is not supported with --combined option.");
         return;
     }
 
+    if (showAllTests && !string.IsNullOrEmpty(url))
+    {
+        Console.WriteLine("Error: --url option is not supported with --show-all-tests option.");
+        return;
+    }
+
+    if (showAllTests && combinedSummary)
+    {
+        Console.WriteLine("Error: --show-all-tests option is not supported with --combined option.");
+        return;
+    }
+
     string report;
-    if (combinedSummary)
+    if (showAllTests)
+    {
+        report = TestSummaryGenerator.CreateAllTestsSummaryReport(dirPathOrTrxFilePath);
+    }
+    else if (combinedSummary)
     {
         report = TestSummaryGenerator.CreateCombinedTestSummaryReport(dirPathOrTrxFilePath);
     }

tools/GenerateTestSummary/TestSummaryGenerator.cs

@@ -199,6 +199,140 @@ public static string CreateCombinedTestSummaryReport(string basePath)
         return overallTableBuilder.ToString();
     }
 
+    public static string CreateAllTestsSummaryReport(string basePath)
+    {
+        var resolved = Path.GetFullPath(basePath);
+        if (!Directory.Exists(resolved))
+        {
+            throw new DirectoryNotFoundException($"The directory '{resolved}' does not exist.");
+        }
+
+        int overallTotalTestCount = 0;
+        int overallPassedTestCount = 0;
+        int overallFailedTestCount = 0;
+        int overallSkippedTestCount = 0;
+
+        // Collect all individual test results with their details
+        var allTests = new List<TestResultEntry>();
+
+        var trxFiles = Directory.EnumerateFiles(basePath, "*.trx", SearchOption.AllDirectories);
+        foreach (var filePath in trxFiles)
+        {
+            TestRun? testRun;
+            try
+            {
+                testRun = TrxReader.DeserializeTrxFile(filePath);
+                if (testRun?.ResultSummary?.Counters is null)
+                {
+                    Console.WriteLine($"Failed to deserialize or find results in file: {filePath}, tr: {testRun}");
+                    continue;
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Failed to deserialize file: {filePath}, exception: {ex}");
+                continue;
+            }
+
+            var counters = testRun.ResultSummary.Counters;
+            overallTotalTestCount += counters.Total;
+            overallPassedTestCount += counters.Passed;
+            overallFailedTestCount += counters.Failed;
+            overallSkippedTestCount += counters.NotExecuted;
+
+            if (testRun.Results?.UnitTestResults is null)
+            {
+                continue;
+            }
+
+            foreach (var test in testRun.Results.UnitTestResults)
+            {
+                if (string.IsNullOrEmpty(test.TestName))
+                {
+                    continue;
+                }
+
+                allTests.Add(new TestResultEntry(
+                    TestName: test.TestName,
+                    Outcome: test.Outcome ?? "Unknown",
+                    ErrorInfo: test.Output?.ErrorInfo?.InnerText,
+                    StdOut: test.Output?.StdOut
+                ));
+            }
+        }
+
+        // Sort by test name
+        allTests.Sort((a, b) => string.Compare(a.TestName, b.TestName, StringComparison.OrdinalIgnoreCase));
+
+        var reportBuilder = new StringBuilder();
+        reportBuilder.AppendLine("# Test Summary");
+        reportBuilder.AppendLine();
+
+        reportBuilder.AppendLine("## Overall Summary");
+        reportBuilder.AppendLine("| Passed | Failed | Skipped | Total |");
+        reportBuilder.AppendLine("|--------|--------|---------|-------|");
+        reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"| {overallPassedTestCount} | {overallFailedTestCount} | {overallSkippedTestCount} | {overallTotalTestCount} |");
+        reportBuilder.AppendLine();
+
+        reportBuilder.AppendLine("## All Tests");
+        reportBuilder.AppendLine();
+
+        foreach (var test in allTests)
+        {
+            if (test.Outcome == "Failed")
+            {
+                // Failed tests get expanded details with ❌
+                reportBuilder.AppendLine("<div>");
+                reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"""
+                    <details><summary>❌ <b>{test.TestName}</b></summary>
+
+                """);
+
+                reportBuilder.AppendLine();
+                reportBuilder.AppendLine("```yml");
+
+                reportBuilder.AppendLine(test.ErrorInfo);
+                if (test.StdOut is not null)
+                {
+                    const int halfLength = 25_000;
+                    var stdOutSpan = test.StdOut.AsSpan();
+
+                    reportBuilder.AppendLine();
+                    reportBuilder.AppendLine("### StdOut");
+
+                    var startSpan = stdOutSpan[..Math.Min(halfLength, stdOutSpan.Length)];
+                    reportBuilder.AppendLine(startSpan.ToString());
+
+                    if (stdOutSpan.Length > halfLength)
+                    {
+                        reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"{Environment.NewLine}... (snip) ...{Environment.NewLine}");
+                        var endSpan = stdOutSpan[^halfLength..];
+                        reportBuilder.Append("... ");
+                        reportBuilder.Append(endSpan);
+                    }
+                }
+
+                reportBuilder.AppendLine("```");
+                reportBuilder.AppendLine();
+                reportBuilder.AppendLine("</details>");
+                reportBuilder.AppendLine("</div>");
+                reportBuilder.AppendLine();
+            }
+            else if (test.Outcome == "Passed")
+            {
+                // Passed tests get simple line with ✅
+                reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"- ✅ {test.TestName}");
+            }
+            else
+            {
+                // Skipped/NotExecuted tests get simple line with ⏭️
+                reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"- ⏭️ {test.TestName}");
+            }
+        }
+
+        return reportBuilder.ToString();
+    }
+
     public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuilder reportBuilder, string? url)
     {
         if (!File.Exists(trxFilePath))
@@ -534,3 +668,5 @@ public static string GetTestTitle(string trxFileName)
 }
 
 internal sealed record TestRunSummary(string Icon, string Os, string Title, int Passed, int Failed, int Skipped, int Total, double DurationMinutes);
+
+internal sealed record TestResultEntry(string TestName, string Outcome, string? ErrorInfo, string? StdOut);