Add GitHub CTRF test reporting (#445)

Author: bradwilson

.github/workflows/pull-request.yaml

@@ -40,3 +40,15 @@ jobs:
           path: artifacts/test
           compression-level: 9
         if: always()
+
+      - name: Publish Test Report
+        uses: ctrf-io/github-test-reporter@v1
+        with:
+          report-path: './artifacts/test/*.ctrf'
+          summary-report: true
+          github-report: true
+          pull-request: true
+          update-comment: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        if: always()

.github/workflows/push-main.yaml

@@ -60,3 +60,11 @@ jobs:
           path: artifacts/packages
           compression-level: 0
         if: always()
+
+      - name: Publish Test Report
+        uses: ctrf-io/github-test-reporter@v1
+        with:
+          report-path: './artifacts/test/*.ctrf'
+          summary-report: true
+          github-report: true
+        if: always()

test/test.v3/Tests.cs

@@ -5,11 +5,9 @@ public class Tests
 	[Fact]
 	public void Passing() { }
 
-	[Fact(Explicit = true)]
-	public void Failing() => Assert.Fail("This is a failing test");
+	[Fact(Skip = "Unconditionally skipped")]
+	public void Skipped() => Assert.Fail("This does not run");
 
-	[Theory]
-	[InlineData(true, Explicit = true)]
-	[InlineData(false)]
-	public void ConditionalSkip(bool value) => Assert.SkipWhen(value, "Conditionally skipped");
+	[Fact(Explicit = true)]
+	public void Explicit() => Assert.Fail("This only runs explicitly.");
 }

tools/builder/targets/TestCore.cs

@@ -12,12 +12,20 @@ public class TestCore
 {
 	public static async Task OnExecute(BuildContext context)
 	{
-		context.BuildStep("Running .NET Core tests");
+		context.BuildStep("Running .NET tests");
 
 		Directory.CreateDirectory(context.TestOutputFolder);
-		File.Delete(Path.Combine(context.TestOutputFolder, "test.xunit.runner.visualstudio-netcore.trx"));
 
-		await context.Exec("dotnet", $"test test/test.xunit.runner.visualstudio -tl:off --configuration {context.Configuration} --no-build --framework net8.0 --logger trx;LogFileName=test.xunit.runner.visualstudio-netcore.trx --results-directory \"{context.TestOutputFolder}\" --verbosity {context.Verbosity}");
+		var testFolder = Path.Combine(context.BaseFolder, "test", "test.xunit.runner.visualstudio", "bin", context.ConfigurationText, "net8.0");
+		var testPath = Path.Combine(testFolder, "test.xunit.runner.visualstudio.dll");
+		var reportPath = Path.Combine(context.TestOutputFolder, "test.xunit.runner.visualstudio-netcore.ctrf");
+		File.Delete(reportPath);
+
+		await context.Exec("dotnet", $"exec {testPath} -ctrf {reportPath}");
+
+		context.BuildStep("Running .NET VSTest integration tests");
+
+		await context.Exec("dotnet", $"test test/test.v3 -tl:off --configuration {context.Configuration} --no-build --framework net8.0 --verbosity {context.Verbosity}");
 		await context.Exec("dotnet", $"test test/test.v2 -tl:off --configuration {context.Configuration} --no-build --framework net8.0 --verbosity {context.Verbosity}");
 	}
 }

tools/builder/targets/TestFx.cs

@@ -15,9 +15,20 @@ public static async Task OnExecute(BuildContext context)
 		context.BuildStep("Running .NET Framework tests");
 
 		Directory.CreateDirectory(context.TestOutputFolder);
-		File.Delete(Path.Combine(context.TestOutputFolder, "test.xunit.runner.visualstudio-netfx.trx"));
 
-		await context.Exec("dotnet", $"test test/test.xunit.runner.visualstudio -tl:off --configuration {context.Configuration} --no-build --framework net472 --logger trx;LogFileName=test.xunit.runner.visualstudio-netfx.trx --results-directory \"{context.TestOutputFolder}\" --verbosity {context.Verbosity}");
+		var testFolder = Path.Combine(context.BaseFolder, "test", "test.xunit.runner.visualstudio", "bin", context.ConfigurationText, "net472");
+		var testPath = Path.Combine(testFolder, "test.xunit.runner.visualstudio.exe");
+		var reportPath = Path.Combine(context.TestOutputFolder, "test.xunit.runner.visualstudio-netfx.ctrf");
+		File.Delete(reportPath);
+
+		await context.Exec(testPath, $"-ctrf {reportPath}", testFolder);
+
+		if (context.NeedMono)
+			return;
+
+		context.BuildStep("Running .NET Framework VSTest integration tests");
+
+		await context.Exec("dotnet", $"test test/test.v3 -tl:off --configuration {context.Configuration} --no-build --framework net472 --verbosity {context.Verbosity}");
 		await context.Exec("dotnet", $"test test/test.v2 -tl:off --configuration {context.Configuration} --no-build --framework net472 --verbosity {context.Verbosity}");
 		await context.Exec("dotnet", $"test test/test.v1 -tl:off --configuration {context.Configuration} --no-build --framework net472 --verbosity {context.Verbosity}");
 	}