Rollback to xunit v2

The v3 upgrade caused all but 33 tests to be quietly omitted from test runs. Fixing that required [opting into the new Microsoft Testing Platform](xunit/xunit#3107 (comment)). But then a few tests failed to be discovered due to this repo's particular requirements for loading MSBuild with the Locator, causing more failures. Analyzing those failures gets harder because [logs are moved or hidden](xunit/xunit#3109). Ultimately after working through all those issues to get at the errors behind tests and test discovery, I'm just out of ideas for how to get msbuild to load before reflection happens on the test assembly, so I'm just rolling back to xunit v2. See also xunit/xunit#3092.
dotnet · Jan 1, 2025 · 081725a · 081725a
1 parent 8b132b5
commit 081725a
Show file tree

Hide file tree

Showing 10 changed files with 31 additions and 80 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -46,7 +46,6 @@ NPM packages too.
 `dotnet test` will run all tests.
 
 The Visual Studio 2022 Test Explorer will list and execute all tests.
-Visual Studio needs a preview feature called "Use testing platform server mode" enabled.
 
 A few tests will fail without a certain VC++ toolset installed.
 

diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -38,12 +38,13 @@
     <PackageVersion Include="System.Text.Json" Version="8.0.5" />
     <PackageVersion Include="System.Threading.Tasks.Dataflow" Version="9.0.0" />
     <PackageVersion Include="Validation" Version="2.5.51" />
-    <PackageVersion Include="Xunit.Combinatorial" Version="2.0.5-alpha" />
+    <PackageVersion Include="Xunit.Combinatorial" Version="1.6.24" />
+    <PackageVersion Include="Xunit.SkippableFact" Version="1.5.23" />
   </ItemGroup>
   <ItemGroup Label="Library.Template">
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="3.0.0" />
-    <PackageVersion Include="xunit.v3" Version="1.0.0" />
+    <PackageVersion Include="xunit" Version="2.9.2" />
   </ItemGroup>
   <ItemGroup>
     <!-- Put repo-specific GlobalPackageReference items in this group. -->

diff --git a/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj b/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj
@@ -2,8 +2,6 @@
 
   <PropertyGroup>
     <TargetFrameworks>net8.0</TargetFrameworks>
-    <OutputType>Exe</OutputType>
-    <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
     <SignAssembly>false</SignAssembly>
     <IsTestProject>true</IsTestProject>
     <IsPackable>false</IsPackable>
@@ -20,7 +18,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="Nerdbank.GitVersioning.LKG" />
     <PackageReference Include="xunit.runner.visualstudio" />
-    <PackageReference Include="xunit.v3" />
+    <PackageReference Include="xunit" />
   </ItemGroup>
 
 </Project>
diff --git a/test/Nerdbank.GitVersioning.Tests/AssemblyInfoTest.cs b/test/Nerdbank.GitVersioning.Tests/AssemblyInfoTest.cs
@@ -13,7 +13,7 @@
 
 public class AssemblyInfoTest : IClassFixture<MSBuildFixture> // The MSBuildFixture throws PlatformNotSupportedException when run on mono.
 {
-    [Theory]
+    [SkippableTheory(typeof(PlatformNotSupportedException))]
     [InlineData(false)]
     [InlineData(true)]
     [InlineData(null)]
@@ -49,15 +49,7 @@ public void FSharpGenerator(bool? thisAssemblyClass)
             info.EmitThisAssemblyClass = thisAssemblyClass.GetValueOrDefault();
         }
 
-        string built;
-        try
-        {
-            built = info.BuildCode();
-        }
-        catch (PlatformNotSupportedException ex)
-        {
-            throw SkipException.ForSkip(ex.Message);
-        }
+        string built = info.BuildCode();
 
         string expected = $@"//------------------------------------------------------------------------------
 // <auto-generated>
@@ -99,7 +91,7 @@ type internal ThisAssembly() =
         Assert.Equal(expected, built);
     }
 
-    [Theory]
+    [SkippableTheory(typeof(PlatformNotSupportedException))]
     [InlineData(null, "MyRootNamespace")]
     [InlineData("", "MyRootNamespace")]
     [InlineData("MyCustomNamespace", null)]
@@ -117,15 +109,7 @@ public void FSharpGeneratorWithNamespace(string thisAssemblyNamespace, string ro
             ThisAssemblyNamespace = thisAssemblyNamespace,
         };
 
-        string built;
-        try
-        {
-            built = info.BuildCode();
-        }
-        catch (PlatformNotSupportedException ex)
-        {
-            throw SkipException.ForSkip(ex.Message);
-        }
+        string built = info.BuildCode();
 
         string expected = $@"//------------------------------------------------------------------------------
 // <auto-generated>
@@ -168,7 +152,7 @@ type internal ThisAssembly() =
         Assert.Equal(expected, built);
     }
 
-    [Theory]
+    [SkippableTheory(typeof(PlatformNotSupportedException))]
     [InlineData(false)]
     [InlineData(true)]
     [InlineData(null)]
@@ -204,15 +188,7 @@ public void CSharpGenerator(bool? thisAssemblyClass)
             info.EmitThisAssemblyClass = thisAssemblyClass.GetValueOrDefault();
         }
 
-        string built;
-        try
-        {
-            built = info.BuildCode();
-        }
-        catch (PlatformNotSupportedException ex)
-        {
-            throw SkipException.ForSkip(ex.Message);
-        }
+        string built = info.BuildCode();
 
         string expected = $@"//------------------------------------------------------------------------------
 // <auto-generated>
@@ -252,7 +228,7 @@ internal static partial class ThisAssembly {{
         Assert.Equal(expected, built);
     }
 
-    [Theory]
+    [SkippableTheory(typeof(PlatformNotSupportedException))]
     [InlineData(null, "MyRootNamespace")]
     [InlineData("", "MyRootNamespace")]
     [InlineData("MyCustomNamespace", null)]
@@ -270,15 +246,7 @@ public void CSharpGeneratorWithNamespace(string thisAssemblyNamespace, string ro
             ThisAssemblyNamespace = thisAssemblyNamespace,
         };
 
-        string built;
-        try
-        {
-            built = info.BuildCode();
-        }
-        catch (PlatformNotSupportedException ex)
-        {
-            throw SkipException.ForSkip(ex.Message);
-        }
+        string built = info.BuildCode();
 
         (string nsStart, string nsEnd) = !string.IsNullOrWhiteSpace(thisAssemblyNamespace)
             ? ($"{Environment.NewLine}namespace {thisAssemblyNamespace} {{", $"{Environment.NewLine}}}")
@@ -318,7 +286,7 @@ internal static partial class ThisAssembly {{
         Assert.Equal(expected, built);
     }
 
-    [Theory]
+    [SkippableTheory(typeof(PlatformNotSupportedException))]
     [InlineData(false)]
     [InlineData(true)]
     [InlineData(null)]
@@ -335,15 +303,7 @@ public void VisualBasicGenerator(bool? thisAssemblyClass)
             info.EmitThisAssemblyClass = thisAssemblyClass.GetValueOrDefault();
         }
 
-        string built;
-        try
-        {
-            built = info.BuildCode();
-        }
-        catch (PlatformNotSupportedException ex)
-        {
-            throw SkipException.ForSkip(ex.Message);
-        }
+        string built = info.BuildCode();
 
         string expected = $@"'------------------------------------------------------------------------------
 ' <auto-generated>
@@ -382,7 +342,7 @@ End Class
         Assert.Equal(expected, built);
     }
 
-    [Theory]
+    [SkippableTheory(typeof(PlatformNotSupportedException))]
     [InlineData(null, "MyRootNamespace")]
     [InlineData("", "MyRootNamespace")]
     [InlineData("MyCustomNamespace", null)]
@@ -400,15 +360,7 @@ public void VisualBasicGeneratorWithNamespace(string thisAssemblyNamespace, stri
             ThisAssemblyNamespace = thisAssemblyNamespace,
         };
 
-        string built;
-        try
-        {
-            built = info.BuildCode();
-        }
-        catch (PlatformNotSupportedException ex)
-        {
-            throw SkipException.ForSkip(ex.Message);
-        }
+        string built = info.BuildCode();
 
         (string nsStart, string nsEnd) = !string.IsNullOrWhiteSpace(thisAssemblyNamespace)
             ? ($"{Environment.NewLine}Namespace {thisAssemblyNamespace}", $"{Environment.NewLine}End Namespace")

diff --git a/test/Nerdbank.GitVersioning.Tests/BuildIntegrationTests.cs b/test/Nerdbank.GitVersioning.Tests/BuildIntegrationTests.cs
@@ -176,8 +176,8 @@ public async Task AssemblyInfo_HasKeyData(string keyFile, bool delaySigned)
                     result.BuildResult.ProjectStateAfterBuild.GetPropertyValue("VersionSourceFile"))));
         this.Logger.WriteLine(versionCsContent);
 
-        SyntaxTree sourceFile = CSharpSyntaxTree.ParseText(versionCsContent, cancellationToken: TestContext.Current.CancellationToken);
-        SyntaxNode syntaxTree = await sourceFile.GetRootAsync(TestContext.Current.CancellationToken);
+        SyntaxTree sourceFile = CSharpSyntaxTree.ParseText(versionCsContent);
+        SyntaxNode syntaxTree = await sourceFile.GetRootAsync();
         IEnumerable<VariableDeclaratorSyntax> fields = syntaxTree.DescendantNodes().OfType<VariableDeclaratorSyntax>();
 
         var publicKeyField = (LiteralExpressionSyntax)fields.SingleOrDefault(f => f.Identifier.ValueText == "PublicKey")?.Initializer.Value;

diff --git a/test/Nerdbank.GitVersioning.Tests/GitContextTests.cs b/test/Nerdbank.GitVersioning.Tests/GitContextTests.cs
@@ -93,14 +93,14 @@ public void SelectCommitByPartialId(bool fromPack, bool oddLength)
         Assert.Equal(this.LibGit2Repository.Head.Tip.Sha, this.Context.GitCommitId);
     }
 
-    [Theory]
+    [SkippableTheory]
     [InlineData(4)]
     [InlineData(7)]
     [InlineData(8)]
     [InlineData(11)]
     public void GetShortUniqueCommitId(int length)
     {
-        Assert.SkipWhen(length < 7 && this.Context is Nerdbank.GitVersioning.LibGit2.LibGit2Context, "LibGit2Sharp never returns commit IDs with fewer than 7 characters.");
+        Skip.If(length < 7 && this.Context is Nerdbank.GitVersioning.LibGit2.LibGit2Context, "LibGit2Sharp never returns commit IDs with fewer than 7 characters.");
         Assert.Equal(this.Context.GitCommitId.Substring(0, length), this.Context.GetShortUniqueCommitId(length));
     }
 

diff --git a/test/Nerdbank.GitVersioning.Tests/LibGit2GitExtensionsTests.cs b/test/Nerdbank.GitVersioning.Tests/LibGit2GitExtensionsTests.cs
@@ -467,7 +467,7 @@ public void GetIdAsVersion_MigrationFromVersionTxtToJson()
     public void TestBiggerRepo()
     {
         string testBiggerRepoPath = @"D:\git\Nerdbank.GitVersioning";
-        Assert.SkipWhen(!Directory.Exists(testBiggerRepoPath), $"{testBiggerRepoPath} does not exist.");
+        Skip.If(!Directory.Exists(testBiggerRepoPath), $"{testBiggerRepoPath} does not exist.");
 
         using var largeRepo = new Repository(testBiggerRepoPath);
         foreach (Commit commit in largeRepo.Head.Commits)

diff --git a/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj b/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj
@@ -2,8 +2,6 @@
   <PropertyGroup>
     <TargetFrameworks>net9.0</TargetFrameworks>
     <TargetFrameworks Condition="$([MSBuild]::IsOSPlatform('Windows'))">$(TargetFrameworks);net472</TargetFrameworks>
-    <OutputType>Exe</OutputType>
-    <TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
     <GenerateBindingRedirectsOutputType>true</GenerateBindingRedirectsOutputType>
     <DebugType>full</DebugType>
@@ -46,7 +44,8 @@
     <PackageReference Include="System.Memory" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" />
     <PackageReference Include="Xunit.Combinatorial" />
+    <PackageReference Include="Xunit.SkippableFact" />
     <PackageReference Include="xunit.runner.visualstudio" />
-    <PackageReference Include="xunit.v3" />
+    <PackageReference Include="xunit" />
   </ItemGroup>
 </Project>
diff --git a/test/Nerdbank.GitVersioning.Tests/Usings.cs b/test/Nerdbank.GitVersioning.Tests/Usings.cs
@@ -0,0 +1,4 @@
+// Copyright (c) .NET Foundation and Contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+global using Xunit.Abstractions;
diff --git a/tools/dotnet-test-cloud.ps1 b/tools/dotnet-test-cloud.ps1
@@ -44,8 +44,8 @@ if ($x86) {
   }
 }
 
-$testBinLog = Join-Path $ArtifactStagingFolder build_logs test.binlog
-$testResultsDirectory = Join-Path $ArtifactStagingFolder test_logs
+$testBinLog = Join-Path $ArtifactStagingFolder (Join-Path build_logs test.binlog)
+$testDiagLog = Join-Path $ArtifactStagingFolder (Join-Path test_logs diag.log)
 
 & $dotnet test $RepoRoot `
     --no-build `
@@ -56,10 +56,8 @@ $testResultsDirectory = Join-Path $ArtifactStagingFolder test_logs
     --blame-hang-timeout 60s `
     --blame-crash `
     -bl:"$testBinLog" `
-    -- `
-    --results-directory $testResultsDirectory `
-    --report-xunit-trx `
-    --long-running 60 `
+    --diag "$testDiagLog;TraceLevel=info" `
+    --logger trx `
 
 $unknownCounter = 0
 Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% {