From 8b132b535bb435f3bd8d099949e37b49e83ffbeb Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 31 Dec 2024 15:21:19 -0700 Subject: [PATCH 1/2] Enable MS Testing Platform This gets all tests running again. It also changes how test logs are collected to match changes from enabling the testing platform. However, discovery fails for some tests due to the need to execute code to load MSBuild from the SDK first, and the new xunit v3 discovery system apparently reflects before instantiating test classes, leading to these failures. --- CONTRIBUTING.md | 7 ++++++- .../Cake.GitVersioning.Tests.csproj | 1 + .../Nerdbank.GitVersioning.Tests.csproj | 1 + tools/dotnet-test-cloud.ps1 | 10 ++++++---- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 357e9c96..b99a89be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,12 @@ NPM packages too. ## Testing -The Visual Studio 2019 Test Explorer will list and execute all tests. +`dotnet test` will run all tests. + +The Visual Studio 2022 Test Explorer will list and execute all tests. +Visual Studio needs a preview feature called "Use testing platform server mode" enabled. + +A few tests will fail without a certain VC++ toolset installed. ## Releases diff --git a/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj b/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj index f03810d6..83d93555 100644 --- a/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj +++ b/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj @@ -3,6 +3,7 @@ net8.0 Exe + true false true false diff --git a/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj b/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj index 0d1d321d..abda9351 100644 --- a/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj +++ b/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj @@ -3,6 +3,7 @@ net9.0 $(TargetFrameworks);net472 Exe + true true true full diff --git a/tools/dotnet-test-cloud.ps1 b/tools/dotnet-test-cloud.ps1 index 10cd4311..89ddc519 100644 --- a/tools/dotnet-test-cloud.ps1 +++ b/tools/dotnet-test-cloud.ps1 @@ -44,8 +44,8 @@ if ($x86) { } } -$testBinLog = Join-Path $ArtifactStagingFolder (Join-Path build_logs test.binlog) -$testDiagLog = Join-Path $ArtifactStagingFolder (Join-Path test_logs diag.log) +$testBinLog = Join-Path $ArtifactStagingFolder build_logs test.binlog +$testResultsDirectory = Join-Path $ArtifactStagingFolder test_logs & $dotnet test $RepoRoot ` --no-build ` @@ -56,8 +56,10 @@ $testDiagLog = Join-Path $ArtifactStagingFolder (Join-Path test_logs diag.log) --blame-hang-timeout 60s ` --blame-crash ` -bl:"$testBinLog" ` - --diag "$testDiagLog;TraceLevel=info" ` - --logger trx ` + -- ` + --results-directory $testResultsDirectory ` + --report-xunit-trx ` + --long-running 60 ` $unknownCounter = 0 Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% { From 081725ac7482a88751aa2d930ebddbe1c643e8cd Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 1 Jan 2025 10:22:51 -0700 Subject: [PATCH 2/2] Rollback to xunit v2 The v3 upgrade caused all but 33 tests to be quietly omitted from test runs. Fixing that required [opting into the new Microsoft Testing Platform](https://github.com/xunit/xunit/issues/3107#issuecomment-2566695118). But then a few tests failed to be discovered due to this repo's particular requirements for loading MSBuild with the Locator, causing more failures. Analyzing those failures gets harder because [logs are moved or hidden](https://github.com/xunit/xunit/discussions/3109). Ultimately after working through all those issues to get at the errors behind tests and test discovery, I'm just out of ideas for how to get msbuild to load before reflection happens on the test assembly, so I'm just rolling back to xunit v2. See also https://github.com/xunit/xunit/issues/3092. --- CONTRIBUTING.md | 1 - Directory.Packages.props | 5 +- .../Cake.GitVersioning.Tests.csproj | 4 +- .../AssemblyInfoTest.cs | 72 ++++--------------- .../BuildIntegrationTests.cs | 4 +- .../GitContextTests.cs | 4 +- .../LibGit2GitExtensionsTests.cs | 2 +- .../Nerdbank.GitVersioning.Tests.csproj | 5 +- test/Nerdbank.GitVersioning.Tests/Usings.cs | 4 ++ tools/dotnet-test-cloud.ps1 | 10 ++- 10 files changed, 31 insertions(+), 80 deletions(-) create mode 100644 test/Nerdbank.GitVersioning.Tests/Usings.cs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b99a89be..a94bd306 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,6 @@ NPM packages too. `dotnet test` will run all tests. The Visual Studio 2022 Test Explorer will list and execute all tests. -Visual Studio needs a preview feature called "Use testing platform server mode" enabled. A few tests will fail without a certain VC++ toolset installed. diff --git a/Directory.Packages.props b/Directory.Packages.props index 094c1a75..40a63c67 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -38,12 +38,13 @@ - + + - + diff --git a/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj b/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj index 83d93555..c27fb890 100644 --- a/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj +++ b/test/Cake.GitVersioning.Tests/Cake.GitVersioning.Tests.csproj @@ -2,8 +2,6 @@ net8.0 - Exe - true false true false @@ -20,7 +18,7 @@ - + diff --git a/test/Nerdbank.GitVersioning.Tests/AssemblyInfoTest.cs b/test/Nerdbank.GitVersioning.Tests/AssemblyInfoTest.cs index 809ab20d..390aa42d 100644 --- a/test/Nerdbank.GitVersioning.Tests/AssemblyInfoTest.cs +++ b/test/Nerdbank.GitVersioning.Tests/AssemblyInfoTest.cs @@ -13,7 +13,7 @@ public class AssemblyInfoTest : IClassFixture // The MSBuildFixture throws PlatformNotSupportedException when run on mono. { - [Theory] + [SkippableTheory(typeof(PlatformNotSupportedException))] [InlineData(false)] [InlineData(true)] [InlineData(null)] @@ -49,15 +49,7 @@ public void FSharpGenerator(bool? thisAssemblyClass) info.EmitThisAssemblyClass = thisAssemblyClass.GetValueOrDefault(); } - string built; - try - { - built = info.BuildCode(); - } - catch (PlatformNotSupportedException ex) - { - throw SkipException.ForSkip(ex.Message); - } + string built = info.BuildCode(); string expected = $@"//------------------------------------------------------------------------------ // @@ -99,7 +91,7 @@ type internal ThisAssembly() = Assert.Equal(expected, built); } - [Theory] + [SkippableTheory(typeof(PlatformNotSupportedException))] [InlineData(null, "MyRootNamespace")] [InlineData("", "MyRootNamespace")] [InlineData("MyCustomNamespace", null)] @@ -117,15 +109,7 @@ public void FSharpGeneratorWithNamespace(string thisAssemblyNamespace, string ro ThisAssemblyNamespace = thisAssemblyNamespace, }; - string built; - try - { - built = info.BuildCode(); - } - catch (PlatformNotSupportedException ex) - { - throw SkipException.ForSkip(ex.Message); - } + string built = info.BuildCode(); string expected = $@"//------------------------------------------------------------------------------ // @@ -168,7 +152,7 @@ type internal ThisAssembly() = Assert.Equal(expected, built); } - [Theory] + [SkippableTheory(typeof(PlatformNotSupportedException))] [InlineData(false)] [InlineData(true)] [InlineData(null)] @@ -204,15 +188,7 @@ public void CSharpGenerator(bool? thisAssemblyClass) info.EmitThisAssemblyClass = thisAssemblyClass.GetValueOrDefault(); } - string built; - try - { - built = info.BuildCode(); - } - catch (PlatformNotSupportedException ex) - { - throw SkipException.ForSkip(ex.Message); - } + string built = info.BuildCode(); string expected = $@"//------------------------------------------------------------------------------ // @@ -252,7 +228,7 @@ internal static partial class ThisAssembly {{ Assert.Equal(expected, built); } - [Theory] + [SkippableTheory(typeof(PlatformNotSupportedException))] [InlineData(null, "MyRootNamespace")] [InlineData("", "MyRootNamespace")] [InlineData("MyCustomNamespace", null)] @@ -270,15 +246,7 @@ public void CSharpGeneratorWithNamespace(string thisAssemblyNamespace, string ro ThisAssemblyNamespace = thisAssemblyNamespace, }; - string built; - try - { - built = info.BuildCode(); - } - catch (PlatformNotSupportedException ex) - { - throw SkipException.ForSkip(ex.Message); - } + string built = info.BuildCode(); (string nsStart, string nsEnd) = !string.IsNullOrWhiteSpace(thisAssemblyNamespace) ? ($"{Environment.NewLine}namespace {thisAssemblyNamespace} {{", $"{Environment.NewLine}}}") @@ -318,7 +286,7 @@ internal static partial class ThisAssembly {{ Assert.Equal(expected, built); } - [Theory] + [SkippableTheory(typeof(PlatformNotSupportedException))] [InlineData(false)] [InlineData(true)] [InlineData(null)] @@ -335,15 +303,7 @@ public void VisualBasicGenerator(bool? thisAssemblyClass) info.EmitThisAssemblyClass = thisAssemblyClass.GetValueOrDefault(); } - string built; - try - { - built = info.BuildCode(); - } - catch (PlatformNotSupportedException ex) - { - throw SkipException.ForSkip(ex.Message); - } + string built = info.BuildCode(); string expected = $@"'------------------------------------------------------------------------------ ' @@ -382,7 +342,7 @@ End Class Assert.Equal(expected, built); } - [Theory] + [SkippableTheory(typeof(PlatformNotSupportedException))] [InlineData(null, "MyRootNamespace")] [InlineData("", "MyRootNamespace")] [InlineData("MyCustomNamespace", null)] @@ -400,15 +360,7 @@ public void VisualBasicGeneratorWithNamespace(string thisAssemblyNamespace, stri ThisAssemblyNamespace = thisAssemblyNamespace, }; - string built; - try - { - built = info.BuildCode(); - } - catch (PlatformNotSupportedException ex) - { - throw SkipException.ForSkip(ex.Message); - } + string built = info.BuildCode(); (string nsStart, string nsEnd) = !string.IsNullOrWhiteSpace(thisAssemblyNamespace) ? ($"{Environment.NewLine}Namespace {thisAssemblyNamespace}", $"{Environment.NewLine}End Namespace") diff --git a/test/Nerdbank.GitVersioning.Tests/BuildIntegrationTests.cs b/test/Nerdbank.GitVersioning.Tests/BuildIntegrationTests.cs index 3ca77583..8badf5d5 100644 --- a/test/Nerdbank.GitVersioning.Tests/BuildIntegrationTests.cs +++ b/test/Nerdbank.GitVersioning.Tests/BuildIntegrationTests.cs @@ -176,8 +176,8 @@ public async Task AssemblyInfo_HasKeyData(string keyFile, bool delaySigned) result.BuildResult.ProjectStateAfterBuild.GetPropertyValue("VersionSourceFile")))); this.Logger.WriteLine(versionCsContent); - SyntaxTree sourceFile = CSharpSyntaxTree.ParseText(versionCsContent, cancellationToken: TestContext.Current.CancellationToken); - SyntaxNode syntaxTree = await sourceFile.GetRootAsync(TestContext.Current.CancellationToken); + SyntaxTree sourceFile = CSharpSyntaxTree.ParseText(versionCsContent); + SyntaxNode syntaxTree = await sourceFile.GetRootAsync(); IEnumerable fields = syntaxTree.DescendantNodes().OfType(); var publicKeyField = (LiteralExpressionSyntax)fields.SingleOrDefault(f => f.Identifier.ValueText == "PublicKey")?.Initializer.Value; diff --git a/test/Nerdbank.GitVersioning.Tests/GitContextTests.cs b/test/Nerdbank.GitVersioning.Tests/GitContextTests.cs index a6a2acd8..681b9e62 100644 --- a/test/Nerdbank.GitVersioning.Tests/GitContextTests.cs +++ b/test/Nerdbank.GitVersioning.Tests/GitContextTests.cs @@ -93,14 +93,14 @@ public void SelectCommitByPartialId(bool fromPack, bool oddLength) Assert.Equal(this.LibGit2Repository.Head.Tip.Sha, this.Context.GitCommitId); } - [Theory] + [SkippableTheory] [InlineData(4)] [InlineData(7)] [InlineData(8)] [InlineData(11)] public void GetShortUniqueCommitId(int length) { - Assert.SkipWhen(length < 7 && this.Context is Nerdbank.GitVersioning.LibGit2.LibGit2Context, "LibGit2Sharp never returns commit IDs with fewer than 7 characters."); + Skip.If(length < 7 && this.Context is Nerdbank.GitVersioning.LibGit2.LibGit2Context, "LibGit2Sharp never returns commit IDs with fewer than 7 characters."); Assert.Equal(this.Context.GitCommitId.Substring(0, length), this.Context.GetShortUniqueCommitId(length)); } diff --git a/test/Nerdbank.GitVersioning.Tests/LibGit2GitExtensionsTests.cs b/test/Nerdbank.GitVersioning.Tests/LibGit2GitExtensionsTests.cs index 17430d2e..f393a7dd 100644 --- a/test/Nerdbank.GitVersioning.Tests/LibGit2GitExtensionsTests.cs +++ b/test/Nerdbank.GitVersioning.Tests/LibGit2GitExtensionsTests.cs @@ -467,7 +467,7 @@ public void GetIdAsVersion_MigrationFromVersionTxtToJson() public void TestBiggerRepo() { string testBiggerRepoPath = @"D:\git\Nerdbank.GitVersioning"; - Assert.SkipWhen(!Directory.Exists(testBiggerRepoPath), $"{testBiggerRepoPath} does not exist."); + Skip.If(!Directory.Exists(testBiggerRepoPath), $"{testBiggerRepoPath} does not exist."); using var largeRepo = new Repository(testBiggerRepoPath); foreach (Commit commit in largeRepo.Head.Commits) diff --git a/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj b/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj index abda9351..8a1fb5a1 100644 --- a/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj +++ b/test/Nerdbank.GitVersioning.Tests/Nerdbank.GitVersioning.Tests.csproj @@ -2,8 +2,6 @@ net9.0 $(TargetFrameworks);net472 - Exe - true true true full @@ -46,7 +44,8 @@ + - + diff --git a/test/Nerdbank.GitVersioning.Tests/Usings.cs b/test/Nerdbank.GitVersioning.Tests/Usings.cs new file mode 100644 index 00000000..55210089 --- /dev/null +++ b/test/Nerdbank.GitVersioning.Tests/Usings.cs @@ -0,0 +1,4 @@ +// Copyright (c) .NET Foundation and Contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +global using Xunit.Abstractions; diff --git a/tools/dotnet-test-cloud.ps1 b/tools/dotnet-test-cloud.ps1 index 89ddc519..10cd4311 100644 --- a/tools/dotnet-test-cloud.ps1 +++ b/tools/dotnet-test-cloud.ps1 @@ -44,8 +44,8 @@ if ($x86) { } } -$testBinLog = Join-Path $ArtifactStagingFolder build_logs test.binlog -$testResultsDirectory = Join-Path $ArtifactStagingFolder test_logs +$testBinLog = Join-Path $ArtifactStagingFolder (Join-Path build_logs test.binlog) +$testDiagLog = Join-Path $ArtifactStagingFolder (Join-Path test_logs diag.log) & $dotnet test $RepoRoot ` --no-build ` @@ -56,10 +56,8 @@ $testResultsDirectory = Join-Path $ArtifactStagingFolder test_logs --blame-hang-timeout 60s ` --blame-crash ` -bl:"$testBinLog" ` - -- ` - --results-directory $testResultsDirectory ` - --report-xunit-trx ` - --long-running 60 ` + --diag "$testDiagLog;TraceLevel=info" ` + --logger trx ` $unknownCounter = 0 Get-ChildItem -Recurse -Path $RepoRoot\test\*.trx |% {