From 561d57ff50d52ced6bba2fe8342b60fcf733abe8 Mon Sep 17 00:00:00 2001 From: David Barbet Date: Thu, 23 Jun 2022 14:46:02 -0700 Subject: [PATCH 01/26] Add two failing tests --- .../CommentSelection/ToggleBlockCommentCommandHandlerTests.cs | 2 ++ .../LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/EditorFeatures/Test/CommentSelection/ToggleBlockCommentCommandHandlerTests.cs b/src/EditorFeatures/Test/CommentSelection/ToggleBlockCommentCommandHandlerTests.cs index d016446a873fc..17d977d5c0f8b 100644 --- a/src/EditorFeatures/Test/CommentSelection/ToggleBlockCommentCommandHandlerTests.cs +++ b/src/EditorFeatures/Test/CommentSelection/ToggleBlockCommentCommandHandlerTests.cs @@ -27,6 +27,8 @@ public void AddComment_EmptyCaret() var expected = @"[|/**/|]"; ToggleComment(markup, expected); + + Assert.False(true); } [WpfFact, Trait(Traits.Feature, Traits.Features.ToggleBlockComment)] diff --git a/src/Features/LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs b/src/Features/LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs index 9127740b2806d..f7da40fe1611f 100644 --- a/src/Features/LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs +++ b/src/Features/LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs @@ -384,6 +384,8 @@ void A.AMethod(int i) testLspServer, expectedLocation).ConfigureAwait(false); Assert.Equal(expectedMarkdown, results.Contents.Third.Value); + + Assert.False(true); } private static async Task RunGetHoverAsync( From 45d49ddcc27dcb9467faddc9aa6dc665d33d68d0 Mon Sep 17 00:00:00 2001 From: David Barbet Date: Thu, 23 Jun 2022 18:59:50 -0700 Subject: [PATCH 02/26] Working on partition --- eng/prepare-tests.ps1 | 3 +- eng/prepare-tests.sh | 2 +- .../ToggleBlockCommentCommandHandlerTests.cs | 2 - .../ProtocolUnitTests/Hover/HoverTests.cs | 2 - src/Tools/PrepareTests/AssemblyInfo.cs | 28 ++ src/Tools/PrepareTests/ListTests.cs | 231 +++++++++++ src/Tools/PrepareTests/ProcessRunner.cs | 202 +++++++++ src/Tools/PrepareTests/Program.cs | 20 +- .../Source/RunTests/AssemblyScheduler.cs | 390 +++++------------- src/Tools/Source/RunTests/ITestExecutor.cs | 12 +- src/Tools/Source/RunTests/ProcessRunner.cs | 202 --------- .../Source/RunTests/ProcessTestExecutor.cs | 62 +-- src/Tools/Source/RunTests/Program.cs | 96 ++--- src/Tools/Source/RunTests/RunTests.csproj | 3 + src/Tools/Source/RunTests/TestRunner.cs | 74 +++- 15 files changed, 721 insertions(+), 608 deletions(-) create mode 100644 src/Tools/PrepareTests/AssemblyInfo.cs create mode 100644 src/Tools/PrepareTests/ListTests.cs create mode 100644 src/Tools/PrepareTests/ProcessRunner.cs delete mode 100644 src/Tools/Source/RunTests/ProcessRunner.cs diff --git a/eng/prepare-tests.ps1 b/eng/prepare-tests.ps1 index cfdcee17742ef..fcd9c03894fcf 100644 --- a/eng/prepare-tests.ps1 +++ b/eng/prepare-tests.ps1 @@ -11,7 +11,8 @@ try { $dotnet = Ensure-DotnetSdk # permissions issues make this a pain to do in PrepareTests itself. Remove-Item -Recurse -Force "$RepoRoot\artifacts\testPayload" -ErrorAction SilentlyContinue - Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload" + $cmdOutput = Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload --dotnetPath `"$dotnet`"" + echo $cmdOutput exit 0 } catch { diff --git a/eng/prepare-tests.sh b/eng/prepare-tests.sh index 5b0310573bd1f..e4ee6bb52089e 100755 --- a/eng/prepare-tests.sh +++ b/eng/prepare-tests.sh @@ -28,4 +28,4 @@ InitializeDotNetCli true # permissions issues make this a pain to do in PrepareTests itself. rm -rf "$repo_root/artifacts/testPayload" -dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix +dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix --dotnetPath ${_InitializeDotNetCli}/dotnet diff --git a/src/EditorFeatures/Test/CommentSelection/ToggleBlockCommentCommandHandlerTests.cs b/src/EditorFeatures/Test/CommentSelection/ToggleBlockCommentCommandHandlerTests.cs index 17d977d5c0f8b..d016446a873fc 100644 --- a/src/EditorFeatures/Test/CommentSelection/ToggleBlockCommentCommandHandlerTests.cs +++ b/src/EditorFeatures/Test/CommentSelection/ToggleBlockCommentCommandHandlerTests.cs @@ -27,8 +27,6 @@ public void AddComment_EmptyCaret() var expected = @"[|/**/|]"; ToggleComment(markup, expected); - - Assert.False(true); } [WpfFact, Trait(Traits.Feature, Traits.Features.ToggleBlockComment)] diff --git a/src/Features/LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs b/src/Features/LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs index f7da40fe1611f..9127740b2806d 100644 --- a/src/Features/LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs +++ b/src/Features/LanguageServer/ProtocolUnitTests/Hover/HoverTests.cs @@ -384,8 +384,6 @@ void A.AMethod(int i) testLspServer, expectedLocation).ConfigureAwait(false); Assert.Equal(expectedMarkdown, results.Contents.Third.Value); - - Assert.False(true); } private static async Task RunGetHoverAsync( diff --git a/src/Tools/PrepareTests/AssemblyInfo.cs b/src/Tools/PrepareTests/AssemblyInfo.cs new file mode 100644 index 0000000000000..9faffef67ab1f --- /dev/null +++ b/src/Tools/PrepareTests/AssemblyInfo.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; + +namespace PrepareTests; + +public readonly record struct AssemblyInfo(string AssemblyPath) : IComparable +{ + public string AssemblyName => Path.GetFileName(AssemblyPath); + + public int CompareTo(object? obj) + { + if (obj == null) + { + return 1; + } + + var otherAssembly = (AssemblyInfo)obj; + + // Ensure we have a consistent ordering by ordering by assembly path. + return this.AssemblyPath.CompareTo(otherAssembly.AssemblyPath); + } +} + +public readonly record struct TypeInfo(string Name, string FullyQualifiedName, int TestCount); diff --git a/src/Tools/PrepareTests/ListTests.cs b/src/Tools/PrepareTests/ListTests.cs new file mode 100644 index 0000000000000..20952a5fc5bc0 --- /dev/null +++ b/src/Tools/PrepareTests/ListTests.cs @@ -0,0 +1,231 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Diagnostics.Contracts; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Mono.Options; + +namespace PrepareTests; +public class ListTests +{ + /// + /// Regex to find test lines from the output of dotnet test. + /// + /// + /// The goal is to match lines that contain a fully qualified test name e.g. + /// Microsoft.CodeAnalysis.Editor.CSharp.UnitTests.RemoveUnusedParametersAndValues.RemoveUnusedValueAssignmentTests.UnusedVarPattern_PartOfIs + /// or + /// action = (int [|"..., pascalCaseSymbol: "void Outer() { System.Action action = (int M)"..., symbolKind: Parameter, accessibility: NotApplicable)]]> + /// But not anything else dotnet test --list-tests outputs, like + /// + /// Microsoft (R) Test Execution Command Line Tool Version 17.3.0-preview-20220414-05 (x64) + /// Copyright(c) Microsoft Corporation. All rights reserved. + /// + /// The following Tests are available: + /// + /// The regex looks for the namespace names (groups of non-whitespace characters followed by a dot) at the beginning of the line. + /// + private static readonly Regex TestOutputFormat = new("^(\\S)*\\..*", RegexOptions.Compiled); + + internal static async Task RunAsync(string sourceDirectory, string dotnetPath) + { + // Find all test assemblies. + var binDirectory = Path.Combine(sourceDirectory, "artifacts", "bin"); + var assemblies = GetTestAssemblyFilePaths(binDirectory); + Console.WriteLine($"Found test assemblies:{Environment.NewLine}{string.Join(Environment.NewLine, assemblies.Select(a => a.AssemblyPath))}"); + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + // Discover tests via `dotnet test --list-tests` and write out the test counts to a file so we can use it for partitioning later. + await GetTypeInfoAsync(assemblies, dotnetPath); + stopwatch.Stop(); + Console.WriteLine($"Discovered tests in {stopwatch.Elapsed}"); + } + + /// + /// Returns the file path of the test data results for a particular assembly. + /// This is written using the output of dotnet test list in + /// and read during each test leg for partitioning. + /// + public static string GetTestDataFilePath(AssemblyInfo assembly) + { + var assemblyDirectory = Path.GetDirectoryName(assembly.AssemblyPath)!; + var fileName = $"{assembly.AssemblyName}_tests.txt"; + var outputPath = Path.Combine(assemblyDirectory, fileName); + return outputPath; + } + + /// + /// Find all unit test assemblies that we need to discover tests on. + /// + public static ImmutableArray GetTestAssemblyFilePaths( + string binDirectory, + Func? shouldSkipTestDirectory = null, + string configurationSearchPattern = "*", + List? targetFrameworks = null) + { + var list = new List(); + + // Find all the project folders that fit our naming scheme for unit tests. + foreach (var project in Directory.EnumerateDirectories(binDirectory, "*.UnitTests", SearchOption.TopDirectoryOnly)) + { + if (shouldSkipTestDirectory != null && shouldSkipTestDirectory(project)) + { + continue; + } + + var name = Path.GetFileName(project); + var fileName = $"{name}.dll"; + + // Find the dlls matching the request configuration and target frameworks. + var configurationDirectories = Directory.EnumerateDirectories(project, configurationSearchPattern, SearchOption.TopDirectoryOnly); + foreach (var configuration in configurationDirectories) + { + var targetFrameworkDirectories = Directory.EnumerateDirectories(configuration, "*", SearchOption.TopDirectoryOnly); + if (targetFrameworks != null) + { + targetFrameworkDirectories = targetFrameworks.Select(tfm => Path.Combine(configuration, tfm)); + } + + foreach (var targetFrameworkDirectory in targetFrameworkDirectories) + { + // In multi-targeting scenarios we will build both .net core and .net framework versions of the assembly on unix. + // If we're on unix and we see the .net framework assembly, skip it as we can't list or run tests on it anyway. + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Path.GetFileName(targetFrameworkDirectory) == "net472") + { + Console.WriteLine($"Skipping net472 assembly on unix: {targetFrameworkDirectory}"); + continue; + } + + var filePath = Path.Combine(targetFrameworkDirectory, fileName); + if (File.Exists(filePath)) + { + list.Add(new AssemblyInfo(filePath)); + } + else if (Directory.Exists(targetFrameworkDirectory) && Directory.GetFiles(targetFrameworkDirectory, searchPattern: "*.UnitTests.dll") is { Length: > 0 } matches) + { + // If the unit test assembly name doesn't match the project folder name, but still matches our "unit test" name pattern, we want to run it. + // If more than one such assembly is present in a project output folder, we assume something is wrong with the build configuration. + // For example, one unit test project might be referencing another unit test project. + if (matches.Length > 1) + { + var message = $"Multiple unit test assemblies found in '{targetFrameworkDirectory}'. Please adjust the build to prevent this. Matches:{Environment.NewLine}{string.Join(Environment.NewLine, matches)}"; + throw new Exception(message); + } + list.Add(new AssemblyInfo(matches[0])); + } + } + } + } + + Contract.Assert(list.Count > 0, $"Did not find any test assemblies"); + + list.Sort(); + return list.ToImmutableArray(); + } + + /// + /// Runs `dotnet test _assembly_ --list-tests` on all test assemblies to get the real count of tests per assembly. + /// + private static async Task GetTypeInfoAsync(ImmutableArray assemblies, string dotnetFilePath) + { + // We run this one assembly at a time because it will not output which test is in which assembly otherwise. + // It's also faster than making a single call to dotnet test with all the assemblies. + await Parallel.ForEachAsync(assemblies, async (assembly, cancellationToken) => + { + var assemblyPath = assembly.AssemblyPath; + var commandArgs = $"test {assemblyPath} --list-tests"; + var processResult = await ProcessRunner.CreateProcess(dotnetFilePath, commandArgs, workingDirectory: Directory.GetCurrentDirectory(), captureOutput: true, displayWindow: false, cancellationToken: cancellationToken).Result; + if (processResult.ExitCode != 0) + { + var errorOutput = string.Join(Environment.NewLine, processResult.ErrorLines); + var output = string.Join(Environment.NewLine, processResult.OutputLines); + throw new InvalidOperationException($"dotnet test failed with {processResult.ExitCode} for {assemblyPath}.{Environment.NewLine}Error output: {errorOutput}{Environment.NewLine}Output: {output}"); + } + + var typeInfo = ParseDotnetTestOutput(processResult.OutputLines); + var testCount = typeInfo.Sum(type => type.TestCount); + Contract.Assert(testCount > 0, $"Did not find any tests in {assembly}, output was {Environment.NewLine}{string.Join(Environment.NewLine, processResult.OutputLines)}"); + + Console.WriteLine($"Found {testCount} tests for {assemblyPath}"); + + await WriteTestDataAsync(assembly, typeInfo, cancellationToken); + }); + + return; + + static async Task WriteTestDataAsync(AssemblyInfo assembly, ImmutableArray typeInfo, CancellationToken cancellationToken) + { + var outputPath = GetTestDataFilePath(assembly); + + using var createStream = File.Create(outputPath); + await JsonSerializer.SerializeAsync(createStream, typeInfo, cancellationToken: cancellationToken); + await createStream.DisposeAsync(); + } + } + + /// + /// Parse the output of `dotnet test` to count the number of tests in the assembly by type name. + /// + private static ImmutableArray ParseDotnetTestOutput(IEnumerable output) + { + // Find all test lines from the output of dotnet test using a regex match. + var testList = output.Select(line => line.TrimStart()).Where(line => TestOutputFormat.IsMatch(line)); + + // Figure out the fully qualified type name for each test. + var typeList = testList + .Where(line => !string.IsNullOrWhiteSpace(line)) + .Select(GetFullyQualifiedTypeName); + + // Count all occurences of the type name in the test list to figure out how many tests per type we have. + var groups = typeList.GroupBy(type => type); + var result = groups.Select(group => new TypeInfo(GetTypeNameFromFullyQualifiedName(group.Key), group.Key, group.Count())).ToImmutableArray(); + return result; + + static string GetFullyQualifiedTypeName(string testLine) + { + // Remove whitespace from the start as the list is always indented. + var test = testLine.TrimStart(); + // The common case is just a fully qualified method name e.g. + // Microsoft.CodeAnalysis.Editor.CSharp.UnitTests.RemoveUnusedParametersAndValues.RemoveUnusedValueAssignmentTests.UnusedVarPattern_PartOfIs + // + // However, we can also have more complex expressions with actual code in them (and periods) like + // Microsoft.CodeAnalysis.Editor.CSharp.UnitTests.Diagnostics.NamingStyles.NamingStylesTests.TestPascalCaseSymbol_ExpectedSymbolAndAccessibility(camelCaseSymbol: "void Outer() { System.Action action = (int [|"..., pascalCaseSymbol: "void Outer() { System.Action action = (int M)"..., symbolKind: Parameter, accessibility: NotApplicable) + // + // So we first split on ( to get the fully qualified method name. This is valid because the namespace, type name, and method name cannot have parens. + // The first part of the split gives us the fully qualified method name. From that we can take everything up until the last period get the fully qualified type name. + var splitString = test.Split("("); + var fullyQualifiedMethod = splitString[0]; + + var periodBeforeMethodName = fullyQualifiedMethod.LastIndexOf("."); + var fullyQualifiedType = fullyQualifiedMethod[..periodBeforeMethodName]; + + return fullyQualifiedType; + } + + static string GetTypeNameFromFullyQualifiedName(string fullyQualifiedTypeName) + { + var periodBeforeTypeName = fullyQualifiedTypeName.LastIndexOf("."); + var typeName = fullyQualifiedTypeName[(periodBeforeTypeName + 1)..]; + return typeName; + } + } +} diff --git a/src/Tools/PrepareTests/ProcessRunner.cs b/src/Tools/PrepareTests/ProcessRunner.cs new file mode 100644 index 0000000000000..f5c247f87d1f9 --- /dev/null +++ b/src/Tools/PrepareTests/ProcessRunner.cs @@ -0,0 +1,202 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace PrepareTests; + +public readonly struct ProcessResult +{ + public Process Process { get; } + public int ExitCode { get; } + public ReadOnlyCollection OutputLines { get; } + public ReadOnlyCollection ErrorLines { get; } + + public ProcessResult(Process process, int exitCode, ReadOnlyCollection outputLines, ReadOnlyCollection errorLines) + { + Process = process; + ExitCode = exitCode; + OutputLines = outputLines; + ErrorLines = errorLines; + } +} + +public readonly struct ProcessInfo +{ + public Process Process { get; } + public ProcessStartInfo StartInfo { get; } + public Task Result { get; } + + public int Id => Process.Id; + + public ProcessInfo(Process process, ProcessStartInfo startInfo, Task result) + { + Process = process; + StartInfo = startInfo; + Result = result; + } +} + +public static class ProcessRunner +{ + public static void OpenFile(string file) + { + if (File.Exists(file)) + { + Process.Start(file); + } + } + + public static ProcessInfo CreateProcess( + string executable, + string arguments, + bool lowPriority = false, + string? workingDirectory = null, + bool captureOutput = false, + bool displayWindow = true, + Dictionary? environmentVariables = null, + Action? onProcessStartHandler = null, + Action? onOutputDataReceived = null, + CancellationToken cancellationToken = default) => + CreateProcess( + CreateProcessStartInfo(executable, arguments, workingDirectory, captureOutput, displayWindow, environmentVariables), + lowPriority: lowPriority, + onProcessStartHandler: onProcessStartHandler, + onOutputDataReceived: onOutputDataReceived, + cancellationToken: cancellationToken); + + public static ProcessInfo CreateProcess( + ProcessStartInfo processStartInfo, + bool lowPriority = false, + Action? onProcessStartHandler = null, + Action? onOutputDataReceived = null, + CancellationToken cancellationToken = default) + { + var errorLines = new List(); + var outputLines = new List(); + var process = new Process(); + var tcs = new TaskCompletionSource(); + + process.EnableRaisingEvents = true; + process.StartInfo = processStartInfo; + + process.OutputDataReceived += (s, e) => + { + if (e.Data != null) + { + onOutputDataReceived?.Invoke(e); + outputLines.Add(e.Data); + } + }; + + process.ErrorDataReceived += (s, e) => + { + if (e.Data != null) + { + errorLines.Add(e.Data); + } + }; + + process.Exited += (s, e) => + { + // We must call WaitForExit to make sure we've received all OutputDataReceived/ErrorDataReceived calls + // or else we'll be returning a list we're still modifying. For paranoia, we'll start a task here rather + // than enter right back into the Process type and start a wait which isn't guaranteed to be safe. + Task.Run(() => + { + process.WaitForExit(); + var result = new ProcessResult( + process, + process.ExitCode, + new ReadOnlyCollection(outputLines), + new ReadOnlyCollection(errorLines)); + tcs.TrySetResult(result); + }, cancellationToken); + }; + + var registration = cancellationToken.Register(() => + { + if (tcs.TrySetCanceled()) + { + // If the underlying process is still running, we should kill it + if (!process.HasExited) + { + try + { + process.Kill(); + } + catch (InvalidOperationException) + { + // Ignore, since the process is already dead + } + } + } + }); + + process.Start(); + onProcessStartHandler?.Invoke(process); + + if (lowPriority) + { + process.PriorityClass = ProcessPriorityClass.BelowNormal; + } + + if (processStartInfo.RedirectStandardOutput) + { + process.BeginOutputReadLine(); + } + + if (processStartInfo.RedirectStandardError) + { + process.BeginErrorReadLine(); + } + + return new ProcessInfo(process, processStartInfo, tcs.Task); + } + + public static ProcessStartInfo CreateProcessStartInfo( + string executable, + string arguments, + string? workingDirectory = null, + bool captureOutput = false, + bool displayWindow = true, + Dictionary? environmentVariables = null) + { + var processStartInfo = new ProcessStartInfo(executable, arguments); + + if (!string.IsNullOrEmpty(workingDirectory)) + { + processStartInfo.WorkingDirectory = workingDirectory; + } + + if (environmentVariables != null) + { + foreach (var pair in environmentVariables) + { + processStartInfo.EnvironmentVariables[pair.Key] = pair.Value; + } + } + + if (captureOutput) + { + processStartInfo.UseShellExecute = false; + processStartInfo.RedirectStandardOutput = true; + processStartInfo.RedirectStandardError = true; + } + else + { + processStartInfo.CreateNoWindow = !displayWindow; + processStartInfo.UseShellExecute = displayWindow; + } + + return processStartInfo; + } +} diff --git a/src/Tools/PrepareTests/Program.cs b/src/Tools/PrepareTests/Program.cs index f012d3b71c12e..ff48425f22f9c 100644 --- a/src/Tools/PrepareTests/Program.cs +++ b/src/Tools/PrepareTests/Program.cs @@ -3,24 +3,29 @@ // See the LICENSE file in the project root for more information. using System; +using System.Threading.Tasks; using Mono.Options; +namespace PrepareTests; + internal static class Program { internal const int ExitFailure = 1; internal const int ExitSuccess = 0; - public static int Main(string[] args) + public static async Task Main(string[] args) { string? source = null; string? destination = null; bool isUnix = false; + string? dotnetPath = null; var options = new OptionSet() { { "source=", "Path to binaries", (string s) => source = s }, { "destination=", "Output path", (string s) => destination = s }, - { "unix", "If true, prepares tests for unix environment instead of Windows", o => isUnix = o is object } + { "unix", "If true, prepares tests for unix environment instead of Windows", o => isUnix = o is object }, + { "dotnetPath=", "Output path", (string s) => dotnetPath = s }, }; options.Parse(args); @@ -36,6 +41,17 @@ public static int Main(string[] args) return ExitFailure; } + if (dotnetPath is null) + { + Console.Error.WriteLine("--dotnetPath argument must be provided"); + return ExitFailure; + } + + // Figure out what tests we need to run before we minimize everything. + Console.WriteLine("Discovering tests..."); + await ListTests.RunAsync(source, dotnetPath); + + Console.WriteLine("Minimizing test payload..."); MinimizeUtil.Run(source, destination, isUnix); return ExitSuccess; } diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index d724f7c1b17cb..6e32838f20638 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -4,357 +4,175 @@ using System.Collections.Generic; using System.Collections.Immutable; -using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; -using System.Reflection.Metadata; -using System.Reflection.PortableExecutable; -using System.Text; +using System.Text.Json; +using PrepareTests; +using TypeInfo = PrepareTests.TypeInfo; namespace RunTests { - internal readonly struct AssemblyInfo - { - internal PartitionInfo PartitionInfo { get; } - internal string TargetFramework { get; } - internal string Architecture { get; } - - internal string AssemblyPath => PartitionInfo.AssemblyPath; - internal string AssemblyName => Path.GetFileName(PartitionInfo.AssemblyPath); - internal string DisplayName => PartitionInfo.DisplayName; - internal AssemblyInfo(PartitionInfo partitionInfo, string targetFramework, string architecture) - { - PartitionInfo = partitionInfo; - TargetFramework = targetFramework; - Architecture = architecture; - } - } - - internal readonly struct PartitionInfo + /// + /// Defines a single work item to run. The work item may contain tests from multiple assemblies + /// that should be run together in the same work item. + /// + internal readonly record struct WorkItemInfo(ImmutableSortedDictionary> TypesToTest, int PartitionIndex) { - internal int? AssemblyPartitionId { get; } - internal string AssemblyPath { get; } - internal string DisplayName { get; } - - /// - /// Specific set of types to test in the assembly. Will be empty when testing the entire assembly - /// - internal readonly ImmutableArray TypeInfoList; - - internal PartitionInfo( - int assemblyPartitionId, - string assemblyPath, - string displayName, - ImmutableArray typeInfoList) + internal string DisplayName { - AssemblyPartitionId = assemblyPartitionId; - AssemblyPath = assemblyPath; - DisplayName = displayName; - TypeInfoList = typeInfoList; - } - - internal PartitionInfo(string assemblyPath) - { - AssemblyPartitionId = null; - AssemblyPath = assemblyPath; - DisplayName = Path.GetFileName(assemblyPath); - TypeInfoList = ImmutableArray.Empty; + get + { + var assembliesString = string.Join("_", TypesToTest.Keys.Select(assembly => Path.GetFileNameWithoutExtension(assembly.AssemblyName).Replace(".", string.Empty))); + return $"{assembliesString}_{PartitionIndex}"; + } } - public override string ToString() => DisplayName; - } - - public readonly struct TypeInfo - { - internal readonly string FullName; - internal readonly int MethodCount; - - internal TypeInfo(string fullName, int methodCount) - { - FullName = fullName; - MethodCount = methodCount; - } + internal static WorkItemInfo CreateFullAssembly(AssemblyInfo assembly, int partitionIndex) + => new(ImmutableSortedDictionary>.Empty.Add(assembly, ImmutableArray.Empty), partitionIndex); } internal sealed class AssemblyScheduler { - /// - /// This is a test class inserted into assemblies to guard against a .NET desktop bug. The tests - /// inside of it counteract the underlying issue. If this test is included in any assembly it - /// must be added to every partition to ensure the work around is present - /// - /// https://github.com/dotnet/corefx/issues/3793 - /// https://github.com/dotnet/roslyn/issues/8936 - /// - private const string EventListenerGuardFullName = "Microsoft.CodeAnalysis.UnitTests.EventListenerGuard"; - - private static class AssemblyInfoBuilder - { - internal static void Build(string assemblyPath, int methodLimit, List typeInfoList, out ImmutableArray partitionInfoList) - { - var list = new List(); - var hasEventListenerGuard = typeInfoList.Any(x => x.FullName == EventListenerGuardFullName); - var currentTypeInfoList = new List(); - var currentClassNameLengthSum = -1; - var currentId = 0; - - BeginPartition(); - - foreach (var typeInfo in typeInfoList) - { - currentTypeInfoList.Add(typeInfo); - currentClassNameLengthSum += typeInfo.FullName.Length; - CheckForPartitionLimit(done: false); - } - - CheckForPartitionLimit(done: true); - - partitionInfoList = ImmutableArray.CreateRange(list); - - void BeginPartition() - { - currentId++; - currentTypeInfoList.Clear(); - currentClassNameLengthSum = 0; - - // Ensure the EventListenerGuard is in every partition. - if (hasEventListenerGuard) - { - currentClassNameLengthSum += EventListenerGuardFullName.Length; - } - } - - void CheckForPartitionLimit(bool done) - { - if (done) - { - // The builder is done looking at types. If there are any TypeInfo that have not - // been added to a partition then do it now. - if (currentTypeInfoList.Count > 0) - { - FinishPartition(); - } - - return; - } - - // One item we have to consider here is the maximum command line length in - // Windows which is 32767 characters (XP is smaller but don't care). Once - // we get close then create a partition and move on. - if (currentTypeInfoList.Sum(x => x.MethodCount) >= methodLimit || - currentClassNameLengthSum > 25000) - { - FinishPartition(); - BeginPartition(); - } - - void FinishPartition() - { - var partitionInfo = new PartitionInfo( - currentId, - assemblyPath, - $"{Path.GetFileName(assemblyPath)}.{currentId}", - ImmutableArray.CreateRange(currentTypeInfoList)); - list.Add(partitionInfo); - } - } - } - } - /// /// Default number of methods to include per partition. /// internal const int DefaultMethodLimit = 2000; - /// - /// Number of methods to include per Helix work item. - /// - internal const int HelixMethodLimit = 500; - private readonly Options _options; - private readonly int _methodLimit; internal AssemblyScheduler(Options options) { _options = options; - _methodLimit = options.UseHelix ? AssemblyScheduler.HelixMethodLimit : AssemblyScheduler.DefaultMethodLimit; } - public ImmutableArray Schedule(string assemblyPath, bool force = false) + public ImmutableArray Schedule(ImmutableArray assemblies) { + Logger.Log($"Scheduling {assemblies.Length} assemblies"); if (_options.Sequential) { - return ImmutableArray.Create(new PartitionInfo(assemblyPath)); + Logger.Log("Building sequential work items"); + // return individual work items per assembly that contain all the tests in that assembly. + return assemblies + .Select(WorkItemInfo.CreateFullAssembly) + .ToImmutableArray(); } - var typeInfoList = GetTypeInfoList(assemblyPath); - AssemblyInfoBuilder.Build(assemblyPath, _methodLimit, typeInfoList, out var partitionList); + var orderedTypeInfos = assemblies.ToImmutableSortedDictionary(assembly => assembly, GetTestData); - // If the scheduling didn't actually produce multiple partition then send back an unpartitioned - // representation. - if (partitionList.Length == 1 && !force) - { - Logger.Log($"Assembly schedule produced a single partition {assemblyPath}"); - return ImmutableArray.Create(new PartitionInfo(assemblyPath)); - } + var testsPerPartition = DefaultMethodLimit; + var totalTestCount = orderedTypeInfos.Values.SelectMany(type => type).Sum(type => type.TestCount); - Logger.Log($"Assembly Schedule: {Path.GetFileName(assemblyPath)}"); - foreach (var partition in partitionList) + // We've calculated the optimal number of helix partitions based on the queuing time and test run times. + // Currently that number is 56 - TODO fill out details of computing this number. + var partitionCount = 56; + + // We then determine how many tests we need to run in each partition based on the total number of tests to run. + if (_options.UseHelix) { - var methodCount = partition.TypeInfoList.Sum(x => x.MethodCount); - var delta = methodCount - _methodLimit; - Logger.Log($" Partition: {partition.AssemblyPartitionId} method count {methodCount} delta {delta}"); - foreach (var typeInfo in partition.TypeInfoList) - { - Logger.Log($" {typeInfo.FullName} {typeInfo.MethodCount}"); - } + testsPerPartition = totalTestCount / partitionCount; } - return partitionList; + ConsoleUtil.WriteLine($"Found {totalTestCount} tests to run, building {partitionCount} partitions with {testsPerPartition} each"); + + // Build work items by adding tests from each type until we hit the limit of tests for each partition. + // This won't always result in exactly the desired number of partitions with exactly the same number of tests - this is because + // we only filter by type name in arguments to dotnet test to avoid command length errors. + // + // While we do our best to run tests from the same assembly together (by building work items in assembly order) it is expected + // that some work items will run tests from multiple assemblies due to large variances in the number of tests per assembly. + var workItems = BuildWorkItems(orderedTypeInfos, testsPerPartition); + LogWorkItems(workItems); + return workItems; } - private static List GetTypeInfoList(string assemblyPath) + private static ImmutableArray GetTestData(AssemblyInfo assembly) { - using (var stream = File.OpenRead(assemblyPath)) - using (var peReader = new PEReader(stream)) - { - var metadataReader = peReader.GetMetadataReader(); - return GetTypeInfoList(metadataReader); - } - } + var testDataFile = ListTests.GetTestDataFilePath(assembly); + Logger.Log($"Reading assembly test data for {assembly.AssemblyName} from {testDataFile}"); - private static List GetTypeInfoList(MetadataReader reader) - { - var list = new List(); - foreach (var handle in reader.TypeDefinitions) - { - var type = reader.GetTypeDefinition(handle); - if (!IsValidIdentifier(reader, type.Name)) - { - continue; - } - - var methodCount = GetMethodCount(reader, type); - if (!ShouldIncludeType(reader, type, methodCount)) - { - continue; - } - - var fullName = GetFullName(reader, type); - list.Add(new TypeInfo(fullName, methodCount)); - } - - // Ensure we get classes back in a deterministic order. - list.Sort((x, y) => x.FullName.CompareTo(y.FullName)); - return list; + using var readStream = File.OpenRead(testDataFile); + var testData = JsonSerializer.Deserialize>(readStream); + return testData; } - /// - /// Determine if this type should be one of the class values passed to xunit. This - /// code doesn't actually resolve base types or trace through inherrited Fact attributes - /// hence we have to error on the side of including types with no tests vs. excluding them. - /// - private static bool ShouldIncludeType(MetadataReader reader, TypeDefinition type, int testMethodCount) + private static ImmutableArray BuildWorkItems(ImmutableSortedDictionary> typeInfos, int methodLimit) { - // xunit only handles public, non-abstract classes - var isPublic = - TypeAttributes.Public == (type.Attributes & TypeAttributes.Public) || - TypeAttributes.NestedPublic == (type.Attributes & TypeAttributes.NestedPublic); - if (!isPublic || - TypeAttributes.Abstract == (type.Attributes & TypeAttributes.Abstract) || - TypeAttributes.Class != (type.Attributes & TypeAttributes.Class)) - { - return false; - } + var workItems = new List(); - // Compiler generated types / methods have the shape of the heuristic that we are looking - // at here. Filter them out as well. - if (!IsValidIdentifier(reader, type.Name)) - { - return false; - } + var currentClassNameLengthSum = 0; + var currentTestCount = 0; + var workItemIndex = 0; - if (testMethodCount > 0) - { - return true; - } - - // The case we still have to consider at this point is a class with 0 defined methods, - // inheritting from a class with > 0 defined test methods. That is a completely valid - // xunit scenario. For now we're just going to exclude types that inherit from object - // or other built-in base types because they clearly don't fit that category. - return !InheritsFromFrameworkBaseType(reader, type); - } + var currentAssemblies = new Dictionary>(); - private static int GetMethodCount(MetadataReader reader, TypeDefinition type) - { - var count = 0; - foreach (var handle in type.GetMethods()) + // Iterate through each assembly and type and build up the work items to run. + // We add types from assemblies 1 by 1 until we hit the work item test limits / command line length limits. + foreach (var (assembly, types) in typeInfos) { - var methodDefinition = reader.GetMethodDefinition(handle); - if (methodDefinition.GetCustomAttributes().Count == 0 || - !IsValidIdentifier(reader, methodDefinition.Name)) + Logger.Log($"Building commands for {assembly.AssemblyName} with {types.Length} types and {types.Sum(type => type.TestCount)} tests"); + foreach (var type in types) { - continue; - } + if (type.TestCount + currentTestCount >= methodLimit) + { + // Adding this type would put us over the method limit for this partition. + // Add our accumulated types and assemblies and end the work item + AddWorkItem(currentAssemblies); + currentAssemblies = new Dictionary>(); + } + else if (currentClassNameLengthSum > 25000) + { + // One item we have to consider here is the maximum command line length in + // Windows which is 32767 characters (XP is smaller but don't care). + // Once we get close we start a new work item. + AddWorkItem(currentAssemblies); + currentAssemblies = new Dictionary>(); + } - count++; + AddType(currentAssemblies, assembly, type); + } } - return count; - } + // Add any remaining tests to the work item. + AddWorkItem(currentAssemblies); + return workItems.ToImmutableArray(); - private static bool IsValidIdentifier(MetadataReader reader, StringHandle handle) - { - var name = reader.GetString(handle); - for (int i = 0; i < name.Length; i++) + void AddWorkItem(Dictionary> typesToTest) { - switch (name[i]) + if (typesToTest.Any()) { - case '<': - case '>': - case '$': - return false; + workItems.Add(new WorkItemInfo(typesToTest.ToImmutableSortedDictionary(kvp => kvp.Key, kvp => kvp.Value.ToImmutableArray()), workItemIndex)); } - } - return true; - } + currentTestCount = 0; + currentClassNameLengthSum = 0; + workItemIndex++; + } - private static bool InheritsFromFrameworkBaseType(MetadataReader reader, TypeDefinition type) - { - if (type.BaseType.Kind != HandleKind.TypeReference) + void AddType(Dictionary> dictionary, AssemblyInfo assemblyInfo, TypeInfo typeInfo) { - return false; - } + var list = dictionary.TryGetValue(assemblyInfo, out var result) ? result : new List(); + list.Add(typeInfo); + dictionary[assemblyInfo] = list; - var typeRef = reader.GetTypeReference((TypeReferenceHandle)type.BaseType); - return - reader.GetString(typeRef.Namespace) == "System" && - reader.GetString(typeRef.Name) is "Object" or "ValueType" or "Enum"; + currentTestCount += typeInfo.TestCount; + currentClassNameLengthSum += typeInfo.Name.Length; + } } - private static string GetFullName(MetadataReader reader, TypeDefinition type) - { - var typeName = reader.GetString(type.Name); - if (TypeAttributes.NestedPublic == (type.Attributes & TypeAttributes.NestedPublic)) - { - // Need to take into account the containing type. - var declaringType = reader.GetTypeDefinition(type.GetDeclaringType()); - var declaringTypeFullName = GetFullName(reader, declaringType); - return $"{declaringTypeFullName}+{typeName}"; - } - - var namespaceName = reader.GetString(type.Namespace); - if (string.IsNullOrEmpty(namespaceName)) + private static void LogWorkItems(ImmutableArray workItems) + { + Logger.Log("==== Work Item List ===="); + foreach (var workItem in workItems) { - return typeName; + Logger.Log($"- Work Item ({workItem.TypesToTest.Values.SelectMany(w => w).Sum(assembly => assembly.TestCount)} tests)"); + foreach (var assembly in workItem.TypesToTest) + { + Logger.Log($" - {assembly.Key.AssemblyName} with {assembly.Value.Sum(type => type.TestCount)} tests"); + } } - - return $"{namespaceName}.{typeName}"; } } } diff --git a/src/Tools/Source/RunTests/ITestExecutor.cs b/src/Tools/Source/RunTests/ITestExecutor.cs index 91e669b6c0bf6..3ef1b3ca877d9 100644 --- a/src/Tools/Source/RunTests/ITestExecutor.cs +++ b/src/Tools/Source/RunTests/ITestExecutor.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Immutable; -using System.IO; +using PrepareTests; namespace RunTests { @@ -69,7 +69,7 @@ internal TestResultInfo(int exitCode, string? resultsFilePath, string? htmlResul internal readonly struct TestResult { internal TestResultInfo TestResultInfo { get; } - internal AssemblyInfo AssemblyInfo { get; } + internal WorkItemInfo WorkItemInfo { get; } internal string CommandLine { get; } internal string? Diagnostics { get; } @@ -78,9 +78,7 @@ internal readonly struct TestResult /// internal ImmutableArray ProcessResults { get; } - internal string AssemblyPath => AssemblyInfo.AssemblyPath; - internal string AssemblyName => Path.GetFileName(AssemblyPath); - internal string DisplayName => AssemblyInfo.DisplayName; + internal string DisplayName => WorkItemInfo.DisplayName; internal bool Succeeded => ExitCode == 0; internal int ExitCode => TestResultInfo.ExitCode; internal TimeSpan Elapsed => TestResultInfo.Elapsed; @@ -88,9 +86,9 @@ internal readonly struct TestResult internal string ErrorOutput => TestResultInfo.ErrorOutput; internal string? ResultsDisplayFilePath => TestResultInfo.HtmlResultsFilePath ?? TestResultInfo.ResultsFilePath; - internal TestResult(AssemblyInfo assemblyInfo, TestResultInfo testResultInfo, string commandLine, ImmutableArray processResults = default, string? diagnostics = null) + internal TestResult(WorkItemInfo workItemInfo, TestResultInfo testResultInfo, string commandLine, ImmutableArray processResults = default, string? diagnostics = null) { - AssemblyInfo = assemblyInfo; + WorkItemInfo = workItemInfo; TestResultInfo = testResultInfo; CommandLine = commandLine; ProcessResults = processResults.IsDefault ? ImmutableArray.Empty : processResults; diff --git a/src/Tools/Source/RunTests/ProcessRunner.cs b/src/Tools/Source/RunTests/ProcessRunner.cs deleted file mode 100644 index 2ac1b555703e9..0000000000000 --- a/src/Tools/Source/RunTests/ProcessRunner.cs +++ /dev/null @@ -1,202 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -namespace RunTests -{ - public readonly struct ProcessResult - { - public Process Process { get; } - public int ExitCode { get; } - public ReadOnlyCollection OutputLines { get; } - public ReadOnlyCollection ErrorLines { get; } - - public ProcessResult(Process process, int exitCode, ReadOnlyCollection outputLines, ReadOnlyCollection errorLines) - { - Process = process; - ExitCode = exitCode; - OutputLines = outputLines; - ErrorLines = errorLines; - } - } - - public readonly struct ProcessInfo - { - public Process Process { get; } - public ProcessStartInfo StartInfo { get; } - public Task Result { get; } - - public int Id => Process.Id; - - public ProcessInfo(Process process, ProcessStartInfo startInfo, Task result) - { - Process = process; - StartInfo = startInfo; - Result = result; - } - } - - public static class ProcessRunner - { - public static void OpenFile(string file) - { - if (File.Exists(file)) - { - Process.Start(file); - } - } - - public static ProcessInfo CreateProcess( - string executable, - string arguments, - bool lowPriority = false, - string? workingDirectory = null, - bool captureOutput = false, - bool displayWindow = true, - Dictionary? environmentVariables = null, - Action? onProcessStartHandler = null, - Action? onOutputDataReceived = null, - CancellationToken cancellationToken = default) => - CreateProcess( - CreateProcessStartInfo(executable, arguments, workingDirectory, captureOutput, displayWindow, environmentVariables), - lowPriority: lowPriority, - onProcessStartHandler: onProcessStartHandler, - onOutputDataReceived: onOutputDataReceived, - cancellationToken: cancellationToken); - - public static ProcessInfo CreateProcess( - ProcessStartInfo processStartInfo, - bool lowPriority = false, - Action? onProcessStartHandler = null, - Action? onOutputDataReceived = null, - CancellationToken cancellationToken = default) - { - var errorLines = new List(); - var outputLines = new List(); - var process = new Process(); - var tcs = new TaskCompletionSource(); - - process.EnableRaisingEvents = true; - process.StartInfo = processStartInfo; - - process.OutputDataReceived += (s, e) => - { - if (e.Data != null) - { - onOutputDataReceived?.Invoke(e); - outputLines.Add(e.Data); - } - }; - - process.ErrorDataReceived += (s, e) => - { - if (e.Data != null) - { - errorLines.Add(e.Data); - } - }; - - process.Exited += (s, e) => - { - // We must call WaitForExit to make sure we've received all OutputDataReceived/ErrorDataReceived calls - // or else we'll be returning a list we're still modifying. For paranoia, we'll start a task here rather - // than enter right back into the Process type and start a wait which isn't guaranteed to be safe. - Task.Run(() => - { - process.WaitForExit(); - var result = new ProcessResult( - process, - process.ExitCode, - new ReadOnlyCollection(outputLines), - new ReadOnlyCollection(errorLines)); - tcs.TrySetResult(result); - }, cancellationToken); - }; - - var registration = cancellationToken.Register(() => - { - if (tcs.TrySetCanceled()) - { - // If the underlying process is still running, we should kill it - if (!process.HasExited) - { - try - { - process.Kill(); - } - catch (InvalidOperationException) - { - // Ignore, since the process is already dead - } - } - } - }); - - process.Start(); - onProcessStartHandler?.Invoke(process); - - if (lowPriority) - { - process.PriorityClass = ProcessPriorityClass.BelowNormal; - } - - if (processStartInfo.RedirectStandardOutput) - { - process.BeginOutputReadLine(); - } - - if (processStartInfo.RedirectStandardError) - { - process.BeginErrorReadLine(); - } - - return new ProcessInfo(process, processStartInfo, tcs.Task); - } - - public static ProcessStartInfo CreateProcessStartInfo( - string executable, - string arguments, - string? workingDirectory = null, - bool captureOutput = false, - bool displayWindow = true, - Dictionary? environmentVariables = null) - { - var processStartInfo = new ProcessStartInfo(executable, arguments); - - if (!string.IsNullOrEmpty(workingDirectory)) - { - processStartInfo.WorkingDirectory = workingDirectory; - } - - if (environmentVariables != null) - { - foreach (var pair in environmentVariables) - { - processStartInfo.EnvironmentVariables[pair.Key] = pair.Value; - } - } - - if (captureOutput) - { - processStartInfo.UseShellExecute = false; - processStartInfo.RedirectStandardOutput = true; - processStartInfo.RedirectStandardError = true; - } - else - { - processStartInfo.CreateNoWindow = !displayWindow; - processStartInfo.UseShellExecute = displayWindow; - } - - return processStartInfo; - } - } -} diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index 193d707d14228..f8ecd4c952436 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -8,11 +8,14 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.IO; +using System.Linq; +using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml.Linq; using System.Xml.XPath; +using PrepareTests; namespace RunTests { @@ -25,7 +28,7 @@ internal ProcessTestExecutor(TestExecutionOptions options) Options = options; } - public string GetCommandLineArguments(AssemblyInfo assemblyInfo, bool useSingleQuotes, bool isHelix) + public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuotes, Options options) { // http://www.gnu.org/software/bash/manual/html_node/Single-Quotes.html // Single quotes are needed in bash to avoid the need to escape characters such as backtick (`) which are found in metadata names. @@ -35,8 +38,11 @@ public string GetCommandLineArguments(AssemblyInfo assemblyInfo, bool useSingleQ var builder = new StringBuilder(); builder.Append($@"test"); - builder.Append($@" {sep}{assemblyInfo.AssemblyName}{sep}"); - var typeInfoList = assemblyInfo.PartitionInfo.TypeInfoList; + + var escapedAssemblyPaths = workItem.TypesToTest.Keys.Select(assembly => $"{sep}{assembly.AssemblyPath}{sep}"); + builder.Append($@" {string.Join(" ", escapedAssemblyPaths)}"); + + var typeInfoList = workItem.TypesToTest.Values.SelectMany(type => type).ToImmutableArray(); if (typeInfoList.Length > 0 || !string.IsNullOrWhiteSpace(Options.TestFilter)) { builder.Append($@" --filter {sep}"); @@ -48,7 +54,7 @@ public string GetCommandLineArguments(AssemblyInfo assemblyInfo, bool useSingleQ // We want to avoid matching other test classes whose names are prefixed with this test class's name. // For example, avoid running 'AttributeTests_WellKnownMember', when the request here is to run 'AttributeTests'. // We append a '.', assuming that all test methods in the class *will* match it, but not methods in other classes. - builder.Append(typeInfo.FullName); + builder.Append(typeInfo.Name); builder.Append('.'); } builder.Append(sep); @@ -70,13 +76,12 @@ void MaybeAddSeparator(char separator = '|') } } - builder.Append($@" --arch {assemblyInfo.Architecture}"); - builder.Append($@" --framework {assemblyInfo.TargetFramework}"); - builder.Append($@" --logger {sep}xunit;LogFilePath={GetResultsFilePath(assemblyInfo, "xml")}{sep}"); + builder.Append($@" --arch {options.Architecture}"); + builder.Append($@" --logger {sep}xunit;LogFilePath={GetResultsFilePath(workItem, options, "xml")}{sep}"); if (Options.IncludeHtml) { - builder.AppendFormat($@" --logger {sep}html;LogFileName={GetResultsFilePath(assemblyInfo, "html")}{sep}"); + builder.AppendFormat($@" --logger {sep}html;LogFileName={GetResultsFilePath(workItem, options, "html")}{sep}"); } if (!Options.CollectDumps) @@ -94,46 +99,46 @@ void MaybeAddSeparator(char separator = '|') // // Helix timeout is 15 minutes as helix jobs fully timeout in 30minutes. So in order to capture dumps we need the timeout // to be 2x shorter than the expected test run time (15min) in case only the last test hangs. - var timeout = isHelix ? "15minutes" : "25minutes"; + var timeout = options.UseHelix ? "15minutes" : "25minutes"; builder.Append($" --blame-hang-dump-type full --blame-hang-timeout {timeout}"); return builder.ToString(); } - private string GetResultsFilePath(AssemblyInfo assemblyInfo, string suffix = "xml") + private string GetResultsFilePath(WorkItemInfo workItemInfo, Options options, string suffix = "xml") { - var fileName = $"{assemblyInfo.DisplayName}_{assemblyInfo.TargetFramework}_{assemblyInfo.Architecture}_test_results.{suffix}"; + var fileName = $"WorkItem_{workItemInfo.PartitionIndex}_{options.Architecture}_test_results.{suffix}"; return Path.Combine(Options.TestResultsDirectory, fileName); } - public async Task RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken) + public async Task RunTestAsync(WorkItemInfo workItemInfo, Options options, CancellationToken cancellationToken) { - var result = await RunTestAsyncInternal(assemblyInfo, retry: false, cancellationToken); + var result = await RunTestAsyncInternal(workItemInfo, options, isRetry: false, cancellationToken); // For integration tests (TestVsi), we make one more attempt to re-run failed tests. - if (Options.Retry && !HasBuiltInRetry(assemblyInfo) && !Options.IncludeHtml && !result.Succeeded) + if (Options.Retry && !HasBuiltInRetry(workItemInfo) && !Options.IncludeHtml && !result.Succeeded) { - return await RunTestAsyncInternal(assemblyInfo, retry: true, cancellationToken); + return await RunTestAsyncInternal(workItemInfo, options, isRetry: true, cancellationToken); } return result; - static bool HasBuiltInRetry(AssemblyInfo assemblyInfo) + static bool HasBuiltInRetry(WorkItemInfo workItemInfo) { // vs-extension-testing handles test retry internally. - return assemblyInfo.AssemblyName == "Microsoft.VisualStudio.LanguageServices.New.IntegrationTests.dll"; + return workItemInfo.TypesToTest.Keys.Any(assembly => assembly.AssemblyName == "Microsoft.VisualStudio.LanguageServices.New.IntegrationTests.dll"); } } - private async Task RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken) + private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, Options options, bool isRetry, CancellationToken cancellationToken) { try { - var commandLineArguments = GetCommandLineArguments(assemblyInfo, useSingleQuotes: false, isHelix: false); - var resultsFilePath = GetResultsFilePath(assemblyInfo); + var commandLineArguments = GetCommandLineArguments(workItemInfo, useSingleQuotes: false, options); + var resultsFilePath = GetResultsFilePath(workItemInfo, options); var resultsDir = Path.GetDirectoryName(resultsFilePath); - var htmlResultsFilePath = Options.IncludeHtml ? GetResultsFilePath(assemblyInfo, "html") : null; + var htmlResultsFilePath = Options.IncludeHtml ? GetResultsFilePath(workItemInfo, options, "html") : null; var processResultList = new List(); ProcessInfo? procDumpProcessInfo = null; @@ -144,7 +149,7 @@ private async Task RunTestAsyncInternal(AssemblyInfo assemblyInfo, b var environmentVariables = new Dictionary(); Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables); - if (retry && File.Exists(resultsFilePath)) + if (isRetry && File.Exists(resultsFilePath)) { ConsoleUtil.WriteLine("Starting a retry. Tests which failed will run a second time to reduce flakiness."); try @@ -178,23 +183,22 @@ private async Task RunTestAsyncInternal(AssemblyInfo assemblyInfo, b ProcessRunner.CreateProcessStartInfo( Options.DotnetFilePath, commandLineArguments, - workingDirectory: Path.GetDirectoryName(assemblyInfo.AssemblyPath), displayWindow: false, captureOutput: true, environmentVariables: environmentVariables), lowPriority: false, cancellationToken: cancellationToken); - Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName}"); + Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName}"); var xunitProcessResult = await dotnetProcessInfo.Result; var span = DateTime.UtcNow - start; - Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}"); + Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName} with code {xunitProcessResult.ExitCode}"); processResultList.Add(xunitProcessResult); if (procDumpProcessInfo != null) { var procDumpProcessResult = await procDumpProcessInfo.Value.Result; - Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}"); + Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName} with code {procDumpProcessResult.ExitCode}"); processResultList.Add(procDumpProcessResult); } @@ -223,7 +227,7 @@ private async Task RunTestAsyncInternal(AssemblyInfo assemblyInfo, b } } - Logger.Log($"Command line {assemblyInfo.DisplayName} completed in {span.TotalSeconds} seconds: {Options.DotnetFilePath} {commandLineArguments}"); + Logger.Log($"Command line {workItemInfo.DisplayName} completed in {span.TotalSeconds} seconds: {Options.DotnetFilePath} {commandLineArguments}"); var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? ""; var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? ""; @@ -236,14 +240,14 @@ private async Task RunTestAsyncInternal(AssemblyInfo assemblyInfo, b errorOutput: errorOutput); return new TestResult( - assemblyInfo, + workItemInfo, testResultInfo, commandLineArguments, processResults: ImmutableArray.CreateRange(processResultList)); } catch (Exception ex) { - throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.DotnetFilePath}. {ex}"); + throw new Exception($"Unable to run {workItemInfo.DisplayName} with {Options.DotnetFilePath}. {ex}"); } } } diff --git a/src/Tools/Source/RunTests/Program.cs b/src/Tools/Source/RunTests/Program.cs index 041fb5af96b0e..740c9f2608858 100644 --- a/src/Tools/Source/RunTests/Program.cs +++ b/src/Tools/Source/RunTests/Program.cs @@ -13,6 +13,8 @@ using System.Collections.Immutable; using System.Diagnostics; using System.Text.RegularExpressions; +using PrepareTests; +using System.Xml.Linq; namespace RunTests { @@ -129,20 +131,20 @@ private static async Task RunAsync(Options options, CancellationToken cance var testExecutor = CreateTestExecutor(options); var testRunner = new TestRunner(options, testExecutor); var start = DateTime.Now; - var assemblyInfoList = GetAssemblyList(options); - if (assemblyInfoList.Count == 0) + var workItems = GetWorkItems(options); + if (workItems.Length == 0) { + WriteLogFile(options); ConsoleUtil.WriteLine(ConsoleColor.Red, "No assemblies to test"); return ExitFailure; } - var assemblyCount = assemblyInfoList.GroupBy(x => x.AssemblyPath).Count(); ConsoleUtil.WriteLine($"Proc dump location: {options.ProcDumpFilePath}"); - ConsoleUtil.WriteLine($"Running {assemblyCount} test assemblies in {assemblyInfoList.Count} partitions"); + ConsoleUtil.WriteLine($"Running tests in {workItems.Length} partitions"); var result = options.UseHelix - ? await testRunner.RunAllOnHelixAsync(assemblyInfoList, cancellationToken).ConfigureAwait(true) - : await testRunner.RunAllAsync(assemblyInfoList, cancellationToken).ConfigureAwait(true); + ? await testRunner.RunAllOnHelixAsync(workItems, options, cancellationToken).ConfigureAwait(true) + : await testRunner.RunAllAsync(workItems, cancellationToken).ConfigureAwait(true); var elapsed = DateTime.Now - start; ConsoleUtil.WriteLine($"Test execution time: {elapsed}"); @@ -280,83 +282,59 @@ async Task DumpProcess(Process targetProcess, string procDumpExeFilePath, string return null; } - private static List GetAssemblyList(Options options) + private static ImmutableArray GetWorkItems(Options options) { var scheduler = new AssemblyScheduler(options); - var list = new List(); var assemblyPaths = GetAssemblyFilePaths(options); - - foreach (var assemblyPath in assemblyPaths.OrderByDescending(x => new FileInfo(x.FilePath).Length)) - { - list.AddRange(scheduler.Schedule(assemblyPath.FilePath).Select(x => new AssemblyInfo(x, assemblyPath.TargetFramework, options.Architecture))); - } - - return list; + var workItems = scheduler.Schedule(assemblyPaths); + return workItems; } - private static List<(string FilePath, string TargetFramework)> GetAssemblyFilePaths(Options options) + private static ImmutableArray GetAssemblyFilePaths(Options options) { - var list = new List<(string, string)>(); + var list = new List(); var binDirectory = Path.Combine(options.ArtifactsDirectory, "bin"); - foreach (var project in Directory.EnumerateDirectories(binDirectory, "*", SearchOption.TopDirectoryOnly)) - { - var name = Path.GetFileName(project); + var assemblies = ListTests.GetTestAssemblyFilePaths(binDirectory, ShouldSkip, options.Configuration, options.TargetFrameworks); + + return assemblies; + + bool ShouldSkip(string projectDirectory) + { + var name = Path.GetFileName(projectDirectory); if (!shouldInclude(name, options) || shouldExclude(name, options)) { - continue; + Logger.Log($"Skipped {name} as it was not included"); + return true; } - var fileName = $"{name}.dll"; - foreach (var targetFramework in options.TargetFrameworks) + return false; + + static bool shouldInclude(string name, Options options) { - var fileContainingDirectory = Path.Combine(project, options.Configuration, targetFramework); - var filePath = Path.Combine(fileContainingDirectory, fileName); - if (File.Exists(filePath)) + foreach (var pattern in options.IncludeFilter) { - list.Add((filePath, targetFramework)); - } - else if (Directory.Exists(fileContainingDirectory) && Directory.GetFiles(fileContainingDirectory, searchPattern: "*.UnitTests.dll") is { Length: > 0 } matches) - { - // If the unit test assembly name doesn't match the project folder name, but still matches our "unit test" name pattern, we want to run it. - // If more than one such assembly is present in a project output folder, we assume something is wrong with the build configuration. - // For example, one unit test project might be referencing another unit test project. - if (matches.Length > 1) + if (Regex.IsMatch(name, pattern.Trim('\'', '"'))) { - var message = $"Multiple unit test assemblies found in '{fileContainingDirectory}'. Please adjust the build to prevent this. Matches:{Environment.NewLine}{string.Join(Environment.NewLine, matches)}"; - throw new Exception(message); + return true; } - list.Add((matches[0], targetFramework)); } - } - } - - return list; - static bool shouldInclude(string name, Options options) - { - foreach (var pattern in options.IncludeFilter) - { - if (Regex.IsMatch(name, pattern.Trim('\'', '"'))) - { - return true; - } + return false; } - return false; - } - - static bool shouldExclude(string name, Options options) - { - foreach (var pattern in options.ExcludeFilter) + static bool shouldExclude(string name, Options options) { - if (Regex.IsMatch(name, pattern.Trim('\'', '"'))) + foreach (var pattern in options.ExcludeFilter) { - return true; + if (Regex.IsMatch(name, pattern.Trim('\'', '"'))) + { + return true; + } } - } - return false; + return false; + } } } diff --git a/src/Tools/Source/RunTests/RunTests.csproj b/src/Tools/Source/RunTests/RunTests.csproj index f2e74fa95453e..0b6a995b667a0 100644 --- a/src/Tools/Source/RunTests/RunTests.csproj +++ b/src/Tools/Source/RunTests/RunTests.csproj @@ -17,4 +17,7 @@ + + + \ No newline at end of file diff --git a/src/Tools/Source/RunTests/TestRunner.cs b/src/Tools/Source/RunTests/TestRunner.cs index 8bf1a7dd09c97..3a36910b04754 100644 --- a/src/Tools/Source/RunTests/TestRunner.cs +++ b/src/Tools/Source/RunTests/TestRunner.cs @@ -14,7 +14,9 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Mono.Options; using Newtonsoft.Json; +using PrepareTests; namespace RunTests { @@ -43,7 +45,7 @@ internal TestRunner(Options options, ProcessTestExecutor testExecutor) _options = options; } - internal async Task RunAllOnHelixAsync(IEnumerable assemblyInfoList, CancellationToken cancellationToken) + internal async Task RunAllOnHelixAsync(ImmutableArray workItems, Options options, CancellationToken cancellationToken) { var sourceBranch = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH"); if (sourceBranch is null) @@ -95,7 +97,7 @@ internal async Task RunAllOnHelixAsync(IEnumerable a Environment.SetEnvironmentVariable("BUILD_REASON", "pr"); var buildNumber = Environment.GetEnvironmentVariable("BUILD_BUILDNUMBER") ?? "0"; - var workItems = assemblyInfoList.Select(ai => makeHelixWorkItemProject(ai)); + var helixWorkItems = workItems.Select(workItem => MakeHelixWorkItemProject(workItem)); var globalJson = JsonConvert.DeserializeAnonymousType(File.ReadAllText(getGlobalJsonPath()), new { sdk = new { version = "" } }) ?? throw new InvalidOperationException("Failed to deserialize global.json."); @@ -116,7 +118,7 @@ internal async Task RunAllOnHelixAsync(IEnumerable a - " + correlationPayload + string.Join("", workItems) + @" + " + correlationPayload + string.Join("", helixWorkItems) + @" "; @@ -148,23 +150,46 @@ static string getGlobalJsonPath() throw new IOException($@"Could not find global.json by walking up from ""{AppContext.BaseDirectory}""."); } - string makeHelixWorkItemProject(AssemblyInfo assemblyInfo) + static void AddRehydrateTestFoldersCommand(StringBuilder commandBuilder, WorkItemInfo workItem, bool isUnix) + { + // Rehydrate assemblies that we need to run as part of this work item. + foreach (var testAssembly in workItem.TypesToTest.Keys) + { + var directoryName = Path.GetDirectoryName(testAssembly.AssemblyPath); + if (isUnix) + { + // If we're on unix make sure we have permissions to run the rehydrate script. + commandBuilder.AppendLine($"chmod +x {directoryName}/rehydrate.sh"); + } + + commandBuilder.AppendLine(isUnix ? $"./{directoryName}/rehydrate.sh" : $@"call {directoryName}\rehydrate.cmd"); + commandBuilder.AppendLine(isUnix ? $"ls -l {directoryName}" : $"dir {directoryName}"); + } + } + + static string GetHelixRelativeAssemblyPath(string assemblyPath) + { + var tfmDir = Path.GetDirectoryName(assemblyPath)!; + var configurationDir = Path.GetDirectoryName(tfmDir)!; + var projectDir = Path.GetDirectoryName(configurationDir)!; + + var assemblyRelativePath = Path.Combine(Path.GetFileName(projectDir), Path.GetFileName(configurationDir), Path.GetFileName(tfmDir), Path.GetFileName(assemblyPath)); + return assemblyRelativePath; + } + + string MakeHelixWorkItemProject(WorkItemInfo workItemInfo) { // Currently, it's required for the client machine to use the same OS family as the target Helix queue. // We could relax this and allow for example Linux clients to kick off Windows jobs, but we'd have to // figure out solutions for issues such as creating file paths in the correct format for the target machine. var isUnix = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - var commandLineArguments = _testExecutor.GetCommandLineArguments(assemblyInfo, useSingleQuotes: isUnix, isHelix: true); - commandLineArguments = SecurityElement.Escape(commandLineArguments); var setEnvironmentVariable = isUnix ? "export" : "set"; var command = new StringBuilder(); - command.AppendLine(isUnix ? "ls -l" : "dir"); - command.AppendLine(isUnix ? $"./rehydrate.sh" : $@"call .\rehydrate.cmd"); - command.AppendLine(isUnix ? "ls -l" : "dir"); command.AppendLine($"{setEnvironmentVariable} DOTNET_ROLL_FORWARD=LatestMajor"); command.AppendLine($"{setEnvironmentVariable} DOTNET_ROLL_FORWARD_TO_PRERELEASE=1"); + command.AppendLine(isUnix ? $"ls -l" : $"dir"); command.AppendLine("dotnet --info"); var knownEnvironmentVariables = new[] { "ROSLYN_TEST_IOPERATION", "ROSLYN_TEST_USEDASSEMBLIES" }; @@ -176,7 +201,24 @@ string makeHelixWorkItemProject(AssemblyInfo assemblyInfo) } } - command.AppendLine($"dotnet {commandLineArguments}"); + // Create a payload directory that contains all the assemblies in the work item in separate folders. + var payloadDirectory = Path.Combine(msbuildTestPayloadRoot, "artifacts", "bin"); + + // Get the assembly path in the context of the helix work item. + workItemInfo = workItemInfo with + { + TypesToTest = workItemInfo.TypesToTest.ToImmutableSortedDictionary( + kvp => kvp.Key with { AssemblyPath = GetHelixRelativeAssemblyPath(kvp.Key.AssemblyPath) }, + kvp => kvp.Value) + }; + + AddRehydrateTestFoldersCommand(command, workItemInfo, isUnix); + + var commandLineArguments = _testExecutor.GetCommandLineArguments(workItemInfo, isUnix, options); + // XML escape the arguments as the commands are output into the helix project xml file. + commandLineArguments = SecurityElement.Escape(commandLineArguments); + var dotnetTestCommand = $"dotnet {commandLineArguments}"; + command.AppendLine(dotnetTestCommand); // We want to collect any dumps during the post command step here; these commands are ran after the // return value of the main command is captured; a Helix Job is considered to fail if the main command returns a @@ -184,8 +226,6 @@ string makeHelixWorkItemProject(AssemblyInfo assemblyInfo) // precisely to address this problem. var postCommands = new StringBuilder(); - var payloadDirectory = Path.Combine(msbuildTestPayloadRoot, Path.GetDirectoryName(assemblyInfo.AssemblyPath)!); - if (isUnix) { // Write out this command into a separate file; unfortunately the use of single quotes and ; that is required @@ -199,7 +239,7 @@ string makeHelixWorkItemProject(AssemblyInfo assemblyInfo) } var workItem = $@" - + {payloadDirectory} {command} @@ -214,13 +254,13 @@ string makeHelixWorkItemProject(AssemblyInfo assemblyInfo) } } - internal async Task RunAllAsync(IEnumerable assemblyInfoList, CancellationToken cancellationToken) + internal async Task RunAllAsync(ImmutableArray workItems, CancellationToken cancellationToken) { // Use 1.5 times the number of processors for unit tests, but only 1 processor for the open integration tests // since they perform actual UI operations (such as mouse clicks and sending keystrokes) and we don't want two // tests to conflict with one-another. var max = _options.Sequential ? 1 : (int)(Environment.ProcessorCount * 1.5); - var waiting = new Stack(assemblyInfoList); + var waiting = new Stack(workItems); var running = new List>(); var completed = new List(); var failures = 0; @@ -275,7 +315,7 @@ internal async Task RunAllAsync(IEnumerable assembly while (running.Count < max && waiting.Count > 0) { - var task = _testExecutor.RunTestAsync(waiting.Pop(), cancellationToken); + var task = _testExecutor.RunTestAsync(waiting.Pop(), _options, cancellationToken); running.Add(task); } @@ -343,7 +383,7 @@ private void PrintFailedTestResult(TestResult testResult) // Save out the error output for easy artifact inspecting var outputLogPath = Path.Combine(_options.LogFilesDirectory, $"xUnitFailure-{testResult.DisplayName}.log"); - ConsoleUtil.WriteLine($"Errors {testResult.AssemblyName}"); + ConsoleUtil.WriteLine($"Errors {testResult.DisplayName}"); ConsoleUtil.WriteLine(testResult.ErrorOutput); // TODO: Put this in the log and take it off the ConsoleUtil output to keep it simple? From 1bdef98f28e272b51b863c7f8e14c25cdadc4fa1 Mon Sep 17 00:00:00 2001 From: David Barbet Date: Wed, 20 Jul 2022 17:10:28 -0700 Subject: [PATCH 03/26] more work --- eng/prepare-tests.ps1 | 2 +- eng/prepare-tests.sh | 2 +- src/Tools/PrepareTests/AssemblyInfo.cs | 28 -- src/Tools/PrepareTests/ListTests.cs | 231 ------------ src/Tools/PrepareTests/ProcessRunner.cs | 202 ---------- src/Tools/PrepareTests/Program.cs | 15 +- .../Source/RunTests/AssemblyScheduler.cs | 350 ++++++++++++++---- src/Tools/Source/RunTests/ITestExecutor.cs | 1 - .../Source/RunTests/ProcessTestExecutor.cs | 7 +- src/Tools/Source/RunTests/Program.cs | 78 ++-- src/Tools/Source/RunTests/RunTests.csproj | 4 +- src/Tools/Source/RunTests/TestRunner.cs | 11 +- 12 files changed, 336 insertions(+), 595 deletions(-) delete mode 100644 src/Tools/PrepareTests/AssemblyInfo.cs delete mode 100644 src/Tools/PrepareTests/ListTests.cs delete mode 100644 src/Tools/PrepareTests/ProcessRunner.cs diff --git a/eng/prepare-tests.ps1 b/eng/prepare-tests.ps1 index fcd9c03894fcf..427f9ba35d0dd 100644 --- a/eng/prepare-tests.ps1 +++ b/eng/prepare-tests.ps1 @@ -11,7 +11,7 @@ try { $dotnet = Ensure-DotnetSdk # permissions issues make this a pain to do in PrepareTests itself. Remove-Item -Recurse -Force "$RepoRoot\artifacts\testPayload" -ErrorAction SilentlyContinue - $cmdOutput = Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload --dotnetPath `"$dotnet`"" + $cmdOutput = Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload" echo $cmdOutput exit 0 } diff --git a/eng/prepare-tests.sh b/eng/prepare-tests.sh index e4ee6bb52089e..5b0310573bd1f 100755 --- a/eng/prepare-tests.sh +++ b/eng/prepare-tests.sh @@ -28,4 +28,4 @@ InitializeDotNetCli true # permissions issues make this a pain to do in PrepareTests itself. rm -rf "$repo_root/artifacts/testPayload" -dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix --dotnetPath ${_InitializeDotNetCli}/dotnet +dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix diff --git a/src/Tools/PrepareTests/AssemblyInfo.cs b/src/Tools/PrepareTests/AssemblyInfo.cs deleted file mode 100644 index 9faffef67ab1f..0000000000000 --- a/src/Tools/PrepareTests/AssemblyInfo.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.IO; - -namespace PrepareTests; - -public readonly record struct AssemblyInfo(string AssemblyPath) : IComparable -{ - public string AssemblyName => Path.GetFileName(AssemblyPath); - - public int CompareTo(object? obj) - { - if (obj == null) - { - return 1; - } - - var otherAssembly = (AssemblyInfo)obj; - - // Ensure we have a consistent ordering by ordering by assembly path. - return this.AssemblyPath.CompareTo(otherAssembly.AssemblyPath); - } -} - -public readonly record struct TypeInfo(string Name, string FullyQualifiedName, int TestCount); diff --git a/src/Tools/PrepareTests/ListTests.cs b/src/Tools/PrepareTests/ListTests.cs deleted file mode 100644 index 20952a5fc5bc0..0000000000000 --- a/src/Tools/PrepareTests/ListTests.cs +++ /dev/null @@ -1,231 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Diagnostics; -using System.Diagnostics.Contracts; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Reflection.Metadata; -using System.Reflection.PortableExecutable; -using System.Runtime.InteropServices; -using System.Runtime.Versioning; -using System.Text; -using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Mono.Options; - -namespace PrepareTests; -public class ListTests -{ - /// - /// Regex to find test lines from the output of dotnet test. - /// - /// - /// The goal is to match lines that contain a fully qualified test name e.g. - /// Microsoft.CodeAnalysis.Editor.CSharp.UnitTests.RemoveUnusedParametersAndValues.RemoveUnusedValueAssignmentTests.UnusedVarPattern_PartOfIs - /// or - /// action = (int [|"..., pascalCaseSymbol: "void Outer() { System.Action action = (int M)"..., symbolKind: Parameter, accessibility: NotApplicable)]]> - /// But not anything else dotnet test --list-tests outputs, like - /// - /// Microsoft (R) Test Execution Command Line Tool Version 17.3.0-preview-20220414-05 (x64) - /// Copyright(c) Microsoft Corporation. All rights reserved. - /// - /// The following Tests are available: - /// - /// The regex looks for the namespace names (groups of non-whitespace characters followed by a dot) at the beginning of the line. - /// - private static readonly Regex TestOutputFormat = new("^(\\S)*\\..*", RegexOptions.Compiled); - - internal static async Task RunAsync(string sourceDirectory, string dotnetPath) - { - // Find all test assemblies. - var binDirectory = Path.Combine(sourceDirectory, "artifacts", "bin"); - var assemblies = GetTestAssemblyFilePaths(binDirectory); - Console.WriteLine($"Found test assemblies:{Environment.NewLine}{string.Join(Environment.NewLine, assemblies.Select(a => a.AssemblyPath))}"); - - var stopwatch = new Stopwatch(); - stopwatch.Start(); - - // Discover tests via `dotnet test --list-tests` and write out the test counts to a file so we can use it for partitioning later. - await GetTypeInfoAsync(assemblies, dotnetPath); - stopwatch.Stop(); - Console.WriteLine($"Discovered tests in {stopwatch.Elapsed}"); - } - - /// - /// Returns the file path of the test data results for a particular assembly. - /// This is written using the output of dotnet test list in - /// and read during each test leg for partitioning. - /// - public static string GetTestDataFilePath(AssemblyInfo assembly) - { - var assemblyDirectory = Path.GetDirectoryName(assembly.AssemblyPath)!; - var fileName = $"{assembly.AssemblyName}_tests.txt"; - var outputPath = Path.Combine(assemblyDirectory, fileName); - return outputPath; - } - - /// - /// Find all unit test assemblies that we need to discover tests on. - /// - public static ImmutableArray GetTestAssemblyFilePaths( - string binDirectory, - Func? shouldSkipTestDirectory = null, - string configurationSearchPattern = "*", - List? targetFrameworks = null) - { - var list = new List(); - - // Find all the project folders that fit our naming scheme for unit tests. - foreach (var project in Directory.EnumerateDirectories(binDirectory, "*.UnitTests", SearchOption.TopDirectoryOnly)) - { - if (shouldSkipTestDirectory != null && shouldSkipTestDirectory(project)) - { - continue; - } - - var name = Path.GetFileName(project); - var fileName = $"{name}.dll"; - - // Find the dlls matching the request configuration and target frameworks. - var configurationDirectories = Directory.EnumerateDirectories(project, configurationSearchPattern, SearchOption.TopDirectoryOnly); - foreach (var configuration in configurationDirectories) - { - var targetFrameworkDirectories = Directory.EnumerateDirectories(configuration, "*", SearchOption.TopDirectoryOnly); - if (targetFrameworks != null) - { - targetFrameworkDirectories = targetFrameworks.Select(tfm => Path.Combine(configuration, tfm)); - } - - foreach (var targetFrameworkDirectory in targetFrameworkDirectories) - { - // In multi-targeting scenarios we will build both .net core and .net framework versions of the assembly on unix. - // If we're on unix and we see the .net framework assembly, skip it as we can't list or run tests on it anyway. - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Path.GetFileName(targetFrameworkDirectory) == "net472") - { - Console.WriteLine($"Skipping net472 assembly on unix: {targetFrameworkDirectory}"); - continue; - } - - var filePath = Path.Combine(targetFrameworkDirectory, fileName); - if (File.Exists(filePath)) - { - list.Add(new AssemblyInfo(filePath)); - } - else if (Directory.Exists(targetFrameworkDirectory) && Directory.GetFiles(targetFrameworkDirectory, searchPattern: "*.UnitTests.dll") is { Length: > 0 } matches) - { - // If the unit test assembly name doesn't match the project folder name, but still matches our "unit test" name pattern, we want to run it. - // If more than one such assembly is present in a project output folder, we assume something is wrong with the build configuration. - // For example, one unit test project might be referencing another unit test project. - if (matches.Length > 1) - { - var message = $"Multiple unit test assemblies found in '{targetFrameworkDirectory}'. Please adjust the build to prevent this. Matches:{Environment.NewLine}{string.Join(Environment.NewLine, matches)}"; - throw new Exception(message); - } - list.Add(new AssemblyInfo(matches[0])); - } - } - } - } - - Contract.Assert(list.Count > 0, $"Did not find any test assemblies"); - - list.Sort(); - return list.ToImmutableArray(); - } - - /// - /// Runs `dotnet test _assembly_ --list-tests` on all test assemblies to get the real count of tests per assembly. - /// - private static async Task GetTypeInfoAsync(ImmutableArray assemblies, string dotnetFilePath) - { - // We run this one assembly at a time because it will not output which test is in which assembly otherwise. - // It's also faster than making a single call to dotnet test with all the assemblies. - await Parallel.ForEachAsync(assemblies, async (assembly, cancellationToken) => - { - var assemblyPath = assembly.AssemblyPath; - var commandArgs = $"test {assemblyPath} --list-tests"; - var processResult = await ProcessRunner.CreateProcess(dotnetFilePath, commandArgs, workingDirectory: Directory.GetCurrentDirectory(), captureOutput: true, displayWindow: false, cancellationToken: cancellationToken).Result; - if (processResult.ExitCode != 0) - { - var errorOutput = string.Join(Environment.NewLine, processResult.ErrorLines); - var output = string.Join(Environment.NewLine, processResult.OutputLines); - throw new InvalidOperationException($"dotnet test failed with {processResult.ExitCode} for {assemblyPath}.{Environment.NewLine}Error output: {errorOutput}{Environment.NewLine}Output: {output}"); - } - - var typeInfo = ParseDotnetTestOutput(processResult.OutputLines); - var testCount = typeInfo.Sum(type => type.TestCount); - Contract.Assert(testCount > 0, $"Did not find any tests in {assembly}, output was {Environment.NewLine}{string.Join(Environment.NewLine, processResult.OutputLines)}"); - - Console.WriteLine($"Found {testCount} tests for {assemblyPath}"); - - await WriteTestDataAsync(assembly, typeInfo, cancellationToken); - }); - - return; - - static async Task WriteTestDataAsync(AssemblyInfo assembly, ImmutableArray typeInfo, CancellationToken cancellationToken) - { - var outputPath = GetTestDataFilePath(assembly); - - using var createStream = File.Create(outputPath); - await JsonSerializer.SerializeAsync(createStream, typeInfo, cancellationToken: cancellationToken); - await createStream.DisposeAsync(); - } - } - - /// - /// Parse the output of `dotnet test` to count the number of tests in the assembly by type name. - /// - private static ImmutableArray ParseDotnetTestOutput(IEnumerable output) - { - // Find all test lines from the output of dotnet test using a regex match. - var testList = output.Select(line => line.TrimStart()).Where(line => TestOutputFormat.IsMatch(line)); - - // Figure out the fully qualified type name for each test. - var typeList = testList - .Where(line => !string.IsNullOrWhiteSpace(line)) - .Select(GetFullyQualifiedTypeName); - - // Count all occurences of the type name in the test list to figure out how many tests per type we have. - var groups = typeList.GroupBy(type => type); - var result = groups.Select(group => new TypeInfo(GetTypeNameFromFullyQualifiedName(group.Key), group.Key, group.Count())).ToImmutableArray(); - return result; - - static string GetFullyQualifiedTypeName(string testLine) - { - // Remove whitespace from the start as the list is always indented. - var test = testLine.TrimStart(); - // The common case is just a fully qualified method name e.g. - // Microsoft.CodeAnalysis.Editor.CSharp.UnitTests.RemoveUnusedParametersAndValues.RemoveUnusedValueAssignmentTests.UnusedVarPattern_PartOfIs - // - // However, we can also have more complex expressions with actual code in them (and periods) like - // Microsoft.CodeAnalysis.Editor.CSharp.UnitTests.Diagnostics.NamingStyles.NamingStylesTests.TestPascalCaseSymbol_ExpectedSymbolAndAccessibility(camelCaseSymbol: "void Outer() { System.Action action = (int [|"..., pascalCaseSymbol: "void Outer() { System.Action action = (int M)"..., symbolKind: Parameter, accessibility: NotApplicable) - // - // So we first split on ( to get the fully qualified method name. This is valid because the namespace, type name, and method name cannot have parens. - // The first part of the split gives us the fully qualified method name. From that we can take everything up until the last period get the fully qualified type name. - var splitString = test.Split("("); - var fullyQualifiedMethod = splitString[0]; - - var periodBeforeMethodName = fullyQualifiedMethod.LastIndexOf("."); - var fullyQualifiedType = fullyQualifiedMethod[..periodBeforeMethodName]; - - return fullyQualifiedType; - } - - static string GetTypeNameFromFullyQualifiedName(string fullyQualifiedTypeName) - { - var periodBeforeTypeName = fullyQualifiedTypeName.LastIndexOf("."); - var typeName = fullyQualifiedTypeName[(periodBeforeTypeName + 1)..]; - return typeName; - } - } -} diff --git a/src/Tools/PrepareTests/ProcessRunner.cs b/src/Tools/PrepareTests/ProcessRunner.cs deleted file mode 100644 index f5c247f87d1f9..0000000000000 --- a/src/Tools/PrepareTests/ProcessRunner.cs +++ /dev/null @@ -1,202 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Diagnostics; -using System.IO; -using System.Runtime.InteropServices; -using System.Threading; -using System.Threading.Tasks; - -namespace PrepareTests; - -public readonly struct ProcessResult -{ - public Process Process { get; } - public int ExitCode { get; } - public ReadOnlyCollection OutputLines { get; } - public ReadOnlyCollection ErrorLines { get; } - - public ProcessResult(Process process, int exitCode, ReadOnlyCollection outputLines, ReadOnlyCollection errorLines) - { - Process = process; - ExitCode = exitCode; - OutputLines = outputLines; - ErrorLines = errorLines; - } -} - -public readonly struct ProcessInfo -{ - public Process Process { get; } - public ProcessStartInfo StartInfo { get; } - public Task Result { get; } - - public int Id => Process.Id; - - public ProcessInfo(Process process, ProcessStartInfo startInfo, Task result) - { - Process = process; - StartInfo = startInfo; - Result = result; - } -} - -public static class ProcessRunner -{ - public static void OpenFile(string file) - { - if (File.Exists(file)) - { - Process.Start(file); - } - } - - public static ProcessInfo CreateProcess( - string executable, - string arguments, - bool lowPriority = false, - string? workingDirectory = null, - bool captureOutput = false, - bool displayWindow = true, - Dictionary? environmentVariables = null, - Action? onProcessStartHandler = null, - Action? onOutputDataReceived = null, - CancellationToken cancellationToken = default) => - CreateProcess( - CreateProcessStartInfo(executable, arguments, workingDirectory, captureOutput, displayWindow, environmentVariables), - lowPriority: lowPriority, - onProcessStartHandler: onProcessStartHandler, - onOutputDataReceived: onOutputDataReceived, - cancellationToken: cancellationToken); - - public static ProcessInfo CreateProcess( - ProcessStartInfo processStartInfo, - bool lowPriority = false, - Action? onProcessStartHandler = null, - Action? onOutputDataReceived = null, - CancellationToken cancellationToken = default) - { - var errorLines = new List(); - var outputLines = new List(); - var process = new Process(); - var tcs = new TaskCompletionSource(); - - process.EnableRaisingEvents = true; - process.StartInfo = processStartInfo; - - process.OutputDataReceived += (s, e) => - { - if (e.Data != null) - { - onOutputDataReceived?.Invoke(e); - outputLines.Add(e.Data); - } - }; - - process.ErrorDataReceived += (s, e) => - { - if (e.Data != null) - { - errorLines.Add(e.Data); - } - }; - - process.Exited += (s, e) => - { - // We must call WaitForExit to make sure we've received all OutputDataReceived/ErrorDataReceived calls - // or else we'll be returning a list we're still modifying. For paranoia, we'll start a task here rather - // than enter right back into the Process type and start a wait which isn't guaranteed to be safe. - Task.Run(() => - { - process.WaitForExit(); - var result = new ProcessResult( - process, - process.ExitCode, - new ReadOnlyCollection(outputLines), - new ReadOnlyCollection(errorLines)); - tcs.TrySetResult(result); - }, cancellationToken); - }; - - var registration = cancellationToken.Register(() => - { - if (tcs.TrySetCanceled()) - { - // If the underlying process is still running, we should kill it - if (!process.HasExited) - { - try - { - process.Kill(); - } - catch (InvalidOperationException) - { - // Ignore, since the process is already dead - } - } - } - }); - - process.Start(); - onProcessStartHandler?.Invoke(process); - - if (lowPriority) - { - process.PriorityClass = ProcessPriorityClass.BelowNormal; - } - - if (processStartInfo.RedirectStandardOutput) - { - process.BeginOutputReadLine(); - } - - if (processStartInfo.RedirectStandardError) - { - process.BeginErrorReadLine(); - } - - return new ProcessInfo(process, processStartInfo, tcs.Task); - } - - public static ProcessStartInfo CreateProcessStartInfo( - string executable, - string arguments, - string? workingDirectory = null, - bool captureOutput = false, - bool displayWindow = true, - Dictionary? environmentVariables = null) - { - var processStartInfo = new ProcessStartInfo(executable, arguments); - - if (!string.IsNullOrEmpty(workingDirectory)) - { - processStartInfo.WorkingDirectory = workingDirectory; - } - - if (environmentVariables != null) - { - foreach (var pair in environmentVariables) - { - processStartInfo.EnvironmentVariables[pair.Key] = pair.Value; - } - } - - if (captureOutput) - { - processStartInfo.UseShellExecute = false; - processStartInfo.RedirectStandardOutput = true; - processStartInfo.RedirectStandardError = true; - } - else - { - processStartInfo.CreateNoWindow = !displayWindow; - processStartInfo.UseShellExecute = displayWindow; - } - - return processStartInfo; - } -} diff --git a/src/Tools/PrepareTests/Program.cs b/src/Tools/PrepareTests/Program.cs index ff48425f22f9c..1e9c3429566df 100644 --- a/src/Tools/PrepareTests/Program.cs +++ b/src/Tools/PrepareTests/Program.cs @@ -13,19 +13,17 @@ internal static class Program internal const int ExitFailure = 1; internal const int ExitSuccess = 0; - public static async Task Main(string[] args) + public static int Main(string[] args) { string? source = null; string? destination = null; bool isUnix = false; - string? dotnetPath = null; var options = new OptionSet() { { "source=", "Path to binaries", (string s) => source = s }, { "destination=", "Output path", (string s) => destination = s }, { "unix", "If true, prepares tests for unix environment instead of Windows", o => isUnix = o is object }, - { "dotnetPath=", "Output path", (string s) => dotnetPath = s }, }; options.Parse(args); @@ -41,17 +39,6 @@ public static async Task Main(string[] args) return ExitFailure; } - if (dotnetPath is null) - { - Console.Error.WriteLine("--dotnetPath argument must be provided"); - return ExitFailure; - } - - // Figure out what tests we need to run before we minimize everything. - Console.WriteLine("Discovering tests..."); - await ListTests.RunAsync(source, dotnetPath); - - Console.WriteLine("Minimizing test payload..."); MinimizeUtil.Run(source, destination, isUnix); return ExitSuccess; } diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index 6e32838f20638..462ebd478a65f 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -2,14 +2,19 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; +using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; using System.Text.Json; -using PrepareTests; -using TypeInfo = PrepareTests.TypeInfo; +using System.Threading; +using System.Threading.Tasks; namespace RunTests { @@ -18,28 +23,26 @@ namespace RunTests /// Defines a single work item to run. The work item may contain tests from multiple assemblies /// that should be run together in the same work item. /// - internal readonly record struct WorkItemInfo(ImmutableSortedDictionary> TypesToTest, int PartitionIndex) + internal readonly record struct WorkItemInfo(ImmutableArray TypesToTest, int PartitionIndex) { internal string DisplayName { get { - var assembliesString = string.Join("_", TypesToTest.Keys.Select(assembly => Path.GetFileNameWithoutExtension(assembly.AssemblyName).Replace(".", string.Empty))); + var assembliesString = string.Join("_", TypesToTest.Select(group => Path.GetFileNameWithoutExtension(group.Assembly.AssemblyName).Replace(".", string.Empty))); return $"{assembliesString}_{PartitionIndex}"; } } internal static WorkItemInfo CreateFullAssembly(AssemblyInfo assembly, int partitionIndex) - => new(ImmutableSortedDictionary>.Empty.Add(assembly, ImmutableArray.Empty), partitionIndex); + => new(ImmutableArray.Empty.Add(new(assembly, ImmutableArray.Empty, true)), partitionIndex); } + internal readonly record struct AssemblyTypeGroup(AssemblyInfo Assembly, ImmutableArray Types, bool ContainsAllTypesInAssembly); + + internal sealed class AssemblyScheduler { - /// - /// Default number of methods to include per partition. - /// - internal const int DefaultMethodLimit = 2000; - private readonly Options _options; internal AssemblyScheduler(Options options) @@ -47,7 +50,7 @@ internal AssemblyScheduler(Options options) _options = options; } - public ImmutableArray Schedule(ImmutableArray assemblies) + public async Task> ScheduleAsync(ImmutableArray assemblies, CancellationToken cancellationToken) { Logger.Log($"Scheduling {assemblies.Length} assemblies"); if (_options.Sequential) @@ -59,120 +62,315 @@ public ImmutableArray Schedule(ImmutableArray assemb .ToImmutableArray(); } - var orderedTypeInfos = assemblies.ToImmutableSortedDictionary(assembly => assembly, GetTestData); - - var testsPerPartition = DefaultMethodLimit; - var totalTestCount = orderedTypeInfos.Values.SelectMany(type => type).Sum(type => type.TestCount); - - // We've calculated the optimal number of helix partitions based on the queuing time and test run times. - // Currently that number is 56 - TODO fill out details of computing this number. - var partitionCount = 56; + var orderedTypeInfos = assemblies.ToImmutableSortedDictionary(assembly => assembly, GetTypeInfoList); + ConsoleUtil.WriteLine($"Found {orderedTypeInfos.Values.SelectMany(t => t).Count()} test types to run in {orderedTypeInfos.Keys.Count()} assemblies"); - // We then determine how many tests we need to run in each partition based on the total number of tests to run. - if (_options.UseHelix) + // Retrieve test runtimes from azure devops historical data. + var testHistory = await TestHistoryManager.GetTestHistoryPerTypeAsync(cancellationToken); + if (testHistory.IsEmpty) { - testsPerPartition = totalTestCount / partitionCount; + // We didn't have any test history from azure devops, just partition by assembly. + return assemblies + .Select(WorkItemInfo.CreateFullAssembly) + .ToImmutableArray(); } - ConsoleUtil.WriteLine($"Found {totalTestCount} tests to run, building {partitionCount} partitions with {testsPerPartition} each"); + // Now for our current set of test methods we got from the assemblies we built, match them to tests from our test run history + // so that we can extract an estimate of the test execution time for each test. + orderedTypeInfos = UpdateTestsWithExecutionTimes(orderedTypeInfos, testHistory); + + // Our execution time limit is 4 minutes. We really want to run tests under 5 minutes, but we need to limit test execution time + // to 4 minutes to account for overhead in things like downloading assets, setting up the test host for each assembly, etc. + var executionTimeLimit = TimeSpan.FromMinutes(4); - // Build work items by adding tests from each type until we hit the limit of tests for each partition. - // This won't always result in exactly the desired number of partitions with exactly the same number of tests - this is because - // we only filter by type name in arguments to dotnet test to avoid command length errors. - // + // Create work items by partitioning tests by historical execution time with the goal of running under our time limit. // While we do our best to run tests from the same assembly together (by building work items in assembly order) it is expected - // that some work items will run tests from multiple assemblies due to large variances in the number of tests per assembly. - var workItems = BuildWorkItems(orderedTypeInfos, testsPerPartition); + // that some work items will run tests from multiple assemblies due to large variances in test execution time. + var workItems = BuildWorkItems(orderedTypeInfos, executionTimeLimit); + + ConsoleUtil.WriteLine($"Built {workItems.Length} work items"); LogWorkItems(workItems); return workItems; } - private static ImmutableArray GetTestData(AssemblyInfo assembly) + private static ImmutableSortedDictionary> UpdateTestsWithExecutionTimes( + ImmutableSortedDictionary> assemblyTypes, + ImmutableDictionary testHistory) { - var testDataFile = ListTests.GetTestDataFilePath(assembly); - Logger.Log($"Reading assembly test data for {assembly.AssemblyName} from {testDataFile}"); + // Determine the average execution time so that we can use it for tests that do not have any history. + var averageExecutionTime = TimeSpan.FromMilliseconds(testHistory.Values.Average(t => t.TotalMilliseconds)); + + // Store the types from our assemblies that we couldn't find a history for to log. + var extraLocalTypes = new HashSet(); + + // Store the types that we were able to match to historical data. + var matchedLocalTypes = new HashSet(); - using var readStream = File.OpenRead(testDataFile); - var testData = JsonSerializer.Deserialize>(readStream); - return testData; + var updated = assemblyTypes.ToImmutableSortedDictionary(kvp => kvp.Key, kvp => kvp.Value.Select(typeInfo => WithExecutionTime(typeInfo)).ToImmutableArray()); + LogResults(matchedLocalTypes, extraLocalTypes); + return updated; + + TypeInfo WithExecutionTime(TypeInfo typeInfo) + { + // Match by fully qualified test method name to azure devops historical data. + // Note for combinatorial tests, azure devops helpfully groups all sub-runs under a top level method (with combined test run times) with the same fully qualified method name + // that we'll get from looking at all the test methods in the assembly with SRM. + if (testHistory.TryGetValue(typeInfo.FullyQualifiedName, out var executionTime)) + { + matchedLocalTypes.Add(typeInfo.FullyQualifiedName); + return typeInfo with { ExecutionTime = executionTime }; + } + + // We didn't find the local type from our assembly in test run historical data. + // This can happen if our SRM heuristic incorrectly counted a normal method as a test method (which it can do often). + extraLocalTypes.Add(typeInfo.FullyQualifiedName); + return typeInfo with { ExecutionTime = averageExecutionTime }; + } + + void LogResults(HashSet matchedLocalTypes, HashSet extraLocalTypes) + { + foreach (var extraLocalType in extraLocalTypes) + { + Logger.Log($"Could not find test execution history for types in {extraLocalType}"); + } + + var extraRemoteTypes = testHistory.Keys.Where(type => !matchedLocalTypes.Contains(type)); + foreach (var extraRemoteType in extraRemoteTypes) + { + Logger.Log($"Found historical data for types in {extraRemoteType} that were not present in local assemblies"); + } + + var totalExpectedRunTime = TimeSpan.FromMilliseconds(updated.Values.SelectMany(types => types).Sum(test => test.ExecutionTime.TotalMilliseconds)); + ConsoleUtil.WriteLine($"Matched {matchedLocalTypes.Count} types with historical data. {extraLocalTypes.Count} types were missing historical data. {extraRemoteTypes.Count()} types were missing in local assemblies. Estimate of total execution time for tests is {totalExpectedRunTime}."); + } } - private static ImmutableArray BuildWorkItems(ImmutableSortedDictionary> typeInfos, int methodLimit) + private static ImmutableArray BuildWorkItems(ImmutableSortedDictionary> typeInfos, TimeSpan executionTimeLimit) { var workItems = new List(); - var currentClassNameLengthSum = 0; - var currentTestCount = 0; + // Keep track of which work item we are creating - used to identify work items in names. var workItemIndex = 0; - var currentAssemblies = new Dictionary>(); + // Keep track of the execution time of the current work item we are adding to. + var currentExecutionTime = TimeSpan.Zero; + + // Keep track of the types we're planning to add to the current work item. + var currentAssemblyTests = new Dictionary(); // Iterate through each assembly and type and build up the work items to run. - // We add types from assemblies 1 by 1 until we hit the work item test limits / command line length limits. + // We add types from assemblies one by one until we hit our execution time limit, + // at which point we create a new work item with the current types and start a new one. foreach (var (assembly, types) in typeInfos) { - Logger.Log($"Building commands for {assembly.AssemblyName} with {types.Length} types and {types.Sum(type => type.TestCount)} tests"); - foreach (var type in types) + // See if we can just add all types from this assembly to the current work item without going over our execution time limit. + var executionTimeForAllTypesInAssembly = TimeSpan.FromMilliseconds(types.Sum(t => t.ExecutionTime.TotalMilliseconds)); + if (executionTimeForAllTypesInAssembly + currentExecutionTime >= executionTimeLimit) { - if (type.TestCount + currentTestCount >= methodLimit) + // We can't add every type - go type by type to add what we can and end the work item where we need to. + foreach (var type in types) { - // Adding this type would put us over the method limit for this partition. - // Add our accumulated types and assemblies and end the work item - AddWorkItem(currentAssemblies); - currentAssemblies = new Dictionary>(); - } - else if (currentClassNameLengthSum > 25000) - { - // One item we have to consider here is the maximum command line length in - // Windows which is 32767 characters (XP is smaller but don't care). - // Once we get close we start a new work item. - AddWorkItem(currentAssemblies); - currentAssemblies = new Dictionary>(); - } + if (type.ExecutionTime + currentExecutionTime >= executionTimeLimit) + { + // Adding this type would put us over the time limit for this partition. + // Add our accumulated tests and types and assemblies and end the work item. + CreateWorkItemWithCurrentAssemblies(); + } - AddType(currentAssemblies, assembly, type); + // Update the current group in the work item with this new type. This is a partial group since we couldn't add all types in the assembly before. + var typeList = currentAssemblyTests.TryGetValue(assembly, out var result) ? result.Types.Add(type) : ImmutableArray.Create(type); + currentAssemblyTests[assembly] = new AssemblyTypeGroup(assembly, typeList, ContainsAllTypesInAssembly: false); + currentExecutionTime += type.ExecutionTime; + } + } + else + { + // All the types in this assembly can safely be added to the current work item. + // Add them and update our work item execution time with the total execution time of tests in the assembly. + currentAssemblyTests.Add(assembly, new AssemblyTypeGroup(assembly, types, ContainsAllTypesInAssembly: true)); + currentExecutionTime += executionTimeForAllTypesInAssembly; } } // Add any remaining tests to the work item. - AddWorkItem(currentAssemblies); + CreateWorkItemWithCurrentAssemblies(); return workItems.ToImmutableArray(); - void AddWorkItem(Dictionary> typesToTest) + void CreateWorkItemWithCurrentAssemblies() { - if (typesToTest.Any()) + if (currentAssemblyTests.Any()) { - workItems.Add(new WorkItemInfo(typesToTest.ToImmutableSortedDictionary(kvp => kvp.Key, kvp => kvp.Value.ToImmutableArray()), workItemIndex)); + workItems.Add(new WorkItemInfo(currentAssemblyTests.Values.ToImmutableArray(), workItemIndex)); + workItemIndex++; } - currentTestCount = 0; - currentClassNameLengthSum = 0; - workItemIndex++; + currentExecutionTime = TimeSpan.Zero; + currentAssemblyTests = new(); } + } - void AddType(Dictionary> dictionary, AssemblyInfo assemblyInfo, TypeInfo typeInfo) + + private static void LogWorkItems(ImmutableArray workItems) + { + Logger.Log("==== Work Item List ===="); + foreach (var workItem in workItems) { - var list = dictionary.TryGetValue(assemblyInfo, out var result) ? result : new List(); - list.Add(typeInfo); - dictionary[assemblyInfo] = list; + var types = workItem.TypesToTest.SelectMany(group => group.Types); + var totalRuntime = TimeSpan.FromMilliseconds(types.Sum(type => type.ExecutionTime.TotalMilliseconds)); + Logger.Log($"- Work Item ({types.Count()} types, runtime {totalRuntime})"); + foreach (var group in workItem.TypesToTest) + { + var typeExecutionTime = TimeSpan.FromMilliseconds(group.Types.Sum(test => test.ExecutionTime.TotalMilliseconds)); + Logger.Log($" - {group.Assembly.AssemblyName} with {group.Types.Length} types, runtime {typeExecutionTime}"); + } + } + } - currentTestCount += typeInfo.TestCount; - currentClassNameLengthSum += typeInfo.Name.Length; + private static ImmutableArray GetTypeInfoList(AssemblyInfo assemblyInfo) + { + using (var stream = File.OpenRead(assemblyInfo.AssemblyPath)) + using (var peReader = new PEReader(stream)) + { + var metadataReader = peReader.GetMetadataReader(); + return GetTypeInfoList(metadataReader); } } + private static ImmutableArray GetTypeInfoList(MetadataReader reader) + { + var list = new List(); + foreach (var handle in reader.TypeDefinitions) + { + var type = reader.GetTypeDefinition(handle); + if (!IsValidIdentifier(reader, type.Name)) + { + continue; + } - private static void LogWorkItems(ImmutableArray workItems) + var (typeName, fullyQualifiedTypeName) = GetTypeName(reader, type); + var methodCount = GetMethodCount(reader, type); + if (!ShouldIncludeType(reader, type, methodCount)) + { + continue; + } + + list.Add(new TypeInfo(typeName, fullyQualifiedTypeName, methodCount, TimeSpan.Zero)); + } + + // Ensure we get classes back in a deterministic order. + list.Sort((x, y) => x.FullyQualifiedName.CompareTo(y.FullyQualifiedName)); + return list.ToImmutableArray(); + } + + /// + /// Determine if this type should be one of the class values passed to xunit. This + /// code doesn't actually resolve base types or trace through inherrited Fact attributes + /// hence we have to error on the side of including types with no tests vs. excluding them. + /// + private static bool ShouldIncludeType(MetadataReader reader, TypeDefinition type, int testMethodCount) { - Logger.Log("==== Work Item List ===="); - foreach (var workItem in workItems) + // See https://docs.microsoft.com/en-us/dotnet/api/system.reflection.typeattributes?view=net-6.0#examples + // for extracting this information from the TypeAttributes. + var visibility = type.Attributes & TypeAttributes.VisibilityMask; + var isPublic = visibility == TypeAttributes.Public || visibility == TypeAttributes.NestedPublic; + + var classSemantics = type.Attributes & TypeAttributes.ClassSemanticsMask; + var isClass = classSemantics == TypeAttributes.Class; + + var isAbstract = (type.Attributes & TypeAttributes.Abstract) != 0; + + // xunit only handles public, non-abstract classes + if (!isPublic || isAbstract || !isClass) + { + return false; + } + + // Compiler generated types / methods have the shape of the heuristic that we are looking + // at here. Filter them out as well. + if (!IsValidIdentifier(reader, type.Name)) + { + return false; + } + + if (testMethodCount > 0) + { + return true; + } + + // The case we still have to consider at this point is a class with 0 defined methods, + // inheritting from a class with > 0 defined test methods. That is a completely valid + // xunit scenario. For now we're just going to exclude types that inherit from object + // or other built-in base types because they clearly don't fit that category. + return !InheritsFromFrameworkBaseType(reader, type); + } + + private static int GetMethodCount(MetadataReader reader, TypeDefinition type) + { + var count = 0; + foreach (var handle in type.GetMethods()) { - Logger.Log($"- Work Item ({workItem.TypesToTest.Values.SelectMany(w => w).Sum(assembly => assembly.TestCount)} tests)"); - foreach (var assembly in workItem.TypesToTest) + var methodDefinition = reader.GetMethodDefinition(handle); + if (methodDefinition.GetCustomAttributes().Count == 0 || + !IsValidIdentifier(reader, methodDefinition.Name)) { - Logger.Log($" - {assembly.Key.AssemblyName} with {assembly.Value.Sum(type => type.TestCount)} tests"); + continue; } + + count++; + } + + return count; + } + + private static bool IsValidIdentifier(MetadataReader reader, StringHandle handle) + { + var name = reader.GetString(handle); + for (int i = 0; i < name.Length; i++) + { + switch (name[i]) + { + case '<': + case '>': + case '$': + return false; + } + } + + return true; + } + + private static bool InheritsFromFrameworkBaseType(MetadataReader reader, TypeDefinition type) + { + if (type.BaseType.Kind != HandleKind.TypeReference) + { + return false; } + + var typeRef = reader.GetTypeReference((TypeReferenceHandle)type.BaseType); + return + reader.GetString(typeRef.Namespace) == "System" && + reader.GetString(typeRef.Name) is "Object" or "ValueType" or "Enum"; + } + + private static (string Name, string FullyQualifiedName) GetTypeName(MetadataReader reader, TypeDefinition type) + { + var typeName = reader.GetString(type.Name); + + if (TypeAttributes.NestedPublic == (type.Attributes & TypeAttributes.NestedPublic)) + { + // Need to take into account the containing type. + var declaringType = reader.GetTypeDefinition(type.GetDeclaringType()); + var (declaringTypeName, declaringTypeFullName) = GetTypeName(reader, declaringType); + return (typeName, $"{declaringTypeFullName}+{typeName}"); + } + + var namespaceName = reader.GetString(type.Namespace); + if (string.IsNullOrEmpty(namespaceName)) + { + return (typeName, typeName); + } + + return (typeName, $"{namespaceName}.{typeName}"); } } } diff --git a/src/Tools/Source/RunTests/ITestExecutor.cs b/src/Tools/Source/RunTests/ITestExecutor.cs index 3ef1b3ca877d9..1cc9fa526c446 100644 --- a/src/Tools/Source/RunTests/ITestExecutor.cs +++ b/src/Tools/Source/RunTests/ITestExecutor.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Immutable; -using PrepareTests; namespace RunTests { diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index f8ecd4c952436..77fa349ff80db 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -15,7 +15,6 @@ using System.Threading.Tasks; using System.Xml.Linq; using System.Xml.XPath; -using PrepareTests; namespace RunTests { @@ -39,10 +38,10 @@ public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuote var builder = new StringBuilder(); builder.Append($@"test"); - var escapedAssemblyPaths = workItem.TypesToTest.Keys.Select(assembly => $"{sep}{assembly.AssemblyPath}{sep}"); + var escapedAssemblyPaths = workItem.TypesToTest.Select(group => $"{sep}{group.Assembly.AssemblyPath}{sep}"); builder.Append($@" {string.Join(" ", escapedAssemblyPaths)}"); - var typeInfoList = workItem.TypesToTest.Values.SelectMany(type => type).ToImmutableArray(); + var typeInfoList = workItem.TypesToTest.SelectMany(group => group.Types).ToImmutableArray(); if (typeInfoList.Length > 0 || !string.IsNullOrWhiteSpace(Options.TestFilter)) { builder.Append($@" --filter {sep}"); @@ -127,7 +126,7 @@ public async Task RunTestAsync(WorkItemInfo workItemInfo, Options op static bool HasBuiltInRetry(WorkItemInfo workItemInfo) { // vs-extension-testing handles test retry internally. - return workItemInfo.TypesToTest.Keys.Any(assembly => assembly.AssemblyName == "Microsoft.VisualStudio.LanguageServices.New.IntegrationTests.dll"); + return workItemInfo.TypesToTest.Any(group => group.Assembly.AssemblyName == "Microsoft.VisualStudio.LanguageServices.New.IntegrationTests.dll"); } } diff --git a/src/Tools/Source/RunTests/Program.cs b/src/Tools/Source/RunTests/Program.cs index 740c9f2608858..f44ff66caeed2 100644 --- a/src/Tools/Source/RunTests/Program.cs +++ b/src/Tools/Source/RunTests/Program.cs @@ -13,8 +13,9 @@ using System.Collections.Immutable; using System.Diagnostics; using System.Text.RegularExpressions; -using PrepareTests; using System.Xml.Linq; +using System.Diagnostics.Contracts; +using System.Runtime.InteropServices; namespace RunTests { @@ -131,7 +132,7 @@ private static async Task RunAsync(Options options, CancellationToken cance var testExecutor = CreateTestExecutor(options); var testRunner = new TestRunner(options, testExecutor); var start = DateTime.Now; - var workItems = GetWorkItems(options); + var workItems = await GetWorkItemsAsync(options, cancellationToken); if (workItems.Length == 0) { WriteLogFile(options); @@ -282,11 +283,11 @@ async Task DumpProcess(Process targetProcess, string procDumpExeFilePath, string return null; } - private static ImmutableArray GetWorkItems(Options options) + private static async Task> GetWorkItemsAsync(Options options, CancellationToken cancellationToken) { var scheduler = new AssemblyScheduler(options); var assemblyPaths = GetAssemblyFilePaths(options); - var workItems = scheduler.Schedule(assemblyPaths); + var workItems = await scheduler.ScheduleAsync(assemblyPaths, cancellationToken); return workItems; } @@ -295,46 +296,69 @@ private static ImmutableArray GetAssemblyFilePaths(Options options var list = new List(); var binDirectory = Path.Combine(options.ArtifactsDirectory, "bin"); - var assemblies = ListTests.GetTestAssemblyFilePaths(binDirectory, ShouldSkip, options.Configuration, options.TargetFrameworks); - - return assemblies; - - bool ShouldSkip(string projectDirectory) + // Find all the project folders that fit our naming scheme for unit tests. + foreach (var project in Directory.EnumerateDirectories(binDirectory, "*.UnitTests", SearchOption.TopDirectoryOnly)) { - var name = Path.GetFileName(projectDirectory); + var name = Path.GetFileName(project); if (!shouldInclude(name, options) || shouldExclude(name, options)) { - Logger.Log($"Skipped {name} as it was not included"); - return true; + continue; } - return false; - - static bool shouldInclude(string name, Options options) + var fileName = $"{name}.dll"; + // Find the dlls matching the request configuration and target frameworks. + foreach (var targetFramework in options.TargetFrameworks) { - foreach (var pattern in options.IncludeFilter) + var targetFrameworkDirectory = Path.Combine(project, options.Configuration, targetFramework); + var filePath = Path.Combine(targetFrameworkDirectory, fileName); + if (File.Exists(filePath)) + { + list.Add(new AssemblyInfo(filePath)); + } + else if (Directory.Exists(targetFrameworkDirectory) && Directory.GetFiles(targetFrameworkDirectory, searchPattern: "*.UnitTests.dll") is { Length: > 0 } matches) { - if (Regex.IsMatch(name, pattern.Trim('\'', '"'))) + // If the unit test assembly name doesn't match the project folder name, but still matches our "unit test" name pattern, we want to run it. + // If more than one such assembly is present in a project output folder, we assume something is wrong with the build configuration. + // For example, one unit test project might be referencing another unit test project. + if (matches.Length > 1) { - return true; + var message = $"Multiple unit test assemblies found in '{targetFrameworkDirectory}'. Please adjust the build to prevent this. Matches:{Environment.NewLine}{string.Join(Environment.NewLine, matches)}"; + throw new Exception(message); } + list.Add(new AssemblyInfo(matches[0])); } - - return false; } + } + + Contract.Assert(list.Count > 0, $"Did not find any test assemblies"); - static bool shouldExclude(string name, Options options) + list.Sort(); + return list.ToImmutableArray(); + + static bool shouldInclude(string name, Options options) + { + foreach (var pattern in options.IncludeFilter) { - foreach (var pattern in options.ExcludeFilter) + if (Regex.IsMatch(name, pattern.Trim('\'', '"'))) { - if (Regex.IsMatch(name, pattern.Trim('\'', '"'))) - { - return true; - } + return true; } + } + + return false; + } - return false; + static bool shouldExclude(string name, Options options) + { + foreach (var pattern in options.ExcludeFilter) + { + if (Regex.IsMatch(name, pattern.Trim('\'', '"'))) + { + return true; + } } + + return false; } } diff --git a/src/Tools/Source/RunTests/RunTests.csproj b/src/Tools/Source/RunTests/RunTests.csproj index 0b6a995b667a0..aefa3a8aa7d11 100644 --- a/src/Tools/Source/RunTests/RunTests.csproj +++ b/src/Tools/Source/RunTests/RunTests.csproj @@ -16,8 +16,6 @@ - - - + \ No newline at end of file diff --git a/src/Tools/Source/RunTests/TestRunner.cs b/src/Tools/Source/RunTests/TestRunner.cs index 3a36910b04754..7378e4a88ecb9 100644 --- a/src/Tools/Source/RunTests/TestRunner.cs +++ b/src/Tools/Source/RunTests/TestRunner.cs @@ -16,7 +16,6 @@ using System.Threading.Tasks; using Mono.Options; using Newtonsoft.Json; -using PrepareTests; namespace RunTests { @@ -153,9 +152,9 @@ static string getGlobalJsonPath() static void AddRehydrateTestFoldersCommand(StringBuilder commandBuilder, WorkItemInfo workItem, bool isUnix) { // Rehydrate assemblies that we need to run as part of this work item. - foreach (var testAssembly in workItem.TypesToTest.Keys) + foreach (var testAssemblyGroup in workItem.TypesToTest) { - var directoryName = Path.GetDirectoryName(testAssembly.AssemblyPath); + var directoryName = Path.GetDirectoryName(testAssemblyGroup.Assembly.AssemblyPath); if (isUnix) { // If we're on unix make sure we have permissions to run the rehydrate script. @@ -204,12 +203,10 @@ string MakeHelixWorkItemProject(WorkItemInfo workItemInfo) // Create a payload directory that contains all the assemblies in the work item in separate folders. var payloadDirectory = Path.Combine(msbuildTestPayloadRoot, "artifacts", "bin"); - // Get the assembly path in the context of the helix work item. + // Update the assembly groups to test with the assembly paths in the context of the helix work item. workItemInfo = workItemInfo with { - TypesToTest = workItemInfo.TypesToTest.ToImmutableSortedDictionary( - kvp => kvp.Key with { AssemblyPath = GetHelixRelativeAssemblyPath(kvp.Key.AssemblyPath) }, - kvp => kvp.Value) + TypesToTest = workItemInfo.TypesToTest.Select(group => group with { Assembly = group.Assembly with { AssemblyPath = GetHelixRelativeAssemblyPath(group.Assembly.AssemblyPath) } }).ToImmutableArray() }; AddRehydrateTestFoldersCommand(command, workItemInfo, isUnix); From b42866198cb0eaafecde17fbefeee38469e6413d Mon Sep 17 00:00:00 2001 From: David Barbet Date: Thu, 21 Jul 2022 14:10:43 -0700 Subject: [PATCH 04/26] working --- eng/prepare-tests.ps1 | 3 +- eng/prepare-tests.sh | 2 +- src/Tools/Source/RunTests/AssemblyInfo.cs | 37 +++ .../Source/RunTests/AssemblyScheduler.cs | 293 +++++++++++++----- src/Tools/Source/RunTests/ProcessRunner.cs | 202 ++++++++++++ .../Source/RunTests/ProcessTestExecutor.cs | 17 +- .../Source/RunTests/TestHistoryManager.cs | 127 ++++++++ src/Tools/Source/RunTests/TestRunner.cs | 11 +- 8 files changed, 593 insertions(+), 99 deletions(-) create mode 100644 src/Tools/Source/RunTests/AssemblyInfo.cs create mode 100644 src/Tools/Source/RunTests/ProcessRunner.cs create mode 100644 src/Tools/Source/RunTests/TestHistoryManager.cs diff --git a/eng/prepare-tests.ps1 b/eng/prepare-tests.ps1 index 427f9ba35d0dd..cfdcee17742ef 100644 --- a/eng/prepare-tests.ps1 +++ b/eng/prepare-tests.ps1 @@ -11,8 +11,7 @@ try { $dotnet = Ensure-DotnetSdk # permissions issues make this a pain to do in PrepareTests itself. Remove-Item -Recurse -Force "$RepoRoot\artifacts\testPayload" -ErrorAction SilentlyContinue - $cmdOutput = Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload" - echo $cmdOutput + Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload" exit 0 } catch { diff --git a/eng/prepare-tests.sh b/eng/prepare-tests.sh index 5b0310573bd1f..0e9144522d020 100755 --- a/eng/prepare-tests.sh +++ b/eng/prepare-tests.sh @@ -28,4 +28,4 @@ InitializeDotNetCli true # permissions issues make this a pain to do in PrepareTests itself. rm -rf "$repo_root/artifacts/testPayload" -dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix +dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix \ No newline at end of file diff --git a/src/Tools/Source/RunTests/AssemblyInfo.cs b/src/Tools/Source/RunTests/AssemblyInfo.cs new file mode 100644 index 0000000000000..8c2cc36e0daa2 --- /dev/null +++ b/src/Tools/Source/RunTests/AssemblyInfo.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Immutable; +using System.IO; + +namespace RunTests; + +public readonly record struct AssemblyInfo(string AssemblyPath) : IComparable +{ + public string AssemblyName => Path.GetFileName(AssemblyPath); + + public int CompareTo(object? obj) + { + if (obj == null) + { + return 1; + } + + var otherAssembly = (AssemblyInfo)obj; + + // Ensure we have a consistent ordering by ordering by assembly path. + return this.AssemblyPath.CompareTo(otherAssembly.AssemblyPath); + } +} + +public readonly record struct TypeInfo(string Name, string FullyQualifiedName, ImmutableArray Tests) +{ + public override string ToString() => $"[Type]{FullyQualifiedName}"; +} + +public readonly record struct MethodInfo(string Name, string FullyQualifiedName, TimeSpan ExecutionTime) +{ + public override string ToString() => $"[Method]{FullyQualifiedName}"; +} diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index 462ebd478a65f..f7d81eee64643 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -18,28 +18,60 @@ namespace RunTests { + internal record struct WorkItemInfo(ImmutableSortedDictionary> Filters, int PartitionIndex) + { + internal string DisplayName => $"{string.Join("_", Filters.Keys.Select(a => Path.GetFileNameWithoutExtension(a.AssemblyName)))}_{PartitionIndex}"; + } + + internal interface ITestFilter + { + internal string GetFilterString(); + + internal TimeSpan GetExecutionTime(); + } - /// - /// Defines a single work item to run. The work item may contain tests from multiple assemblies - /// that should be run together in the same work item. - /// - internal readonly record struct WorkItemInfo(ImmutableArray TypesToTest, int PartitionIndex) + internal record struct AssemblyTestFilter(ImmutableArray TypesInAssembly) : ITestFilter { - internal string DisplayName + TimeSpan ITestFilter.GetExecutionTime() { - get - { - var assembliesString = string.Join("_", TypesToTest.Select(group => Path.GetFileNameWithoutExtension(group.Assembly.AssemblyName).Replace(".", string.Empty))); - return $"{assembliesString}_{PartitionIndex}"; - } + return TimeSpan.FromMilliseconds(TypesInAssembly.SelectMany(type => type.Tests).Sum(test => test.ExecutionTime.TotalMilliseconds)); } - internal static WorkItemInfo CreateFullAssembly(AssemblyInfo assembly, int partitionIndex) - => new(ImmutableArray.Empty.Add(new(assembly, ImmutableArray.Empty, true)), partitionIndex); + string ITestFilter.GetFilterString() + { + return string.Empty; + } } - internal readonly record struct AssemblyTypeGroup(AssemblyInfo Assembly, ImmutableArray Types, bool ContainsAllTypesInAssembly); + internal record struct TypeTestFilter(TypeInfo Type) : ITestFilter + { + TimeSpan ITestFilter.GetExecutionTime() + { + return TimeSpan.FromMilliseconds(Type.Tests.Sum(test => test.ExecutionTime.TotalMilliseconds)); + } + + string ITestFilter.GetFilterString() + { + // https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests?pivots=mstest#syntax + // We want to avoid matching other test classes whose names are prefixed with this test class's name. + // For example, avoid running 'AttributeTests_WellKnownMember', when the request here is to run 'AttributeTests'. + // We append a '.', assuming that all test methods in the class *will* match it, but not methods in other classes. + return $"{Type.Name}."; + } + } + internal record struct MethodTestFilter(MethodInfo Test) : ITestFilter + { + TimeSpan ITestFilter.GetExecutionTime() + { + return Test.ExecutionTime; + } + + string ITestFilter.GetFilterString() + { + return Test.FullyQualifiedName; + } + } internal sealed class AssemblyScheduler { @@ -53,26 +85,24 @@ internal AssemblyScheduler(Options options) public async Task> ScheduleAsync(ImmutableArray assemblies, CancellationToken cancellationToken) { Logger.Log($"Scheduling {assemblies.Length} assemblies"); + + var orderedTypeInfos = assemblies.ToImmutableSortedDictionary(assembly => assembly, GetTypeInfoList); + + ConsoleUtil.WriteLine($"Found {orderedTypeInfos.Values.SelectMany(t => t).SelectMany(t => t.Tests).Count()} tests to run in {orderedTypeInfos.Keys.Count()} assemblies"); + if (_options.Sequential) { Logger.Log("Building sequential work items"); // return individual work items per assembly that contain all the tests in that assembly. - return assemblies - .Select(WorkItemInfo.CreateFullAssembly) - .ToImmutableArray(); + return CreateWorkItemsForFullAssemblies(orderedTypeInfos); } - var orderedTypeInfos = assemblies.ToImmutableSortedDictionary(assembly => assembly, GetTypeInfoList); - ConsoleUtil.WriteLine($"Found {orderedTypeInfos.Values.SelectMany(t => t).Count()} test types to run in {orderedTypeInfos.Keys.Count()} assemblies"); - // Retrieve test runtimes from azure devops historical data. - var testHistory = await TestHistoryManager.GetTestHistoryPerTypeAsync(cancellationToken); + var testHistory = await TestHistoryManager.GetTestHistoryAsync(cancellationToken); if (testHistory.IsEmpty) { // We didn't have any test history from azure devops, just partition by assembly. - return assemblies - .Select(WorkItemInfo.CreateFullAssembly) - .ToImmutableArray(); + return CreateWorkItemsForFullAssemblies(orderedTypeInfos); } // Now for our current set of test methods we got from the assemblies we built, match them to tests from our test run history @@ -88,9 +118,22 @@ public async Task> ScheduleAsync(ImmutableArray CreateWorkItemsForFullAssemblies(ImmutableSortedDictionary> orderedTypeInfos) + { + var workItems = new List(); + var partitionIndex = 0; + foreach (var orderedTypeInfo in orderedTypeInfos) + { + var currentWorkItem = ImmutableSortedDictionary>.Empty.Add(orderedTypeInfo.Key, ImmutableArray.Create((ITestFilter)new AssemblyTestFilter(orderedTypeInfo.Value))); + workItems.Add(new WorkItemInfo(currentWorkItem, partitionIndex++)); + } + + return workItems.ToImmutableArray(); + } } private static ImmutableSortedDictionary> UpdateTestsWithExecutionTimes( @@ -100,48 +143,51 @@ private static ImmutableSortedDictionary> // Determine the average execution time so that we can use it for tests that do not have any history. var averageExecutionTime = TimeSpan.FromMilliseconds(testHistory.Values.Average(t => t.TotalMilliseconds)); - // Store the types from our assemblies that we couldn't find a history for to log. - var extraLocalTypes = new HashSet(); + // Store the tests from our assemblies that we couldn't find a history for to log. + var extraLocalTests = new HashSet(); - // Store the types that we were able to match to historical data. - var matchedLocalTypes = new HashSet(); + // Store the tests that we were able to match to historical data. + var matchedLocalTests = new HashSet(); - var updated = assemblyTypes.ToImmutableSortedDictionary(kvp => kvp.Key, kvp => kvp.Value.Select(typeInfo => WithExecutionTime(typeInfo)).ToImmutableArray()); - LogResults(matchedLocalTypes, extraLocalTypes); + var updated = assemblyTypes.ToImmutableSortedDictionary( + kvp => kvp.Key, + kvp => kvp.Value.Select(typeInfo => typeInfo with { Tests = typeInfo.Tests.Select(method => WithExecutionTime(method)).ToImmutableArray() }) + .ToImmutableArray()); + LogResults(matchedLocalTests, extraLocalTests); return updated; - TypeInfo WithExecutionTime(TypeInfo typeInfo) + MethodInfo WithExecutionTime(MethodInfo methodInfo) { // Match by fully qualified test method name to azure devops historical data. // Note for combinatorial tests, azure devops helpfully groups all sub-runs under a top level method (with combined test run times) with the same fully qualified method name // that we'll get from looking at all the test methods in the assembly with SRM. - if (testHistory.TryGetValue(typeInfo.FullyQualifiedName, out var executionTime)) + if (testHistory.TryGetValue(methodInfo.FullyQualifiedName, out var executionTime)) { - matchedLocalTypes.Add(typeInfo.FullyQualifiedName); - return typeInfo with { ExecutionTime = executionTime }; + matchedLocalTests.Add(methodInfo.FullyQualifiedName); + return methodInfo with { ExecutionTime = executionTime }; } // We didn't find the local type from our assembly in test run historical data. // This can happen if our SRM heuristic incorrectly counted a normal method as a test method (which it can do often). - extraLocalTypes.Add(typeInfo.FullyQualifiedName); - return typeInfo with { ExecutionTime = averageExecutionTime }; + extraLocalTests.Add(methodInfo.FullyQualifiedName); + return methodInfo with { ExecutionTime = averageExecutionTime }; } - void LogResults(HashSet matchedLocalTypes, HashSet extraLocalTypes) + void LogResults(HashSet matchedLocalTests, HashSet extraLocalTests) { - foreach (var extraLocalType in extraLocalTypes) + foreach (var extraLocalTest in extraLocalTests) { - Logger.Log($"Could not find test execution history for types in {extraLocalType}"); + Logger.Log($"Could not find test execution history for test {extraLocalTest}"); } - var extraRemoteTypes = testHistory.Keys.Where(type => !matchedLocalTypes.Contains(type)); - foreach (var extraRemoteType in extraRemoteTypes) + var extraRemoteTests = testHistory.Keys.Where(type => !matchedLocalTests.Contains(type)); + foreach (var extraRemoteTest in extraRemoteTests) { - Logger.Log($"Found historical data for types in {extraRemoteType} that were not present in local assemblies"); + Logger.Log($"Found historical data for test {extraRemoteTest} that was not present in local assemblies"); } - var totalExpectedRunTime = TimeSpan.FromMilliseconds(updated.Values.SelectMany(types => types).Sum(test => test.ExecutionTime.TotalMilliseconds)); - ConsoleUtil.WriteLine($"Matched {matchedLocalTypes.Count} types with historical data. {extraLocalTypes.Count} types were missing historical data. {extraRemoteTypes.Count()} types were missing in local assemblies. Estimate of total execution time for tests is {totalExpectedRunTime}."); + var totalExpectedRunTime = TimeSpan.FromMilliseconds(updated.Values.SelectMany(types => types).SelectMany(type => type.Tests).Sum(test => test.ExecutionTime.TotalMilliseconds)); + ConsoleUtil.WriteLine($"Matched {matchedLocalTests.Count} tests with historical data. {extraLocalTests.Count} tests were missing historical data. {extraRemoteTests.Count()} tests were missing in local assemblies. Estimate of total execution time for tests is {totalExpectedRunTime}."); } } @@ -156,7 +202,7 @@ private static ImmutableArray BuildWorkItems(ImmutableSortedDictio var currentExecutionTime = TimeSpan.Zero; // Keep track of the types we're planning to add to the current work item. - var currentAssemblyTests = new Dictionary(); + var currentFilters = new SortedDictionary>(); // Iterate through each assembly and type and build up the work items to run. // We add types from assemblies one by one until we hit our execution time limit, @@ -164,48 +210,77 @@ private static ImmutableArray BuildWorkItems(ImmutableSortedDictio foreach (var (assembly, types) in typeInfos) { // See if we can just add all types from this assembly to the current work item without going over our execution time limit. - var executionTimeForAllTypesInAssembly = TimeSpan.FromMilliseconds(types.Sum(t => t.ExecutionTime.TotalMilliseconds)); + var executionTimeForAllTypesInAssembly = TimeSpan.FromMilliseconds(types.SelectMany(type => type.Tests).Sum(t => t.ExecutionTime.TotalMilliseconds)); if (executionTimeForAllTypesInAssembly + currentExecutionTime >= executionTimeLimit) { // We can't add every type - go type by type to add what we can and end the work item where we need to. foreach (var type in types) { - if (type.ExecutionTime + currentExecutionTime >= executionTimeLimit) + // See if we can add every test in this type to the current work item without going over our execution time limit. + var executionTimeForAllTestsInType = TimeSpan.FromMilliseconds(type.Tests.Sum(method => method.ExecutionTime.TotalMilliseconds)); + if (executionTimeForAllTestsInType + currentExecutionTime >= executionTimeLimit) { - // Adding this type would put us over the time limit for this partition. - // Add our accumulated tests and types and assemblies and end the work item. - CreateWorkItemWithCurrentAssemblies(); + // We can't add every test, go test by test to add what we can and end the work item when we hit the limit. + foreach (var test in type.Tests) + { + if (test.ExecutionTime + currentExecutionTime >= executionTimeLimit) + { + // Adding this type would put us over the time limit for this partition. + // Add the current work item to our list and start a new one. + AddCurrentWorkItem(); + } + + // Update the current group in the work item with this new type. + AddFilter(assembly, new MethodTestFilter(test)); + currentExecutionTime += test.ExecutionTime; + } + } + else + { + // All the tests in this type can be safely added to the current work item. + // Add them and update our work item execution time with the total execution time of tests in the type. + AddFilter(assembly, new TypeTestFilter(type)); + currentExecutionTime += executionTimeForAllTestsInType; } - - // Update the current group in the work item with this new type. This is a partial group since we couldn't add all types in the assembly before. - var typeList = currentAssemblyTests.TryGetValue(assembly, out var result) ? result.Types.Add(type) : ImmutableArray.Create(type); - currentAssemblyTests[assembly] = new AssemblyTypeGroup(assembly, typeList, ContainsAllTypesInAssembly: false); - currentExecutionTime += type.ExecutionTime; } } else { // All the types in this assembly can safely be added to the current work item. // Add them and update our work item execution time with the total execution time of tests in the assembly. - currentAssemblyTests.Add(assembly, new AssemblyTypeGroup(assembly, types, ContainsAllTypesInAssembly: true)); + AddFilter(assembly, new AssemblyTestFilter(types)); currentExecutionTime += executionTimeForAllTypesInAssembly; } } // Add any remaining tests to the work item. - CreateWorkItemWithCurrentAssemblies(); + AddCurrentWorkItem(); return workItems.ToImmutableArray(); - void CreateWorkItemWithCurrentAssemblies() + void AddCurrentWorkItem() { - if (currentAssemblyTests.Any()) + if (currentFilters.Any()) { - workItems.Add(new WorkItemInfo(currentAssemblyTests.Values.ToImmutableArray(), workItemIndex)); + workItems.Add(new WorkItemInfo(currentFilters.ToImmutableSortedDictionary(kvp => kvp.Key, kvp => kvp.Value.ToImmutableArray()), workItemIndex)); workItemIndex++; } + currentFilters = new(); currentExecutionTime = TimeSpan.Zero; - currentAssemblyTests = new(); + } + + void AddFilter(AssemblyInfo assembly, ITestFilter filter) + { + if (currentFilters.TryGetValue(assembly, out var assemblyFilters)) + { + assemblyFilters.Add(filter); + } + else + { + var filterList = new List(); + filterList.Add(filter); + currentFilters.Add(assembly, filterList); + } } } @@ -215,13 +290,16 @@ private static void LogWorkItems(ImmutableArray workItems) Logger.Log("==== Work Item List ===="); foreach (var workItem in workItems) { - var types = workItem.TypesToTest.SelectMany(group => group.Types); - var totalRuntime = TimeSpan.FromMilliseconds(types.Sum(type => type.ExecutionTime.TotalMilliseconds)); - Logger.Log($"- Work Item ({types.Count()} types, runtime {totalRuntime})"); - foreach (var group in workItem.TypesToTest) + var totalRuntime = TimeSpan.FromMilliseconds(workItem.Filters.Values.SelectMany(f => f).Sum(f => f.GetExecutionTime().TotalMilliseconds)); + Logger.Log($"- Work Item (Runtime {totalRuntime})"); + foreach (var assembly in workItem.Filters) { - var typeExecutionTime = TimeSpan.FromMilliseconds(group.Types.Sum(test => test.ExecutionTime.TotalMilliseconds)); - Logger.Log($" - {group.Assembly.AssemblyName} with {group.Types.Length} types, runtime {typeExecutionTime}"); + var assemblyRuntime = TimeSpan.FromMilliseconds(assembly.Value.Sum(f => f.GetExecutionTime().TotalMilliseconds)); + Logger.Log($" - {assembly.Key.AssemblyName} with runtime {assemblyRuntime}"); + foreach (var filter in assembly.Value) + { + Logger.Log($" - {filter} with runtime {filter.GetExecutionTime()}"); + } } } } @@ -254,7 +332,10 @@ private static ImmutableArray GetTypeInfoList(MetadataReader reader) continue; } - list.Add(new TypeInfo(typeName, fullyQualifiedTypeName, methodCount, TimeSpan.Zero)); + var methodList = new List(); + GetMethods(reader, type, methodList, fullyQualifiedTypeName); + + list.Add(new TypeInfo(typeName, fullyQualifiedTypeName, methodList.ToImmutableArray())); } // Ensure we get classes back in a deterministic order. @@ -262,25 +343,37 @@ private static ImmutableArray GetTypeInfoList(MetadataReader reader) return list.ToImmutableArray(); } - /// - /// Determine if this type should be one of the class values passed to xunit. This - /// code doesn't actually resolve base types or trace through inherrited Fact attributes - /// hence we have to error on the side of including types with no tests vs. excluding them. - /// - private static bool ShouldIncludeType(MetadataReader reader, TypeDefinition type, int testMethodCount) + private static bool IsPublicType(TypeDefinition type) { // See https://docs.microsoft.com/en-us/dotnet/api/system.reflection.typeattributes?view=net-6.0#examples // for extracting this information from the TypeAttributes. var visibility = type.Attributes & TypeAttributes.VisibilityMask; var isPublic = visibility == TypeAttributes.Public || visibility == TypeAttributes.NestedPublic; + return isPublic; + } + private static bool IsClass(TypeDefinition type) + { var classSemantics = type.Attributes & TypeAttributes.ClassSemanticsMask; var isClass = classSemantics == TypeAttributes.Class; + return isClass; + } + private static bool IsAbstract(TypeDefinition type) + { var isAbstract = (type.Attributes & TypeAttributes.Abstract) != 0; + return isAbstract; + } + /// + /// Determine if this type should be one of the class values passed to xunit. This + /// code doesn't actually resolve base types or trace through inherrited Fact attributes + /// hence we have to error on the side of including types with no tests vs. excluding them. + /// + private static bool ShouldIncludeType(MetadataReader reader, TypeDefinition type, int testMethodCount) + { // xunit only handles public, non-abstract classes - if (!isPublic || isAbstract || !isClass) + if (!IsPublicType(type) || IsAbstract(type) || !IsClass(type)) { return false; } @@ -304,14 +397,58 @@ private static bool ShouldIncludeType(MetadataReader reader, TypeDefinition type return !InheritsFromFrameworkBaseType(reader, type); } + private static void GetMethods(MetadataReader reader, TypeDefinition type, List methodList, string originalFullyQualifiedTypeName) + { + var methods = type.GetMethods(); + + foreach (var methodHandle in methods) + { + var method = reader.GetMethodDefinition(methodHandle); + var methodInfo = GetMethodInfo(reader, method, originalFullyQualifiedTypeName); + if (ShouldIncludeMethod(reader, method)) + { + methodList.Add(methodInfo); + } + } + + var baseTypeHandle = type.BaseType; + if (!baseTypeHandle.IsNil && baseTypeHandle.Kind is HandleKind.TypeDefinition) + { + var baseType = reader.GetTypeDefinition((TypeDefinitionHandle)baseTypeHandle); + + // We only want to look for test methods in public base types. + if (IsPublicType(baseType) && IsClass(baseType) && !InheritsFromFrameworkBaseType(reader, type)) + { + GetMethods(reader, baseType, methodList, originalFullyQualifiedTypeName); + } + } + } + + private static bool ShouldIncludeMethod(MetadataReader reader, MethodDefinition method) + { + var visibility = method.Attributes & MethodAttributes.MemberAccessMask; + var isPublic = visibility == MethodAttributes.Public; + + var hasMethodAttributes = method.GetCustomAttributes().Count > 0; + + var isValidIdentifier = IsValidIdentifier(reader, method.Name); + + return isPublic && hasMethodAttributes && isValidIdentifier; + } + + private static MethodInfo GetMethodInfo(MetadataReader reader, MethodDefinition method, string fullyQualifiedTypeName) + { + var methodName = reader.GetString(method.Name); + return new MethodInfo(methodName, $"{fullyQualifiedTypeName}.{methodName}", TimeSpan.Zero); + } + private static int GetMethodCount(MetadataReader reader, TypeDefinition type) { var count = 0; foreach (var handle in type.GetMethods()) { var methodDefinition = reader.GetMethodDefinition(handle); - if (methodDefinition.GetCustomAttributes().Count == 0 || - !IsValidIdentifier(reader, methodDefinition.Name)) + if (!ShouldIncludeMethod(reader, methodDefinition)) { continue; } diff --git a/src/Tools/Source/RunTests/ProcessRunner.cs b/src/Tools/Source/RunTests/ProcessRunner.cs new file mode 100644 index 0000000000000..e110c4ebc7e11 --- /dev/null +++ b/src/Tools/Source/RunTests/ProcessRunner.cs @@ -0,0 +1,202 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace RunTests; + +public readonly struct ProcessResult +{ + public Process Process { get; } + public int ExitCode { get; } + public ReadOnlyCollection OutputLines { get; } + public ReadOnlyCollection ErrorLines { get; } + + public ProcessResult(Process process, int exitCode, ReadOnlyCollection outputLines, ReadOnlyCollection errorLines) + { + Process = process; + ExitCode = exitCode; + OutputLines = outputLines; + ErrorLines = errorLines; + } +} + +public readonly struct ProcessInfo +{ + public Process Process { get; } + public ProcessStartInfo StartInfo { get; } + public Task Result { get; } + + public int Id => Process.Id; + + public ProcessInfo(Process process, ProcessStartInfo startInfo, Task result) + { + Process = process; + StartInfo = startInfo; + Result = result; + } +} + +public static class ProcessRunner +{ + public static void OpenFile(string file) + { + if (File.Exists(file)) + { + Process.Start(file); + } + } + + public static ProcessInfo CreateProcess( + string executable, + string arguments, + bool lowPriority = false, + string? workingDirectory = null, + bool captureOutput = false, + bool displayWindow = true, + Dictionary? environmentVariables = null, + Action? onProcessStartHandler = null, + Action? onOutputDataReceived = null, + CancellationToken cancellationToken = default) => + CreateProcess( + CreateProcessStartInfo(executable, arguments, workingDirectory, captureOutput, displayWindow, environmentVariables), + lowPriority: lowPriority, + onProcessStartHandler: onProcessStartHandler, + onOutputDataReceived: onOutputDataReceived, + cancellationToken: cancellationToken); + + public static ProcessInfo CreateProcess( + ProcessStartInfo processStartInfo, + bool lowPriority = false, + Action? onProcessStartHandler = null, + Action? onOutputDataReceived = null, + CancellationToken cancellationToken = default) + { + var errorLines = new List(); + var outputLines = new List(); + var process = new Process(); + var tcs = new TaskCompletionSource(); + + process.EnableRaisingEvents = true; + process.StartInfo = processStartInfo; + + process.OutputDataReceived += (s, e) => + { + if (e.Data != null) + { + onOutputDataReceived?.Invoke(e); + outputLines.Add(e.Data); + } + }; + + process.ErrorDataReceived += (s, e) => + { + if (e.Data != null) + { + errorLines.Add(e.Data); + } + }; + + process.Exited += (s, e) => + { + // We must call WaitForExit to make sure we've received all OutputDataReceived/ErrorDataReceived calls + // or else we'll be returning a list we're still modifying. For paranoia, we'll start a task here rather + // than enter right back into the Process type and start a wait which isn't guaranteed to be safe. + Task.Run(() => + { + process.WaitForExit(); + var result = new ProcessResult( + process, + process.ExitCode, + new ReadOnlyCollection(outputLines), + new ReadOnlyCollection(errorLines)); + tcs.TrySetResult(result); + }, cancellationToken); + }; + + var registration = cancellationToken.Register(() => + { + if (tcs.TrySetCanceled()) + { + // If the underlying process is still running, we should kill it + if (!process.HasExited) + { + try + { + process.Kill(); + } + catch (InvalidOperationException) + { + // Ignore, since the process is already dead + } + } + } + }); + + process.Start(); + onProcessStartHandler?.Invoke(process); + + if (lowPriority) + { + process.PriorityClass = ProcessPriorityClass.BelowNormal; + } + + if (processStartInfo.RedirectStandardOutput) + { + process.BeginOutputReadLine(); + } + + if (processStartInfo.RedirectStandardError) + { + process.BeginErrorReadLine(); + } + + return new ProcessInfo(process, processStartInfo, tcs.Task); + } + + public static ProcessStartInfo CreateProcessStartInfo( + string executable, + string arguments, + string? workingDirectory = null, + bool captureOutput = false, + bool displayWindow = true, + Dictionary? environmentVariables = null) + { + var processStartInfo = new ProcessStartInfo(executable, arguments); + + if (!string.IsNullOrEmpty(workingDirectory)) + { + processStartInfo.WorkingDirectory = workingDirectory; + } + + if (environmentVariables != null) + { + foreach (var pair in environmentVariables) + { + processStartInfo.EnvironmentVariables[pair.Key] = pair.Value; + } + } + + if (captureOutput) + { + processStartInfo.UseShellExecute = false; + processStartInfo.RedirectStandardOutput = true; + processStartInfo.RedirectStandardError = true; + } + else + { + processStartInfo.CreateNoWindow = !displayWindow; + processStartInfo.UseShellExecute = displayWindow; + } + + return processStartInfo; + } +} diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index 77fa349ff80db..674b40e67d4b4 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -38,23 +38,18 @@ public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuote var builder = new StringBuilder(); builder.Append($@"test"); - var escapedAssemblyPaths = workItem.TypesToTest.Select(group => $"{sep}{group.Assembly.AssemblyPath}{sep}"); + var escapedAssemblyPaths = workItem.Filters.Keys.Select(assembly => $"{sep}{assembly.AssemblyPath}{sep}"); builder.Append($@" {string.Join(" ", escapedAssemblyPaths)}"); - var typeInfoList = workItem.TypesToTest.SelectMany(group => group.Types).ToImmutableArray(); - if (typeInfoList.Length > 0 || !string.IsNullOrWhiteSpace(Options.TestFilter)) + var filters = workItem.Filters.Values.SelectMany(filter => filter).Where(filter => !string.IsNullOrEmpty(filter.GetFilterString())).ToImmutableArray(); + if (filters.Length > 0 || !string.IsNullOrWhiteSpace(Options.TestFilter)) { builder.Append($@" --filter {sep}"); var any = false; - foreach (var typeInfo in typeInfoList) + foreach (var filter in filters) { MaybeAddSeparator(); - // https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests?pivots=mstest#syntax - // We want to avoid matching other test classes whose names are prefixed with this test class's name. - // For example, avoid running 'AttributeTests_WellKnownMember', when the request here is to run 'AttributeTests'. - // We append a '.', assuming that all test methods in the class *will* match it, but not methods in other classes. - builder.Append(typeInfo.Name); - builder.Append('.'); + builder.Append(filter.GetFilterString()); } builder.Append(sep); @@ -126,7 +121,7 @@ public async Task RunTestAsync(WorkItemInfo workItemInfo, Options op static bool HasBuiltInRetry(WorkItemInfo workItemInfo) { // vs-extension-testing handles test retry internally. - return workItemInfo.TypesToTest.Any(group => group.Assembly.AssemblyName == "Microsoft.VisualStudio.LanguageServices.New.IntegrationTests.dll"); + return workItemInfo.Filters.Keys.Any(key => key.AssemblyName == "Microsoft.VisualStudio.LanguageServices.New.IntegrationTests.dll"); } } diff --git a/src/Tools/Source/RunTests/TestHistoryManager.cs b/src/Tools/Source/RunTests/TestHistoryManager.cs new file mode 100644 index 0000000000000..263c5164abdba --- /dev/null +++ b/src/Tools/Source/RunTests/TestHistoryManager.cs @@ -0,0 +1,127 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.TeamFoundation.Build.WebApi; +using Microsoft.TeamFoundation.TestManagement.WebApi; +using Microsoft.VisualStudio.Services.TestResults.WebApi; +using Microsoft.VisualStudio.Services.WebApi; + +namespace RunTests; +internal class TestHistoryManager +{ + /// + /// Azure devops limits the number of tests returned per request to 1000. + /// + private const int MaxTestsReturnedPerRequest = 10000; + + /// + /// Todo - build definition comes from System.DefinitionId + /// Todo - stageName comes from System.StageName to identify which test runs to get from the build. + /// + public static async Task> GetTestHistoryAsync(/*int buildDefinitionId, string stageName, string branchName*/CancellationToken cancellationToken) + { + Debugger.Launch(); + var pat = Environment.GetEnvironmentVariable("TEST_PAT"); + var credentials = new Microsoft.VisualStudio.Services.Common.VssBasicCredential(string.Empty, pat); + var adoUri = new Uri(@"https://dev.azure.com/dnceng"); + + var connection = new VssConnection(adoUri, credentials); + + var buildDefinitionId = 15; + var stageName = "Test_Windows_Desktop_Release_64"; + var targetBranchName = "main"; + + using var buildClient = connection.GetClient(); + + // Tries to get the latest succeeded build from the input build definition (pipeline) from dnceng/public with the specified branch. + Logger.Log($"Branch name: {targetBranchName}"); + Logger.Log($"Stage name: {stageName}"); + + ConsoleUtil.WriteLine($"Looking up test execution data from last successful run on branch {targetBranchName} and stage {stageName}"); + + var adoBranchName = $"refs/heads/{targetBranchName}"; + var builds = await buildClient.GetBuildsAsync2(project: "public", new int[] { buildDefinitionId }, resultFilter: BuildResult.Succeeded, queryOrder: BuildQueryOrder.FinishTimeDescending, maxBuildsPerDefinition: 1, reasonFilter: BuildReason.IndividualCI, branchName: adoBranchName, cancellationToken: cancellationToken); + var lastSuccessfulBuild = builds?.FirstOrDefault(); + if (lastSuccessfulBuild == null) + { + // If this is a new branch we may not have any historical data for it. + ConsoleUtil.WriteLine($"##[warning]Unable to get the last successful build for definition {buildDefinitionId} and branch {adoBranchName}"); + return ImmutableDictionary.Empty; + } + + using var testClient = connection.GetClient(); + + // API requires us to pass a time range to query runs for. So just pass the times from the build. + var minTime = lastSuccessfulBuild.QueueTime!.Value; + var maxTime = lastSuccessfulBuild.FinishTime!.Value; + var runsInBuild = await testClient.QueryTestRunsAsync2("public", minTime, maxTime, buildIds: new int[] { lastSuccessfulBuild.Id }, cancellationToken: cancellationToken); + + var runForThisStage = runsInBuild.SingleOrDefault(r => r.Name.Contains(stageName)); + if (runForThisStage == null) + { + // If this is a new stage, historical runs will not have any data for it. + ConsoleUtil.WriteLine($"##[warning]Unable to get a run with name {stageName} from build {lastSuccessfulBuild.Url}."); + return ImmutableDictionary.Empty; + } + + var totalTests = runForThisStage.TotalTests; + Logger.Log($"Expecting {totalTests} tests from build {lastSuccessfulBuild.Id} and run {runForThisStage.Name}"); + + Dictionary testInfos = new(); + + // Get runtimes for all tests. + var timer = new Stopwatch(); + timer.Start(); + for (var i = 0; i < totalTests; i += MaxTestsReturnedPerRequest) + { + var testResults = await testClient.GetTestResultsAsync("public", runForThisStage.Id, skip: i, top: MaxTestsReturnedPerRequest, cancellationToken: cancellationToken); + foreach (var testResult in testResults) + { + // Helix outputs results for the whole dll work item suffixed with WorkItemExecution which we should ignore. + if (testResult.AutomatedTestName.Contains("WorkItemExecution")) + { + Logger.Log($"Skipping overall result for work item {testResult.AutomatedTestName}"); + continue; + } + + //var testName = CleanTestName(testResult.AutomatedTestName); + + if (!testInfos.TryAdd(testResult.AutomatedTestName, TimeSpan.FromMilliseconds(testResult.DurationInMs))) + { + // We can get duplicate tests if a test file is included in multiple assemblies (e.g. analyzer codestyle tests). + // This is fine, we'll just use capture one of the run times since it is the same test being run in both cases. + // + // Another case that can happen is if a test is incorrectly authored to have the same name and namespace as a test in another assembly. For example + // a test that applies to both VB and C#, but the tests in both the C# and VB assembly accidentally use the C# namespace. + // It may have a different run time, but ADO does not let us differentiate by assembly name, so we just have to pick one. + Logger.Log($"Found tests with duplicate fully qualified name {testResult.AutomatedTestName}."); + } + } + } + + timer.Stop(); + + var totalTestRuntime = TimeSpan.FromMilliseconds(testInfos.Values.Sum(t => t.TotalMilliseconds)); + ConsoleUtil.WriteLine($"Retrieved {testInfos.Keys.Count} tests from AzureDevops in {timer.Elapsed}. Total runtime of all tests is {totalTestRuntime}"); + return testInfos.ToImmutableDictionary(); + } + + private static string CleanTestName(string fullyQualifiedTestName) + { + // Some test names contain test arguments, so take everything before the first paren (since they are not valid in identifiers). + var beforeMethodArgs = fullyQualifiedTestName.Split('(')[0]; + return beforeMethodArgs; + } + +} diff --git a/src/Tools/Source/RunTests/TestRunner.cs b/src/Tools/Source/RunTests/TestRunner.cs index 7378e4a88ecb9..861dadd2ab24b 100644 --- a/src/Tools/Source/RunTests/TestRunner.cs +++ b/src/Tools/Source/RunTests/TestRunner.cs @@ -149,12 +149,12 @@ static string getGlobalJsonPath() throw new IOException($@"Could not find global.json by walking up from ""{AppContext.BaseDirectory}""."); } - static void AddRehydrateTestFoldersCommand(StringBuilder commandBuilder, WorkItemInfo workItem, bool isUnix) + static void AddRehydrateTestFoldersCommand(StringBuilder commandBuilder, WorkItemInfo workItemInfo, bool isUnix) { // Rehydrate assemblies that we need to run as part of this work item. - foreach (var testAssemblyGroup in workItem.TypesToTest) + foreach (var testAssembly in workItemInfo.Filters.Keys) { - var directoryName = Path.GetDirectoryName(testAssemblyGroup.Assembly.AssemblyPath); + var directoryName = Path.GetDirectoryName(testAssembly.AssemblyPath); if (isUnix) { // If we're on unix make sure we have permissions to run the rehydrate script. @@ -204,10 +204,7 @@ string MakeHelixWorkItemProject(WorkItemInfo workItemInfo) var payloadDirectory = Path.Combine(msbuildTestPayloadRoot, "artifacts", "bin"); // Update the assembly groups to test with the assembly paths in the context of the helix work item. - workItemInfo = workItemInfo with - { - TypesToTest = workItemInfo.TypesToTest.Select(group => group with { Assembly = group.Assembly with { AssemblyPath = GetHelixRelativeAssemblyPath(group.Assembly.AssemblyPath) } }).ToImmutableArray() - }; + workItemInfo = workItemInfo with { Filters = workItemInfo.Filters.ToImmutableSortedDictionary(kvp => kvp.Key with { AssemblyPath = GetHelixRelativeAssemblyPath(kvp.Key.AssemblyPath) }, kvp => kvp.Value) }; AddRehydrateTestFoldersCommand(command, workItemInfo, isUnix); From eec06a989951d6e6431ff2117f5b5f2c4026793c Mon Sep 17 00:00:00 2001 From: David Barbet Date: Fri, 22 Jul 2022 16:10:38 -0700 Subject: [PATCH 05/26] more work --- eng/prepare-tests.ps1 | 2 +- eng/prepare-tests.sh | 2 +- src/Tools/PrepareTests/PrepareTests.csproj | 3 + src/Tools/PrepareTests/Program.cs | 12 +- src/Tools/PrepareTests/TestDiscovery.cs | 109 ++++++++++++++++++ .../Source/RunTests/AssemblyScheduler.cs | 82 ++++++++++++- 6 files changed, 201 insertions(+), 9 deletions(-) create mode 100644 src/Tools/PrepareTests/TestDiscovery.cs diff --git a/eng/prepare-tests.ps1 b/eng/prepare-tests.ps1 index cfdcee17742ef..2d2bd79c8296f 100644 --- a/eng/prepare-tests.ps1 +++ b/eng/prepare-tests.ps1 @@ -11,7 +11,7 @@ try { $dotnet = Ensure-DotnetSdk # permissions issues make this a pain to do in PrepareTests itself. Remove-Item -Recurse -Force "$RepoRoot\artifacts\testPayload" -ErrorAction SilentlyContinue - Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload" + Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload --dotnetPath `"$dotnet`"" exit 0 } catch { diff --git a/eng/prepare-tests.sh b/eng/prepare-tests.sh index 0e9144522d020..4d6cff2d5da83 100755 --- a/eng/prepare-tests.sh +++ b/eng/prepare-tests.sh @@ -28,4 +28,4 @@ InitializeDotNetCli true # permissions issues make this a pain to do in PrepareTests itself. rm -rf "$repo_root/artifacts/testPayload" -dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix \ No newline at end of file +dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix --dotnetPath ${_InitializeDotNetCli}/dotnet \ No newline at end of file diff --git a/src/Tools/PrepareTests/PrepareTests.csproj b/src/Tools/PrepareTests/PrepareTests.csproj index 6ccdf0f041470..542840d45ce49 100644 --- a/src/Tools/PrepareTests/PrepareTests.csproj +++ b/src/Tools/PrepareTests/PrepareTests.csproj @@ -10,5 +10,8 @@ + + + diff --git a/src/Tools/PrepareTests/Program.cs b/src/Tools/PrepareTests/Program.cs index 1e9c3429566df..a08739630a576 100644 --- a/src/Tools/PrepareTests/Program.cs +++ b/src/Tools/PrepareTests/Program.cs @@ -13,17 +13,19 @@ internal static class Program internal const int ExitFailure = 1; internal const int ExitSuccess = 0; - public static int Main(string[] args) + public static async Task Main(string[] args) { string? source = null; string? destination = null; bool isUnix = false; + string? dotnetPath = null; var options = new OptionSet() { { "source=", "Path to binaries", (string s) => source = s }, { "destination=", "Output path", (string s) => destination = s }, { "unix", "If true, prepares tests for unix environment instead of Windows", o => isUnix = o is object }, + { "dotnetPath=", "Output path", (string s) => dotnetPath = s }, }; options.Parse(args); @@ -39,6 +41,14 @@ public static int Main(string[] args) return ExitFailure; } + if (dotnetPath is null) + { + Console.Error.WriteLine("--dotnetPath argument must be provided"); + return ExitFailure; + } + + TestDiscovery.RunDiscovery(source, dotnetPath, isUnix); + MinimizeUtil.Run(source, destination, isUnix); return ExitSuccess; } diff --git a/src/Tools/PrepareTests/TestDiscovery.cs b/src/Tools/PrepareTests/TestDiscovery.cs new file mode 100644 index 0000000000000..09838ceb8f937 --- /dev/null +++ b/src/Tools/PrepareTests/TestDiscovery.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Diagnostics.Contracts; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace PrepareTests; +internal class TestDiscovery +{ + public static void RunDiscovery(string sourceDirectory, string dotnetPath, bool isUnix) + { + var binDirectory = Path.Combine("artifacts", "bin"); + var assemblies = GetAssemblies(binDirectory, isUnix); + + Console.WriteLine($"Found {assemblies} test assemblies"); + + var vsTestConsole = Directory.EnumerateFiles(Path.Combine(Path.GetDirectoryName(dotnetPath), "sdk"), "vstest.console.dll", SearchOption.AllDirectories).Last(); + + var vstestConsoleWrapper = new VsTestConsoleWrapper(vsTestConsole, new ConsoleParameters + { + LogFilePath = Path.Combine(AppContext.BaseDirectory, "logs", "vstestconsolelogs.txt"), + TraceLevel = TraceLevel.Verbose, + }); + + var tests = new ConcurrentBag(); + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + Parallel.ForEach(assemblies, (assembly) => + { + Console.WriteLine($"Discovering {assembly}"); + vstestConsoleWrapper.DiscoverTests(assemblies, @"", new DiscoveryHandler((test) => tests.Add(test))); + }); + + stopwatch.Stop(); + Console.WriteLine($"Discovered {tests.ToList()} in {stopwatch.Elapsed}"); + } + + private class DiscoveryHandler : ITestDiscoveryEventsHandler + { + private List _tests = new(); + private bool _isComplete = false; + + private readonly Action _addTestsAction; + + public DiscoveryHandler(Action addTestsAction) + { + _addTestsAction = addTestsAction; + } + + public void HandleDiscoveredTests(IEnumerable discoveredTestCases) + { + foreach (var test in discoveredTestCases) + { + _addTestsAction(test.FullyQualifiedName); + } + } + + public void HandleDiscoveryComplete(long totalTests, IEnumerable lastChunk, bool isAborted) + { + _isComplete = true; + } + + public void HandleLogMessage(TestMessageLevel level, string message) + { + Console.WriteLine(message); + } + + public void HandleRawMessage(string rawMessage) + { + } + + public ImmutableArray GetTests() + { + Contract.Assert(_isComplete); + return _tests.Select(t => t.FullyQualifiedName).ToImmutableArray(); + } + } + + + private static List GetAssemblies(string binDirectory, bool isUnix) + { + var unitTestAssemblies = Directory.GetFiles(binDirectory, "*UnitTests.dll", SearchOption.AllDirectories).Where(ShouldInclude); + return unitTestAssemblies.ToList(); + + bool ShouldInclude(string path) + { + if (isUnix) + { + return Path.GetFileName(Path.GetDirectoryName(path)) != "net472"; + } + + return true; + } + } +} diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index f7d81eee64643..8e5cbda49bf68 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -11,6 +11,7 @@ using System.Linq; using System.Reflection; using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; using System.Reflection.PortableExecutable; using System.Text.Json; using System.Threading; @@ -151,13 +152,20 @@ private static ImmutableSortedDictionary> var updated = assemblyTypes.ToImmutableSortedDictionary( kvp => kvp.Key, - kvp => kvp.Value.Select(typeInfo => typeInfo with { Tests = typeInfo.Tests.Select(method => WithExecutionTime(method)).ToImmutableArray() }) + kvp => kvp.Value.Select(typeInfo => typeInfo with { Tests = typeInfo.Tests.Select(method => WithExecutionTime(method, typeInfo)).ToImmutableArray() }) .ToImmutableArray()); LogResults(matchedLocalTests, extraLocalTests); return updated; - MethodInfo WithExecutionTime(MethodInfo methodInfo) + MethodInfo WithExecutionTime(MethodInfo methodInfo, TypeInfo typeInfo) { + if (methodInfo.Name == "SRM_PLACEHOLDER") + { + // SRM failed to find all the methods in this type so we need to lookup test history + // by type name instead of test name. + + } + // Match by fully qualified test method name to azure devops historical data. // Note for combinatorial tests, azure devops helpfully groups all sub-runs under a top level method (with combined test run times) with the same fully qualified method name // that we'll get from looking at all the test methods in the assembly with SRM. @@ -186,6 +194,12 @@ void LogResults(HashSet matchedLocalTests, HashSet extraLocalTes Logger.Log($"Found historical data for test {extraRemoteTest} that was not present in local assemblies"); } + var allTests = assemblyTypes.Values.SelectMany(v => v).SelectMany(v => v.Tests).Select(t => t.FullyQualifiedName).ToList(); + + Debugger.Launch(); + + + var totalExpectedRunTime = TimeSpan.FromMilliseconds(updated.Values.SelectMany(types => types).SelectMany(type => type.Tests).Sum(test => test.ExecutionTime.TotalMilliseconds)); ConsoleUtil.WriteLine($"Matched {matchedLocalTests.Count} tests with historical data. {extraLocalTests.Count} tests were missing historical data. {extraRemoteTests.Count()} tests were missing in local assemblies. Estimate of total execution time for tests is {totalExpectedRunTime}."); } @@ -295,11 +309,11 @@ private static void LogWorkItems(ImmutableArray workItems) foreach (var assembly in workItem.Filters) { var assemblyRuntime = TimeSpan.FromMilliseconds(assembly.Value.Sum(f => f.GetExecutionTime().TotalMilliseconds)); + var typeFilters = assembly.Value.Where(f => f is TypeTestFilter).Count(); + var testFilters = assembly.Value.Where(f => f is MethodTestFilter).Count(); Logger.Log($" - {assembly.Key.AssemblyName} with runtime {assemblyRuntime}"); - foreach (var filter in assembly.Value) - { - Logger.Log($" - {filter} with runtime {filter.GetExecutionTime()}"); - } + Logger.Log($" - {typeFilters} types to filter on"); + Logger.Log($" - {testFilters} tests to filter on"); } } } @@ -326,6 +340,12 @@ private static ImmutableArray GetTypeInfoList(MetadataReader reader) } var (typeName, fullyQualifiedTypeName) = GetTypeName(reader, type); + + if (fullyQualifiedTypeName.Contains("SegmentedHashSet_Generic_Tests_int_With_Comparer_AbsOfInt")) + { + Debugger.Launch(); + } + var methodCount = GetMethodCount(reader, type); if (!ShouldIncludeType(reader, type, methodCount)) { @@ -411,7 +431,9 @@ private static void GetMethods(MetadataReader reader, TypeDefinition type, List< } } + // We might have a base type that defines test methods. var baseTypeHandle = type.BaseType; + if (!baseTypeHandle.IsNil && baseTypeHandle.Kind is HandleKind.TypeDefinition) { var baseType = reader.GetTypeDefinition((TypeDefinitionHandle)baseTypeHandle); @@ -422,6 +444,54 @@ private static void GetMethods(MetadataReader reader, TypeDefinition type, List< GetMethods(reader, baseType, methodList, originalFullyQualifiedTypeName); } } + else if (!baseTypeHandle.IsNil) + { + // We have a base type in a different assembly. We can't really figure out what types are in it + // so we'll just add a dummy test method. Later when we lookup test data from ADO we'll group tests in this type. + var methodName = "SRM_PLACEHOLDER"; + methodList.Add(new MethodInfo(methodName, $"{originalFullyQualifiedTypeName}.{methodName}", TimeSpan.Zero)); + } + } + + private class SignatureTypeEntityHandleProvider : ISignatureTypeProvider + { + public static readonly SignatureTypeEntityHandleProvider Instance = new SignatureTypeEntityHandleProvider(); + + public EntityHandle GetTypeFromSpecification(MetadataReader reader, TypeSpecificationHandle handle) + { + // Create a decoder to process the type specification (which happens with + // instantiated generics). It will call back into us to get the first handle + // for the type def or type ref that the specification starts with. + var sigReader = reader.GetBlobReader(reader.GetTypeSpecification(handle).Signature); + return new SignatureDecoder(this, reader, genericContext: null).DecodeType(ref sigReader); + } + + public EntityHandle GetTypeFromSpecification(MetadataReader reader, object? genericContext, TypeSpecificationHandle handle, byte rawTypeKind) => + GetTypeFromSpecification(reader, handle); + + public EntityHandle GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => handle; + public EntityHandle GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => handle; + + // We want the first handle as is, without any handles for the generic args. + public EntityHandle GetGenericInstantiation(EntityHandle genericType, ImmutableArray typeArguments) => genericType; + + // All the signature elements that would normally augment the passed in type will + // just pass it along unchanged. + public EntityHandle GetModifiedType(EntityHandle modifier, EntityHandle unmodifiedType, bool isRequired) => unmodifiedType; + public EntityHandle GetPinnedType(EntityHandle elementType) => elementType; + public EntityHandle GetArrayType(EntityHandle elementType, ArrayShape shape) => elementType; + public EntityHandle GetByReferenceType(EntityHandle elementType) => elementType; + public EntityHandle GetPointerType(EntityHandle elementType) => elementType; + public EntityHandle GetSZArrayType(EntityHandle elementType) => elementType; + + // We'll never get function pointer types in any types we care about, so we can + // just return the empty string. Similarly, as we never construct generics, + // there is no need to provide anything for the generic parameter names. + public EntityHandle GetFunctionPointerType(MethodSignature signature) => default(EntityHandle); + public EntityHandle GetGenericMethodParameter(object? genericContext, int index) => default(EntityHandle); + public EntityHandle GetGenericTypeParameter(object? genericContext, int index) => default(EntityHandle); + + public EntityHandle GetPrimitiveType(PrimitiveTypeCode typeCode) => default(EntityHandle); } private static bool ShouldIncludeMethod(MetadataReader reader, MethodDefinition method) From e0a231616aa3b555c87ac42fed4649b7b3b2989d Mon Sep 17 00:00:00 2001 From: David Barbet Date: Fri, 22 Jul 2022 18:25:48 -0700 Subject: [PATCH 06/26] some more work on prepare tests --- src/Tools/PrepareTests/Program.cs | 2 +- src/Tools/PrepareTests/TestDiscovery.cs | 52 ++- src/Tools/Source/RunTests/AssemblyInfo.cs | 4 +- .../Source/RunTests/AssemblyScheduler.cs | 390 ++++-------------- .../Source/RunTests/TestHistoryManager.cs | 40 +- 5 files changed, 138 insertions(+), 350 deletions(-) diff --git a/src/Tools/PrepareTests/Program.cs b/src/Tools/PrepareTests/Program.cs index a08739630a576..0438a43548ad6 100644 --- a/src/Tools/PrepareTests/Program.cs +++ b/src/Tools/PrepareTests/Program.cs @@ -13,7 +13,7 @@ internal static class Program internal const int ExitFailure = 1; internal const int ExitSuccess = 0; - public static async Task Main(string[] args) + public static int Main(string[] args) { string? source = null; string? destination = null; diff --git a/src/Tools/PrepareTests/TestDiscovery.cs b/src/Tools/PrepareTests/TestDiscovery.cs index 09838ceb8f937..b46ca8e727d40 100644 --- a/src/Tools/PrepareTests/TestDiscovery.cs +++ b/src/Tools/PrepareTests/TestDiscovery.cs @@ -11,6 +11,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -22,50 +23,57 @@ internal class TestDiscovery { public static void RunDiscovery(string sourceDirectory, string dotnetPath, bool isUnix) { - var binDirectory = Path.Combine("artifacts", "bin"); + var binDirectory = Path.Combine(sourceDirectory, "artifacts", "bin"); var assemblies = GetAssemblies(binDirectory, isUnix); - Console.WriteLine($"Found {assemblies} test assemblies"); + Console.WriteLine($"Found {assemblies.Count} test assemblies"); - var vsTestConsole = Directory.EnumerateFiles(Path.Combine(Path.GetDirectoryName(dotnetPath), "sdk"), "vstest.console.dll", SearchOption.AllDirectories).Last(); + var vsTestConsole = Directory.EnumerateFiles(Path.Combine(Path.GetDirectoryName(dotnetPath)!, "sdk"), "vstest.console.dll", SearchOption.AllDirectories).Last(); var vstestConsoleWrapper = new VsTestConsoleWrapper(vsTestConsole, new ConsoleParameters { - LogFilePath = Path.Combine(AppContext.BaseDirectory, "logs", "vstestconsolelogs.txt"), - TraceLevel = TraceLevel.Verbose, + LogFilePath = Path.Combine(sourceDirectory, "logs", "test_discovery_logs.txt"), + TraceLevel = TraceLevel.Error, }); - var tests = new ConcurrentBag(); + var discoveryHandler = new DiscoveryHandler(); var stopwatch = new Stopwatch(); stopwatch.Start(); - Parallel.ForEach(assemblies, (assembly) => + vstestConsoleWrapper.DiscoverTests(assemblies, @"0", discoveryHandler); + stopwatch.Stop(); + + var tests = discoveryHandler.GetTests(); + + Console.WriteLine($"Discovered {tests.Length} tests in {stopwatch.Elapsed}"); + + stopwatch.Restart(); + var testGroupedByAssembly = tests.GroupBy(test => test.Source); + foreach (var assemblyGroup in testGroupedByAssembly) { - Console.WriteLine($"Discovering {assembly}"); - vstestConsoleWrapper.DiscoverTests(assemblies, @"", new DiscoveryHandler((test) => tests.Add(test))); - }); + var directory = Path.GetDirectoryName(assemblyGroup.Key); + + // Tests with combinatorial data are output multiple times with the same fully qualified test name. + // We only need to include it once as run all combinations under the same filter. + var testToWrite = assemblyGroup.Select(test => test.FullyQualifiedName).Distinct().ToList(); + using var fileStream = File.Create(Path.Combine(directory!, "testlist.json")); + JsonSerializer.Serialize(fileStream, testToWrite); + } stopwatch.Stop(); - Console.WriteLine($"Discovered {tests.ToList()} in {stopwatch.Elapsed}"); + Console.WriteLine($"Serialized tests in {stopwatch.Elapsed}"); } private class DiscoveryHandler : ITestDiscoveryEventsHandler { - private List _tests = new(); + private readonly ConcurrentBag _tests = new(); private bool _isComplete = false; - private readonly Action _addTestsAction; - - public DiscoveryHandler(Action addTestsAction) - { - _addTestsAction = addTestsAction; - } - public void HandleDiscoveredTests(IEnumerable discoveredTestCases) { foreach (var test in discoveredTestCases) { - _addTestsAction(test.FullyQualifiedName); + _tests.Add(test); } } @@ -83,10 +91,10 @@ public void HandleRawMessage(string rawMessage) { } - public ImmutableArray GetTests() + public ImmutableArray GetTests() { Contract.Assert(_isComplete); - return _tests.Select(t => t.FullyQualifiedName).ToImmutableArray(); + return _tests.ToImmutableArray(); } } diff --git a/src/Tools/Source/RunTests/AssemblyInfo.cs b/src/Tools/Source/RunTests/AssemblyInfo.cs index 8c2cc36e0daa2..4b8e7e45a3999 100644 --- a/src/Tools/Source/RunTests/AssemblyInfo.cs +++ b/src/Tools/Source/RunTests/AssemblyInfo.cs @@ -26,12 +26,12 @@ public int CompareTo(object? obj) } } -public readonly record struct TypeInfo(string Name, string FullyQualifiedName, ImmutableArray Tests) +public readonly record struct TypeInfo(string Name, string FullyQualifiedName, ImmutableArray Tests) { public override string ToString() => $"[Type]{FullyQualifiedName}"; } -public readonly record struct MethodInfo(string Name, string FullyQualifiedName, TimeSpan ExecutionTime) +public readonly record struct TestMethodInfo(string Name, string FullyQualifiedName, TimeSpan ExecutionTime) { public override string ToString() => $"[Method]{FullyQualifiedName}"; } diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index 8e5cbda49bf68..8f9756bd52f5a 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.Contracts; using System.IO; using System.Linq; using System.Reflection; @@ -16,6 +17,8 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Microsoft.TeamFoundation.Build.WebApi; +using Microsoft.VisualStudio.Services.Common; namespace RunTests { @@ -33,23 +36,18 @@ internal interface ITestFilter internal record struct AssemblyTestFilter(ImmutableArray TypesInAssembly) : ITestFilter { - TimeSpan ITestFilter.GetExecutionTime() - { - return TimeSpan.FromMilliseconds(TypesInAssembly.SelectMany(type => type.Tests).Sum(test => test.ExecutionTime.TotalMilliseconds)); - } + TimeSpan ITestFilter.GetExecutionTime() => TimeSpan.FromMilliseconds(TypesInAssembly.SelectMany(type => type.Tests).Sum(test => test.ExecutionTime.TotalMilliseconds)); - string ITestFilter.GetFilterString() - { - return string.Empty; - } + /// + /// TODO - NOT FINE IF THERE ARE OTHER FILTERS - + /// + /// + string ITestFilter.GetFilterString() => string.Empty; } internal record struct TypeTestFilter(TypeInfo Type) : ITestFilter { - TimeSpan ITestFilter.GetExecutionTime() - { - return TimeSpan.FromMilliseconds(Type.Tests.Sum(test => test.ExecutionTime.TotalMilliseconds)); - } + TimeSpan ITestFilter.GetExecutionTime() => TimeSpan.FromMilliseconds(Type.Tests.Sum(test => test.ExecutionTime.TotalMilliseconds)); string ITestFilter.GetFilterString() { @@ -61,21 +59,24 @@ string ITestFilter.GetFilterString() } } - internal record struct MethodTestFilter(MethodInfo Test) : ITestFilter + internal record struct MethodTestFilter(TestMethodInfo Test) : ITestFilter { - TimeSpan ITestFilter.GetExecutionTime() - { - return Test.ExecutionTime; - } + TimeSpan ITestFilter.GetExecutionTime() => Test.ExecutionTime; - string ITestFilter.GetFilterString() - { - return Test.FullyQualifiedName; - } + string ITestFilter.GetFilterString() => Test.FullyQualifiedName; } internal sealed class AssemblyScheduler { + /// + /// Our execution time limit is 3 minutes. We really want to run tests under 5 minutes, but we need to limit test execution time + /// to 3 minutes to account for overhead elsewhere in setting up the test run, for example + /// 1. Test discovery. + /// 2. Downloading assets to the helix machine. + /// 3. Setting up the test host for each assembly. + /// + private static readonly TimeSpan s_maxExecutionTime = TimeSpan.FromMinutes(3); + private readonly Options _options; internal AssemblyScheduler(Options options) @@ -110,16 +111,12 @@ public async Task> ScheduleAsync(ImmutableArray> // Determine the average execution time so that we can use it for tests that do not have any history. var averageExecutionTime = TimeSpan.FromMilliseconds(testHistory.Values.Average(t => t.TotalMilliseconds)); - // Store the tests from our assemblies that we couldn't find a history for to log. - var extraLocalTests = new HashSet(); + // Store the tests we found locally that were missing remote historical data. + var unmatchedLocalTests = new HashSet(); - // Store the tests that we were able to match to historical data. - var matchedLocalTests = new HashSet(); + // Store the tests we found in the remote historical data so we can report any we didn't find locally. + var matchedRemoteTests = new HashSet(); var updated = assemblyTypes.ToImmutableSortedDictionary( kvp => kvp.Key, - kvp => kvp.Value.Select(typeInfo => typeInfo with { Tests = typeInfo.Tests.Select(method => WithExecutionTime(method, typeInfo)).ToImmutableArray() }) - .ToImmutableArray()); - LogResults(matchedLocalTests, extraLocalTests); + kvp => kvp.Value.Select(WithTypeExecutionTime).ToImmutableArray()); + + LogResults(); return updated; - MethodInfo WithExecutionTime(MethodInfo methodInfo, TypeInfo typeInfo) + TypeInfo WithTypeExecutionTime(TypeInfo typeInfo) { - if (methodInfo.Name == "SRM_PLACEHOLDER") - { - // SRM failed to find all the methods in this type so we need to lookup test history - // by type name instead of test name. - - } + var tests = typeInfo.Tests.Select(WithTestExecutionTime).ToImmutableArray(); + return typeInfo with { Tests = tests }; + } + TestMethodInfo WithTestExecutionTime(TestMethodInfo methodInfo) + { // Match by fully qualified test method name to azure devops historical data. // Note for combinatorial tests, azure devops helpfully groups all sub-runs under a top level method (with combined test run times) with the same fully qualified method name - // that we'll get from looking at all the test methods in the assembly with SRM. + // that we get during test discovery. Since we only filter by the single method name (and not individual combinatorial runs) we do want the combined execution time. if (testHistory.TryGetValue(methodInfo.FullyQualifiedName, out var executionTime)) { - matchedLocalTests.Add(methodInfo.FullyQualifiedName); + matchedRemoteTests.Add(methodInfo.FullyQualifiedName); return methodInfo with { ExecutionTime = executionTime }; } // We didn't find the local type from our assembly in test run historical data. // This can happen if our SRM heuristic incorrectly counted a normal method as a test method (which it can do often). - extraLocalTests.Add(methodInfo.FullyQualifiedName); + unmatchedLocalTests.Add(methodInfo.FullyQualifiedName); return methodInfo with { ExecutionTime = averageExecutionTime }; } - void LogResults(HashSet matchedLocalTests, HashSet extraLocalTests) + void LogResults() { - foreach (var extraLocalTest in extraLocalTests) + foreach (var unmatchedLocalTest in unmatchedLocalTests) { - Logger.Log($"Could not find test execution history for test {extraLocalTest}"); + Logger.Log($"Could not find test execution history for test {unmatchedLocalTest}"); } - var extraRemoteTests = testHistory.Keys.Where(type => !matchedLocalTests.Contains(type)); - foreach (var extraRemoteTest in extraRemoteTests) + var unmatchedRemoteTests = testHistory.Keys.Where(type => !matchedRemoteTests.Contains(type)); + foreach (var unmatchedRemoteTest in unmatchedRemoteTests) { - Logger.Log($"Found historical data for test {extraRemoteTest} that was not present in local assemblies"); + Logger.Log($"Found historical data for test {unmatchedRemoteTest} that was not present in local assemblies"); } var allTests = assemblyTypes.Values.SelectMany(v => v).SelectMany(v => v.Tests).Select(t => t.FullyQualifiedName).ToList(); - Debugger.Launch(); - - - var totalExpectedRunTime = TimeSpan.FromMilliseconds(updated.Values.SelectMany(types => types).SelectMany(type => type.Tests).Sum(test => test.ExecutionTime.TotalMilliseconds)); - ConsoleUtil.WriteLine($"Matched {matchedLocalTests.Count} tests with historical data. {extraLocalTests.Count} tests were missing historical data. {extraRemoteTests.Count()} tests were missing in local assemblies. Estimate of total execution time for tests is {totalExpectedRunTime}."); + ConsoleUtil.WriteLine($"{unmatchedLocalTests.Count} tests were missing historical data. {unmatchedRemoteTests.Count()} tests were missing in local assemblies. Estimate of total execution time for tests is {totalExpectedRunTime}."); } } @@ -304,280 +296,56 @@ private static void LogWorkItems(ImmutableArray workItems) Logger.Log("==== Work Item List ===="); foreach (var workItem in workItems) { - var totalRuntime = TimeSpan.FromMilliseconds(workItem.Filters.Values.SelectMany(f => f).Sum(f => f.GetExecutionTime().TotalMilliseconds)); - Logger.Log($"- Work Item (Runtime {totalRuntime})"); - foreach (var assembly in workItem.Filters) + var totalExecutionTime = TimeSpan.FromMilliseconds(workItem.Filters.Values.SelectMany(f => f).Sum(f => f.GetExecutionTime().TotalMilliseconds)); + Logger.Log($"- Work Item {workItem.PartitionIndex} (Execution time {totalExecutionTime})"); + if (totalExecutionTime > s_maxExecutionTime) { - var assemblyRuntime = TimeSpan.FromMilliseconds(assembly.Value.Sum(f => f.GetExecutionTime().TotalMilliseconds)); - var typeFilters = assembly.Value.Where(f => f is TypeTestFilter).Count(); - var testFilters = assembly.Value.Where(f => f is MethodTestFilter).Count(); - Logger.Log($" - {assembly.Key.AssemblyName} with runtime {assemblyRuntime}"); - Logger.Log($" - {typeFilters} types to filter on"); - Logger.Log($" - {testFilters} tests to filter on"); + ConsoleUtil.WriteLine($"##[warning]Work item {workItem.PartitionIndex} estimated execution {totalExecutionTime} time exceeds max execution time {s_maxExecutionTime}. See runtests.log for details."); } - } - } - private static ImmutableArray GetTypeInfoList(AssemblyInfo assemblyInfo) - { - using (var stream = File.OpenRead(assemblyInfo.AssemblyPath)) - using (var peReader = new PEReader(stream)) - { - var metadataReader = peReader.GetMetadataReader(); - return GetTypeInfoList(metadataReader); - } - } - - private static ImmutableArray GetTypeInfoList(MetadataReader reader) - { - var list = new List(); - foreach (var handle in reader.TypeDefinitions) - { - var type = reader.GetTypeDefinition(handle); - if (!IsValidIdentifier(reader, type.Name)) - { - continue; - } - - var (typeName, fullyQualifiedTypeName) = GetTypeName(reader, type); - - if (fullyQualifiedTypeName.Contains("SegmentedHashSet_Generic_Tests_int_With_Comparer_AbsOfInt")) - { - Debugger.Launch(); - } - - var methodCount = GetMethodCount(reader, type); - if (!ShouldIncludeType(reader, type, methodCount)) - { - continue; - } - - var methodList = new List(); - GetMethods(reader, type, methodList, fullyQualifiedTypeName); - - list.Add(new TypeInfo(typeName, fullyQualifiedTypeName, methodList.ToImmutableArray())); - } - - // Ensure we get classes back in a deterministic order. - list.Sort((x, y) => x.FullyQualifiedName.CompareTo(y.FullyQualifiedName)); - return list.ToImmutableArray(); - } - - private static bool IsPublicType(TypeDefinition type) - { - // See https://docs.microsoft.com/en-us/dotnet/api/system.reflection.typeattributes?view=net-6.0#examples - // for extracting this information from the TypeAttributes. - var visibility = type.Attributes & TypeAttributes.VisibilityMask; - var isPublic = visibility == TypeAttributes.Public || visibility == TypeAttributes.NestedPublic; - return isPublic; - } - - private static bool IsClass(TypeDefinition type) - { - var classSemantics = type.Attributes & TypeAttributes.ClassSemanticsMask; - var isClass = classSemantics == TypeAttributes.Class; - return isClass; - } - - private static bool IsAbstract(TypeDefinition type) - { - var isAbstract = (type.Attributes & TypeAttributes.Abstract) != 0; - return isAbstract; - } - - /// - /// Determine if this type should be one of the class values passed to xunit. This - /// code doesn't actually resolve base types or trace through inherrited Fact attributes - /// hence we have to error on the side of including types with no tests vs. excluding them. - /// - private static bool ShouldIncludeType(MetadataReader reader, TypeDefinition type, int testMethodCount) - { - // xunit only handles public, non-abstract classes - if (!IsPublicType(type) || IsAbstract(type) || !IsClass(type)) - { - return false; - } - - // Compiler generated types / methods have the shape of the heuristic that we are looking - // at here. Filter them out as well. - if (!IsValidIdentifier(reader, type.Name)) - { - return false; - } - - if (testMethodCount > 0) - { - return true; - } - - // The case we still have to consider at this point is a class with 0 defined methods, - // inheritting from a class with > 0 defined test methods. That is a completely valid - // xunit scenario. For now we're just going to exclude types that inherit from object - // or other built-in base types because they clearly don't fit that category. - return !InheritsFromFrameworkBaseType(reader, type); - } - - private static void GetMethods(MetadataReader reader, TypeDefinition type, List methodList, string originalFullyQualifiedTypeName) - { - var methods = type.GetMethods(); - - foreach (var methodHandle in methods) - { - var method = reader.GetMethodDefinition(methodHandle); - var methodInfo = GetMethodInfo(reader, method, originalFullyQualifiedTypeName); - if (ShouldIncludeMethod(reader, method)) - { - methodList.Add(methodInfo); - } - } - - // We might have a base type that defines test methods. - var baseTypeHandle = type.BaseType; - - if (!baseTypeHandle.IsNil && baseTypeHandle.Kind is HandleKind.TypeDefinition) - { - var baseType = reader.GetTypeDefinition((TypeDefinitionHandle)baseTypeHandle); - - // We only want to look for test methods in public base types. - if (IsPublicType(baseType) && IsClass(baseType) && !InheritsFromFrameworkBaseType(reader, type)) - { - GetMethods(reader, baseType, methodList, originalFullyQualifiedTypeName); - } - } - else if (!baseTypeHandle.IsNil) - { - // We have a base type in a different assembly. We can't really figure out what types are in it - // so we'll just add a dummy test method. Later when we lookup test data from ADO we'll group tests in this type. - var methodName = "SRM_PLACEHOLDER"; - methodList.Add(new MethodInfo(methodName, $"{originalFullyQualifiedTypeName}.{methodName}", TimeSpan.Zero)); - } - } - - private class SignatureTypeEntityHandleProvider : ISignatureTypeProvider - { - public static readonly SignatureTypeEntityHandleProvider Instance = new SignatureTypeEntityHandleProvider(); - - public EntityHandle GetTypeFromSpecification(MetadataReader reader, TypeSpecificationHandle handle) - { - // Create a decoder to process the type specification (which happens with - // instantiated generics). It will call back into us to get the first handle - // for the type def or type ref that the specification starts with. - var sigReader = reader.GetBlobReader(reader.GetTypeSpecification(handle).Signature); - return new SignatureDecoder(this, reader, genericContext: null).DecodeType(ref sigReader); - } - - public EntityHandle GetTypeFromSpecification(MetadataReader reader, object? genericContext, TypeSpecificationHandle handle, byte rawTypeKind) => - GetTypeFromSpecification(reader, handle); - - public EntityHandle GetTypeFromDefinition(MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => handle; - public EntityHandle GetTypeFromReference(MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => handle; - - // We want the first handle as is, without any handles for the generic args. - public EntityHandle GetGenericInstantiation(EntityHandle genericType, ImmutableArray typeArguments) => genericType; - - // All the signature elements that would normally augment the passed in type will - // just pass it along unchanged. - public EntityHandle GetModifiedType(EntityHandle modifier, EntityHandle unmodifiedType, bool isRequired) => unmodifiedType; - public EntityHandle GetPinnedType(EntityHandle elementType) => elementType; - public EntityHandle GetArrayType(EntityHandle elementType, ArrayShape shape) => elementType; - public EntityHandle GetByReferenceType(EntityHandle elementType) => elementType; - public EntityHandle GetPointerType(EntityHandle elementType) => elementType; - public EntityHandle GetSZArrayType(EntityHandle elementType) => elementType; - - // We'll never get function pointer types in any types we care about, so we can - // just return the empty string. Similarly, as we never construct generics, - // there is no need to provide anything for the generic parameter names. - public EntityHandle GetFunctionPointerType(MethodSignature signature) => default(EntityHandle); - public EntityHandle GetGenericMethodParameter(object? genericContext, int index) => default(EntityHandle); - public EntityHandle GetGenericTypeParameter(object? genericContext, int index) => default(EntityHandle); - - public EntityHandle GetPrimitiveType(PrimitiveTypeCode typeCode) => default(EntityHandle); - } - - private static bool ShouldIncludeMethod(MetadataReader reader, MethodDefinition method) - { - var visibility = method.Attributes & MethodAttributes.MemberAccessMask; - var isPublic = visibility == MethodAttributes.Public; - - var hasMethodAttributes = method.GetCustomAttributes().Count > 0; - - var isValidIdentifier = IsValidIdentifier(reader, method.Name); - - return isPublic && hasMethodAttributes && isValidIdentifier; - } - - private static MethodInfo GetMethodInfo(MetadataReader reader, MethodDefinition method, string fullyQualifiedTypeName) - { - var methodName = reader.GetString(method.Name); - return new MethodInfo(methodName, $"{fullyQualifiedTypeName}.{methodName}", TimeSpan.Zero); - } - - private static int GetMethodCount(MetadataReader reader, TypeDefinition type) - { - var count = 0; - foreach (var handle in type.GetMethods()) - { - var methodDefinition = reader.GetMethodDefinition(handle); - if (!ShouldIncludeMethod(reader, methodDefinition)) + foreach (var assembly in workItem.Filters) { - continue; - } - - count++; - } + var assemblyRuntime = TimeSpan.FromMilliseconds(assembly.Value.Sum(f => f.GetExecutionTime().TotalMilliseconds)); + Logger.Log($" - {assembly.Key.AssemblyName} with execution time {assemblyRuntime}"); - return count; - } + var typeFilters = assembly.Value.Where(f => f is TypeTestFilter); + if (typeFilters.Count() > 0) + { + Logger.Log($" - {typeFilters.Count()} types: {string.Join(",", typeFilters)}"); + } - private static bool IsValidIdentifier(MetadataReader reader, StringHandle handle) - { - var name = reader.GetString(handle); - for (int i = 0; i < name.Length; i++) - { - switch (name[i]) - { - case '<': - case '>': - case '$': - return false; + var testFilters = assembly.Value.Where(f => f is MethodTestFilter); + if (testFilters.Count() > 0) + { + Logger.Log($" - {testFilters.Count()} tests: {string.Join(",", testFilters)}"); + } } } - - return true; } - private static bool InheritsFromFrameworkBaseType(MetadataReader reader, TypeDefinition type) + private static ImmutableArray GetTypeInfoList(AssemblyInfo assemblyInfo) { - if (type.BaseType.Kind != HandleKind.TypeReference) - { - return false; - } + var assemblyDirectory = Path.GetDirectoryName(assemblyInfo.AssemblyPath); + var testListPath = Path.Combine(assemblyDirectory!, "testlist.json"); + var deserialized = JsonSerializer.Deserialize>(File.ReadAllText(testListPath)); - var typeRef = reader.GetTypeReference((TypeReferenceHandle)type.BaseType); - return - reader.GetString(typeRef.Namespace) == "System" && - reader.GetString(typeRef.Name) is "Object" or "ValueType" or "Enum"; - } - - private static (string Name, string FullyQualifiedName) GetTypeName(MetadataReader reader, TypeDefinition type) - { - var typeName = reader.GetString(type.Name); + Contract.Assert(deserialized != null); + var tests = deserialized.GroupBy(GetTypeName) + .Select(group => new TypeInfo(GetName(group.Key), group.Key, group.Select(test => new TestMethodInfo(GetName(test), test, TimeSpan.Zero)).ToImmutableArray())) + .ToImmutableArray(); + return tests; - if (TypeAttributes.NestedPublic == (type.Attributes & TypeAttributes.NestedPublic)) + static string GetTypeName(string fullyQualifiedTestName) { - // Need to take into account the containing type. - var declaringType = reader.GetTypeDefinition(type.GetDeclaringType()); - var (declaringTypeName, declaringTypeFullName) = GetTypeName(reader, declaringType); - return (typeName, $"{declaringTypeFullName}+{typeName}"); + var periodBeforeMethod = fullyQualifiedTestName.LastIndexOf("."); + return fullyQualifiedTestName[..periodBeforeMethod]; } - var namespaceName = reader.GetString(type.Namespace); - if (string.IsNullOrEmpty(namespaceName)) + static string GetName(string fullyQualifiedName) { - return (typeName, typeName); + var lastPeriod = fullyQualifiedName.LastIndexOf("."); + return fullyQualifiedName[(lastPeriod + 1)..]; } - - return (typeName, $"{namespaceName}.{typeName}"); } } } diff --git a/src/Tools/Source/RunTests/TestHistoryManager.cs b/src/Tools/Source/RunTests/TestHistoryManager.cs index 263c5164abdba..eb52dda439a39 100644 --- a/src/Tools/Source/RunTests/TestHistoryManager.cs +++ b/src/Tools/Source/RunTests/TestHistoryManager.cs @@ -25,20 +25,27 @@ internal class TestHistoryManager /// private const int MaxTestsReturnedPerRequest = 10000; + /// + /// The pipeline id for roslyn-ci, see https://dev.azure.com/dnceng/public/_build?definitionId=15&_a=summary + /// + private const int RoslynCiBuildDefinitionId = 15; + + /// + /// The Azure devops project that the build pipeline is located in. + /// + private static readonly Uri s_projectUri = new(@"https://dev.azure.com/dnceng"); + /// /// Todo - build definition comes from System.DefinitionId /// Todo - stageName comes from System.StageName to identify which test runs to get from the build. /// public static async Task> GetTestHistoryAsync(/*int buildDefinitionId, string stageName, string branchName*/CancellationToken cancellationToken) { - Debugger.Launch(); var pat = Environment.GetEnvironmentVariable("TEST_PAT"); var credentials = new Microsoft.VisualStudio.Services.Common.VssBasicCredential(string.Empty, pat); - var adoUri = new Uri(@"https://dev.azure.com/dnceng"); - var connection = new VssConnection(adoUri, credentials); + var connection = new VssConnection(s_projectUri, credentials); - var buildDefinitionId = 15; var stageName = "Test_Windows_Desktop_Release_64"; var targetBranchName = "main"; @@ -48,15 +55,13 @@ public static async Task> GetTestHistoryAs Logger.Log($"Branch name: {targetBranchName}"); Logger.Log($"Stage name: {stageName}"); - ConsoleUtil.WriteLine($"Looking up test execution data from last successful run on branch {targetBranchName} and stage {stageName}"); - var adoBranchName = $"refs/heads/{targetBranchName}"; - var builds = await buildClient.GetBuildsAsync2(project: "public", new int[] { buildDefinitionId }, resultFilter: BuildResult.Succeeded, queryOrder: BuildQueryOrder.FinishTimeDescending, maxBuildsPerDefinition: 1, reasonFilter: BuildReason.IndividualCI, branchName: adoBranchName, cancellationToken: cancellationToken); + var builds = await buildClient.GetBuildsAsync2(project: "public", new int[] { RoslynCiBuildDefinitionId }, resultFilter: BuildResult.Succeeded, queryOrder: BuildQueryOrder.FinishTimeDescending, maxBuildsPerDefinition: 1, reasonFilter: BuildReason.IndividualCI, branchName: adoBranchName, cancellationToken: cancellationToken); var lastSuccessfulBuild = builds?.FirstOrDefault(); if (lastSuccessfulBuild == null) { // If this is a new branch we may not have any historical data for it. - ConsoleUtil.WriteLine($"##[warning]Unable to get the last successful build for definition {buildDefinitionId} and branch {adoBranchName}"); + ConsoleUtil.WriteLine($"##[warning]Unable to get the last successful build for definition {RoslynCiBuildDefinitionId} and branch {adoBranchName}"); return ImmutableDictionary.Empty; } @@ -75,10 +80,12 @@ public static async Task> GetTestHistoryAs return ImmutableDictionary.Empty; } + ConsoleUtil.WriteLine($"Looking up test execution data from last successful build {lastSuccessfulBuild.Id} on branch {targetBranchName} and stage {stageName}"); + var totalTests = runForThisStage.TotalTests; - Logger.Log($"Expecting {totalTests} tests from build {lastSuccessfulBuild.Id} and run {runForThisStage.Name}"); Dictionary testInfos = new(); + var duplicateCount = 0; // Get runtimes for all tests. var timer = new Stopwatch(); @@ -95,23 +102,28 @@ public static async Task> GetTestHistoryAs continue; } - //var testName = CleanTestName(testResult.AutomatedTestName); + var testName = CleanTestName(testResult.AutomatedTestName); - if (!testInfos.TryAdd(testResult.AutomatedTestName, TimeSpan.FromMilliseconds(testResult.DurationInMs))) + if (!testInfos.TryAdd(testName, TimeSpan.FromMilliseconds(testResult.DurationInMs))) { // We can get duplicate tests if a test file is included in multiple assemblies (e.g. analyzer codestyle tests). - // This is fine, we'll just use capture one of the run times since it is the same test being run in both cases. + // This is fine, we'll just use capture one of the run times since it is the same test being run in both cases and unlikely to have different run times. // // Another case that can happen is if a test is incorrectly authored to have the same name and namespace as a test in another assembly. For example // a test that applies to both VB and C#, but the tests in both the C# and VB assembly accidentally use the C# namespace. // It may have a different run time, but ADO does not let us differentiate by assembly name, so we just have to pick one. - Logger.Log($"Found tests with duplicate fully qualified name {testResult.AutomatedTestName}."); + duplicateCount++; } } } timer.Stop(); + if (duplicateCount > 0) + { + Logger.Log($"Found {duplicateCount} duplicate tests in run {runForThisStage.Name}."); + } + var totalTestRuntime = TimeSpan.FromMilliseconds(testInfos.Values.Sum(t => t.TotalMilliseconds)); ConsoleUtil.WriteLine($"Retrieved {testInfos.Keys.Count} tests from AzureDevops in {timer.Elapsed}. Total runtime of all tests is {totalTestRuntime}"); return testInfos.ToImmutableDictionary(); @@ -119,7 +131,7 @@ public static async Task> GetTestHistoryAs private static string CleanTestName(string fullyQualifiedTestName) { - // Some test names contain test arguments, so take everything before the first paren (since they are not valid in identifiers). + // Some test names contain test arguments, so take everything before the first paren (since they are not valid in the fully qualified test name). var beforeMethodArgs = fullyQualifiedTestName.Split('(')[0]; return beforeMethodArgs; } From 43991a344882f4da035dd6925e1d96e01ecfab0d Mon Sep 17 00:00:00 2001 From: David Barbet Date: Sat, 23 Jul 2022 00:13:46 -0700 Subject: [PATCH 07/26] Cleanup unneeded code --- src/Tools/Source/RunTests/ITestExecutor.cs | 4 +- src/Tools/Source/RunTests/ProcDumpUtil.cs | 69 ------------------- .../Source/RunTests/ProcessTestExecutor.cs | 37 +++------- src/Tools/Source/RunTests/Program.cs | 31 ++------- 4 files changed, 16 insertions(+), 125 deletions(-) diff --git a/src/Tools/Source/RunTests/ITestExecutor.cs b/src/Tools/Source/RunTests/ITestExecutor.cs index 1cc9fa526c446..8860870bd9cc4 100644 --- a/src/Tools/Source/RunTests/ITestExecutor.cs +++ b/src/Tools/Source/RunTests/ITestExecutor.cs @@ -10,17 +10,15 @@ namespace RunTests internal readonly struct TestExecutionOptions { internal string DotnetFilePath { get; } - internal ProcDumpInfo? ProcDumpInfo { get; } internal string TestResultsDirectory { get; } internal string? TestFilter { get; } internal bool IncludeHtml { get; } internal bool Retry { get; } internal bool CollectDumps { get; } - internal TestExecutionOptions(string dotnetFilePath, ProcDumpInfo? procDumpInfo, string testResultsDirectory, string? testFilter, bool includeHtml, bool retry, bool collectDumps) + internal TestExecutionOptions(string dotnetFilePath, string testResultsDirectory, string? testFilter, bool includeHtml, bool retry, bool collectDumps) { DotnetFilePath = dotnetFilePath; - ProcDumpInfo = procDumpInfo; TestResultsDirectory = testResultsDirectory; TestFilter = testFilter; IncludeHtml = includeHtml; diff --git a/src/Tools/Source/RunTests/ProcDumpUtil.cs b/src/Tools/Source/RunTests/ProcDumpUtil.cs index efcb7b1b77b79..6fe289137063a 100644 --- a/src/Tools/Source/RunTests/ProcDumpUtil.cs +++ b/src/Tools/Source/RunTests/ProcDumpUtil.cs @@ -12,44 +12,6 @@ namespace RunTests { - internal readonly struct ProcDumpInfo - { - private const string KeyProcDumpFilePath = "ProcDumpFilePath"; - private const string KeyProcDumpDirectory = "ProcDumpOutputPath"; - - internal string ProcDumpFilePath { get; } - internal string DumpDirectory { get; } - - internal ProcDumpInfo(string procDumpFilePath, string dumpDirectory) - { - Debug.Assert(Path.IsPathRooted(procDumpFilePath)); - Debug.Assert(Path.IsPathRooted(dumpDirectory)); - ProcDumpFilePath = procDumpFilePath; - DumpDirectory = dumpDirectory; - } - - internal void WriteEnvironmentVariables(Dictionary environment) - { - environment[KeyProcDumpFilePath] = ProcDumpFilePath; - environment[KeyProcDumpDirectory] = DumpDirectory; - } - - internal static ProcDumpInfo? ReadFromEnvironment() - { - bool validate([NotNullWhen(true)] string? s) => !string.IsNullOrEmpty(s) && Path.IsPathRooted(s); - - var procDumpFilePath = Environment.GetEnvironmentVariable(KeyProcDumpFilePath); - var dumpDirectory = Environment.GetEnvironmentVariable(KeyProcDumpDirectory); - - if (!validate(procDumpFilePath) || !validate(dumpDirectory)) - { - return null; - } - - return new ProcDumpInfo(procDumpFilePath, dumpDirectory); - } - } - internal static class DumpUtil { #pragma warning disable CA1416 // Validate platform compatibility @@ -81,35 +43,4 @@ internal static bool IsAdministrator() } #pragma warning restore CA1416 // Validate platform compatibility } - - internal static class ProcDumpUtil - { - internal static Process AttachProcDump(ProcDumpInfo procDumpInfo, int processId) - { - return AttachProcDump(procDumpInfo.ProcDumpFilePath, processId, procDumpInfo.DumpDirectory); - } - - internal static string GetProcDumpCommandLine(int processId, string dumpDirectory) - { - // /accepteula command line option to automatically accept the Sysinternals license agreement. - // -ma Write a 'Full' dump file. Includes All the Image, Mapped and Private memory. - // -e Write a dump when the process encounters an unhandled exception. Include the 1 to create dump on first chance exceptions. - // -f C00000FD.STACK_OVERFLOWC Dump when a stack overflow first chance exception is encountered. - const string procDumpSwitches = "/accepteula -ma -e -f C00000FD.STACK_OVERFLOW"; - dumpDirectory = dumpDirectory.TrimEnd('\\'); - return $" {procDumpSwitches} {processId} \"{dumpDirectory}\""; - } - - /// - /// Attaches a new procdump.exe against the specified process. - /// - /// The path to the procdump executable - /// process id - /// destination directory for dumps - internal static Process AttachProcDump(string procDumpFilePath, int processId, string dumpDirectory) - { - Directory.CreateDirectory(dumpDirectory); - return Process.Start(procDumpFilePath, GetProcDumpCommandLine(processId, dumpDirectory)); - } - } } diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index 674b40e67d4b4..3075ec0130bde 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -20,13 +20,6 @@ namespace RunTests { internal sealed class ProcessTestExecutor { - public TestExecutionOptions Options { get; } - - internal ProcessTestExecutor(TestExecutionOptions options) - { - Options = options; - } - public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuotes, Options options) { // http://www.gnu.org/software/bash/manual/html_node/Single-Quotes.html @@ -42,7 +35,7 @@ public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuote builder.Append($@" {string.Join(" ", escapedAssemblyPaths)}"); var filters = workItem.Filters.Values.SelectMany(filter => filter).Where(filter => !string.IsNullOrEmpty(filter.GetFilterString())).ToImmutableArray(); - if (filters.Length > 0 || !string.IsNullOrWhiteSpace(Options.TestFilter)) + if (filters.Length > 0 || !string.IsNullOrWhiteSpace(options.TestFilter)) { builder.Append($@" --filter {sep}"); var any = false; @@ -53,10 +46,10 @@ public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuote } builder.Append(sep); - if (Options.TestFilter is object) + if (options.TestFilter is object) { MaybeAddSeparator(); - builder.Append(Options.TestFilter); + builder.Append(options.TestFilter); } void MaybeAddSeparator(char separator = '|') @@ -73,12 +66,12 @@ void MaybeAddSeparator(char separator = '|') builder.Append($@" --arch {options.Architecture}"); builder.Append($@" --logger {sep}xunit;LogFilePath={GetResultsFilePath(workItem, options, "xml")}{sep}"); - if (Options.IncludeHtml) + if (options.IncludeHtml) { builder.AppendFormat($@" --logger {sep}html;LogFileName={GetResultsFilePath(workItem, options, "html")}{sep}"); } - if (!Options.CollectDumps) + if (!options.CollectDumps) { // The 'CollectDumps' option uses operating system features to collect dumps when a process crashes. We // only enable the test executor blame feature in remaining cases, as the latter relies on ProcDump and @@ -103,7 +96,7 @@ void MaybeAddSeparator(char separator = '|') private string GetResultsFilePath(WorkItemInfo workItemInfo, Options options, string suffix = "xml") { var fileName = $"WorkItem_{workItemInfo.PartitionIndex}_{options.Architecture}_test_results.{suffix}"; - return Path.Combine(Options.TestResultsDirectory, fileName); + return Path.Combine(options.TestResultsDirectory, fileName); } public async Task RunTestAsync(WorkItemInfo workItemInfo, Options options, CancellationToken cancellationToken) @@ -111,7 +104,7 @@ public async Task RunTestAsync(WorkItemInfo workItemInfo, Options op var result = await RunTestAsyncInternal(workItemInfo, options, isRetry: false, cancellationToken); // For integration tests (TestVsi), we make one more attempt to re-run failed tests. - if (Options.Retry && !HasBuiltInRetry(workItemInfo) && !Options.IncludeHtml && !result.Succeeded) + if (options.Retry && !HasBuiltInRetry(workItemInfo) && !options.IncludeHtml && !result.Succeeded) { return await RunTestAsyncInternal(workItemInfo, options, isRetry: true, cancellationToken); } @@ -132,16 +125,14 @@ private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, O var commandLineArguments = GetCommandLineArguments(workItemInfo, useSingleQuotes: false, options); var resultsFilePath = GetResultsFilePath(workItemInfo, options); var resultsDir = Path.GetDirectoryName(resultsFilePath); - var htmlResultsFilePath = Options.IncludeHtml ? GetResultsFilePath(workItemInfo, options, "html") : null; + var htmlResultsFilePath = options.IncludeHtml ? GetResultsFilePath(workItemInfo, options, "html") : null; var processResultList = new List(); - ProcessInfo? procDumpProcessInfo = null; // NOTE: xUnit doesn't always create the log directory Directory.CreateDirectory(resultsDir!); // Define environment variables for processes started via ProcessRunner. var environmentVariables = new Dictionary(); - Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables); if (isRetry && File.Exists(resultsFilePath)) { @@ -175,7 +166,7 @@ private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, O var start = DateTime.UtcNow; var dotnetProcessInfo = ProcessRunner.CreateProcess( ProcessRunner.CreateProcessStartInfo( - Options.DotnetFilePath, + options.DotnetFilePath, commandLineArguments, displayWindow: false, captureOutput: true, @@ -189,12 +180,6 @@ private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, O Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName} with code {xunitProcessResult.ExitCode}"); processResultList.Add(xunitProcessResult); - if (procDumpProcessInfo != null) - { - var procDumpProcessResult = await procDumpProcessInfo.Value.Result; - Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName} with code {procDumpProcessResult.ExitCode}"); - processResultList.Add(procDumpProcessResult); - } if (xunitProcessResult.ExitCode != 0) { @@ -221,7 +206,7 @@ private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, O } } - Logger.Log($"Command line {workItemInfo.DisplayName} completed in {span.TotalSeconds} seconds: {Options.DotnetFilePath} {commandLineArguments}"); + Logger.Log($"Command line {workItemInfo.DisplayName} completed in {span.TotalSeconds} seconds: {options.DotnetFilePath} {commandLineArguments}"); var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? ""; var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? ""; @@ -241,7 +226,7 @@ private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, O } catch (Exception ex) { - throw new Exception($"Unable to run {workItemInfo.DisplayName} with {Options.DotnetFilePath}. {ex}"); + throw new Exception($"Unable to run {workItemInfo.DisplayName} with {options.DotnetFilePath}. {ex}"); } } } diff --git a/src/Tools/Source/RunTests/Program.cs b/src/Tools/Source/RunTests/Program.cs index f44ff66caeed2..78c5c1a9218d1 100644 --- a/src/Tools/Source/RunTests/Program.cs +++ b/src/Tools/Source/RunTests/Program.cs @@ -129,7 +129,7 @@ private static async Task RunAsync(Options options, TimeSpan timeout, Cance private static async Task RunAsync(Options options, CancellationToken cancellationToken) { - var testExecutor = CreateTestExecutor(options); + var testExecutor = new ProcessTestExecutor(); var testRunner = new TestRunner(options, testExecutor); var start = DateTime.Now; var workItems = await GetWorkItemsAsync(options, cancellationToken); @@ -256,16 +256,16 @@ async Task DumpProcess(Process targetProcess, string procDumpExeFilePath, string } } - if (options.CollectDumps && GetProcDumpInfo(options) is { } procDumpInfo) + if (options.CollectDumps && !string.IsNullOrEmpty(options.ProcDumpFilePath)) { ConsoleUtil.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes"); var counter = 0; foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName)) { - var dumpDir = procDumpInfo.DumpDirectory; + var dumpDir = options.LogFilesDirectory; var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp"); - await DumpProcess(proc, procDumpInfo.ProcDumpFilePath, dumpFilePath); + await DumpProcess(proc, options.ProcDumpFilePath, dumpFilePath); counter++; } } @@ -273,16 +273,6 @@ async Task DumpProcess(Process targetProcess, string procDumpExeFilePath, string WriteLogFile(options); } - private static ProcDumpInfo? GetProcDumpInfo(Options options) - { - if (!string.IsNullOrEmpty(options.ProcDumpFilePath)) - { - return new ProcDumpInfo(options.ProcDumpFilePath, options.LogFilesDirectory); - } - - return null; - } - private static async Task> GetWorkItemsAsync(Options options, CancellationToken cancellationToken) { var scheduler = new AssemblyScheduler(options); @@ -390,19 +380,6 @@ private static void DisplayResults(Display display, ImmutableArray t } } - private static ProcessTestExecutor CreateTestExecutor(Options options) - { - var testExecutionOptions = new TestExecutionOptions( - dotnetFilePath: options.DotnetFilePath, - procDumpInfo: options.CollectDumps ? GetProcDumpInfo(options) : null, - testResultsDirectory: options.TestResultsDirectory, - testFilter: options.TestFilter, - includeHtml: options.IncludeHtml, - retry: options.Retry, - collectDumps: options.CollectDumps); - return new ProcessTestExecutor(testExecutionOptions); - } - /// /// Checks the total size of dump file and removes files exceeding a limit. /// From 04290aead60381d850e9e2e5bc64b6dc1e682f8f Mon Sep 17 00:00:00 2001 From: David Barbet Date: Sat, 23 Jul 2022 04:06:53 -0700 Subject: [PATCH 08/26] Use test platform APIs for running tests --- Roslyn.sln | 7 + src/Tools/PrepareTests/MinimizeUtil.cs | 1 + .../Source/RunTests/AssemblyScheduler.cs | 26 ++-- src/Tools/Source/RunTests/ITestExecutor.cs | 76 +--------- src/Tools/Source/RunTests/ProcessRunner.cs | 2 - .../Source/RunTests/ProcessTestExecutor.cs | 141 +++++++++--------- src/Tools/Source/RunTests/Program.cs | 30 ++-- src/Tools/Source/RunTests/RunTests.csproj | 5 +- .../Source/RunTests/TestHistoryManager.cs | 6 +- src/Tools/Source/RunTests/TestRunner.cs | 75 +++++----- src/Tools/TestExecutor/Program.cs | 82 ++++++++++ src/Tools/TestExecutor/RunTestsHandler.cs | 45 ++++++ src/Tools/TestExecutor/TestExecutor.csproj | 17 +++ src/Tools/TestExecutor/TestPlatformWrapper.cs | 99 ++++++++++++ src/Tools/TestExecutor/TestResultInfo.cs | 34 +++++ 15 files changed, 426 insertions(+), 220 deletions(-) create mode 100644 src/Tools/TestExecutor/Program.cs create mode 100644 src/Tools/TestExecutor/RunTestsHandler.cs create mode 100644 src/Tools/TestExecutor/TestExecutor.csproj create mode 100644 src/Tools/TestExecutor/TestPlatformWrapper.cs create mode 100644 src/Tools/TestExecutor/TestResultInfo.cs diff --git a/Roslyn.sln b/Roslyn.sln index 7b3342cc3ad33..b3b383ea26f18 100644 --- a/Roslyn.sln +++ b/Roslyn.sln @@ -511,6 +511,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Net.Compilers.Too EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests", "src\Compilers\CSharp\Test\EndToEnd\Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests.csproj", "{C247414A-8946-4BAB-BE1F-C82B90C63EF6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestExecutor", "src\Tools\TestExecutor\TestExecutor.csproj", "{822B8815-3E3B-4EB6-945C-1E34E729D0F7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1241,6 +1243,10 @@ Global {C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Debug|Any CPU.Build.0 = Debug|Any CPU {C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Release|Any CPU.ActiveCfg = Release|Any CPU {C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Release|Any CPU.Build.0 = Release|Any CPU + {822B8815-3E3B-4EB6-945C-1E34E729D0F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {822B8815-3E3B-4EB6-945C-1E34E729D0F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {822B8815-3E3B-4EB6-945C-1E34E729D0F7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {822B8815-3E3B-4EB6-945C-1E34E729D0F7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1473,6 +1479,7 @@ Global {6131713D-DFB4-49B5-8010-50071FED3E85} = {C52D8057-43AF-40E6-A01B-6CDBB7301985} {A9A8ADE5-F123-4109-9FA4-4B92F1657043} = {C52D8057-43AF-40E6-A01B-6CDBB7301985} {C247414A-8946-4BAB-BE1F-C82B90C63EF6} = {32A48625-F0AD-419D-828B-A50BDABA38EA} + {822B8815-3E3B-4EB6-945C-1E34E729D0F7} = {FD0FAF5F-1DED-485C-99FA-84B97F3A8EEC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {604E6B91-7BC0-4126-AE07-D4D2FEFC3D29} diff --git a/src/Tools/PrepareTests/MinimizeUtil.cs b/src/Tools/PrepareTests/MinimizeUtil.cs index 19295e8f2d485..6a7576caaae60 100644 --- a/src/Tools/PrepareTests/MinimizeUtil.cs +++ b/src/Tools/PrepareTests/MinimizeUtil.cs @@ -58,6 +58,7 @@ Dictionary> initialWalk() var artifactsDir = Path.Combine(sourceDirectory, "artifacts/bin"); directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "*.UnitTests")); directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "RunTests")); + directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "TestExecutor")); var idToFilePathMap = directories.AsParallel() .SelectMany(unitDirPath => walkDirectory(unitDirPath, sourceDirectory, destinationDirectory)) diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index 8f9756bd52f5a..8fae26e684eca 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -14,6 +14,7 @@ using System.Reflection.Metadata; using System.Reflection.Metadata.Ecma335; using System.Reflection.PortableExecutable; +using System.Security.Cryptography.X509Certificates; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -88,23 +89,24 @@ public async Task> ScheduleAsync(ImmutableArray assembly, GetTypeInfoList); - - ConsoleUtil.WriteLine($"Found {orderedTypeInfos.Values.SelectMany(t => t).SelectMany(t => t.Tests).Count()} tests to run in {orderedTypeInfos.Keys.Count()} assemblies"); - - if (_options.Sequential) + if (_options.Sequential || !_options.UseHelix) { - Logger.Log("Building sequential work items"); + Logger.Log("Building work items with one assembly each."); // return individual work items per assembly that contain all the tests in that assembly. - return CreateWorkItemsForFullAssemblies(orderedTypeInfos); + return CreateWorkItemsForFullAssemblies(assemblies); } + var orderedTypeInfos = assemblies.ToImmutableSortedDictionary(assembly => assembly, GetTypeInfoList); + + ConsoleUtil.WriteLine($"Found {orderedTypeInfos.Values.SelectMany(t => t).SelectMany(t => t.Tests).Count()} tests to run in {orderedTypeInfos.Keys.Count()} assemblies"); + // Retrieve test runtimes from azure devops historical data. var testHistory = await TestHistoryManager.GetTestHistoryAsync(cancellationToken); if (testHistory.IsEmpty) { // We didn't have any test history from azure devops, just partition by assembly. - return CreateWorkItemsForFullAssemblies(orderedTypeInfos); + ConsoleUtil.WriteLine($"##[warning]Could not look up test history - building a single work item per assembly"); + return CreateWorkItemsForFullAssemblies(assemblies); } // Now for our current set of test methods we got from the assemblies we built, match them to tests from our test run history @@ -120,13 +122,13 @@ public async Task> ScheduleAsync(ImmutableArray CreateWorkItemsForFullAssemblies(ImmutableSortedDictionary> orderedTypeInfos) + static ImmutableArray CreateWorkItemsForFullAssemblies(ImmutableArray assemblies) { var workItems = new List(); var partitionIndex = 0; - foreach (var orderedTypeInfo in orderedTypeInfos) + foreach (var assembly in assemblies) { - var currentWorkItem = ImmutableSortedDictionary>.Empty.Add(orderedTypeInfo.Key, ImmutableArray.Create((ITestFilter)new AssemblyTestFilter(orderedTypeInfo.Value))); + var currentWorkItem = ImmutableSortedDictionary>.Empty.Add(assembly, ImmutableArray.Empty); workItems.Add(new WorkItemInfo(currentWorkItem, partitionIndex++)); } @@ -327,6 +329,8 @@ private static ImmutableArray GetTypeInfoList(AssemblyInfo assemblyInf { var assemblyDirectory = Path.GetDirectoryName(assemblyInfo.AssemblyPath); var testListPath = Path.Combine(assemblyDirectory!, "testlist.json"); + Contract.Assert(File.Exists(testListPath)); + var deserialized = JsonSerializer.Deserialize>(File.ReadAllText(testListPath)); Contract.Assert(deserialized != null); diff --git a/src/Tools/Source/RunTests/ITestExecutor.cs b/src/Tools/Source/RunTests/ITestExecutor.cs index 8860870bd9cc4..17b970269e64e 100644 --- a/src/Tools/Source/RunTests/ITestExecutor.cs +++ b/src/Tools/Source/RunTests/ITestExecutor.cs @@ -3,93 +3,31 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Collections.Immutable; +using TestExecutor; namespace RunTests { - internal readonly struct TestExecutionOptions - { - internal string DotnetFilePath { get; } - internal string TestResultsDirectory { get; } - internal string? TestFilter { get; } - internal bool IncludeHtml { get; } - internal bool Retry { get; } - internal bool CollectDumps { get; } - - internal TestExecutionOptions(string dotnetFilePath, string testResultsDirectory, string? testFilter, bool includeHtml, bool retry, bool collectDumps) - { - DotnetFilePath = dotnetFilePath; - TestResultsDirectory = testResultsDirectory; - TestFilter = testFilter; - IncludeHtml = includeHtml; - Retry = retry; - CollectDumps = collectDumps; - } - } - - /// - /// The actual results from running the xunit tests. - /// - /// - /// The difference between and is the former - /// is specifically for the actual test execution results while the latter can contain extra metadata - /// about the results. For example whether it was cached, or had diagnostic, output, etc ... - /// - internal readonly struct TestResultInfo - { - internal int ExitCode { get; } - internal TimeSpan Elapsed { get; } - internal string StandardOutput { get; } - internal string ErrorOutput { get; } - - /// - /// Path to the XML results file. - /// - internal string? ResultsFilePath { get; } - - /// - /// Path to the HTML results file if HTML output is enabled, otherwise, . - /// - internal string? HtmlResultsFilePath { get; } - - internal TestResultInfo(int exitCode, string? resultsFilePath, string? htmlResultsFilePath, TimeSpan elapsed, string standardOutput, string errorOutput) - { - ExitCode = exitCode; - ResultsFilePath = resultsFilePath; - HtmlResultsFilePath = htmlResultsFilePath; - Elapsed = elapsed; - StandardOutput = standardOutput; - ErrorOutput = errorOutput; - } - } - internal readonly struct TestResult { internal TestResultInfo TestResultInfo { get; } internal WorkItemInfo WorkItemInfo { get; } - internal string CommandLine { get; } - internal string? Diagnostics { get; } - - /// - /// Collection of processes the runner explicitly ran to get the result. - /// - internal ImmutableArray ProcessResults { get; } - internal string DisplayName => WorkItemInfo.DisplayName; internal bool Succeeded => ExitCode == 0; internal int ExitCode => TestResultInfo.ExitCode; internal TimeSpan Elapsed => TestResultInfo.Elapsed; internal string StandardOutput => TestResultInfo.StandardOutput; internal string ErrorOutput => TestResultInfo.ErrorOutput; - internal string? ResultsDisplayFilePath => TestResultInfo.HtmlResultsFilePath ?? TestResultInfo.ResultsFilePath; + internal string? ResultsDisplayFilePath { get; } + internal string? HtmlResultsFilePath { get; } - internal TestResult(WorkItemInfo workItemInfo, TestResultInfo testResultInfo, string commandLine, ImmutableArray processResults = default, string? diagnostics = null) + internal TestResult(WorkItemInfo workItemInfo, TestResultInfo testResultInfo, string? resultsFilePath, string? htmlResultsFilePath) { WorkItemInfo = workItemInfo; TestResultInfo = testResultInfo; - CommandLine = commandLine; - ProcessResults = processResults.IsDefault ? ImmutableArray.Empty : processResults; - Diagnostics = diagnostics; + ResultsDisplayFilePath = resultsFilePath; + HtmlResultsFilePath = htmlResultsFilePath; } } } diff --git a/src/Tools/Source/RunTests/ProcessRunner.cs b/src/Tools/Source/RunTests/ProcessRunner.cs index e110c4ebc7e11..67c736ba296bd 100644 --- a/src/Tools/Source/RunTests/ProcessRunner.cs +++ b/src/Tools/Source/RunTests/ProcessRunner.cs @@ -106,7 +106,6 @@ public static ProcessInfo CreateProcess( }; process.Exited += (s, e) => - { // We must call WaitForExit to make sure we've received all OutputDataReceived/ErrorDataReceived calls // or else we'll be returning a list we're still modifying. For paranoia, we'll start a task here rather // than enter right back into the Process type and start a wait which isn't guaranteed to be safe. @@ -120,7 +119,6 @@ public static ProcessInfo CreateProcess( new ReadOnlyCollection(errorLines)); tcs.TrySetResult(result); }, cancellationToken); - }; var registration = cancellationToken.Register(() => { diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index 3075ec0130bde..0b40624cd319d 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -13,38 +13,27 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using System.Xml; using System.Xml.Linq; using System.Xml.XPath; +using TestExecutor; namespace RunTests { internal sealed class ProcessTestExecutor { - public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuotes, Options options) + public static string GetFilterString(WorkItemInfo workItemInfo, Options options) { - // http://www.gnu.org/software/bash/manual/html_node/Single-Quotes.html - // Single quotes are needed in bash to avoid the need to escape characters such as backtick (`) which are found in metadata names. - // Batch scripts don't need to worry about escaping backticks, but they don't support single quoted strings, so we have to use double quotes. - // We also need double quotes when building an arguments string for Process.Start in .NET Core so that splitting/unquoting works as expected. - var sep = useSingleQuotes ? "'" : @""""; - var builder = new StringBuilder(); - builder.Append($@"test"); - - var escapedAssemblyPaths = workItem.Filters.Keys.Select(assembly => $"{sep}{assembly.AssemblyPath}{sep}"); - builder.Append($@" {string.Join(" ", escapedAssemblyPaths)}"); - - var filters = workItem.Filters.Values.SelectMany(filter => filter).Where(filter => !string.IsNullOrEmpty(filter.GetFilterString())).ToImmutableArray(); - if (filters.Length > 0 || !string.IsNullOrWhiteSpace(options.TestFilter)) + var filters = workItemInfo.Filters.Values.SelectMany(filter => filter); + if (filters.Any() || !string.IsNullOrWhiteSpace(options.TestFilter)) { - builder.Append($@" --filter {sep}"); var any = false; foreach (var filter in filters) { MaybeAddSeparator(); builder.Append(filter.GetFilterString()); } - builder.Append(sep); if (options.TestFilter is object) { @@ -63,21 +52,12 @@ void MaybeAddSeparator(char separator = '|') } } - builder.Append($@" --arch {options.Architecture}"); - builder.Append($@" --logger {sep}xunit;LogFilePath={GetResultsFilePath(workItem, options, "xml")}{sep}"); - - if (options.IncludeHtml) - { - builder.AppendFormat($@" --logger {sep}html;LogFileName={GetResultsFilePath(workItem, options, "html")}{sep}"); - } + return builder.ToString(); + } - if (!options.CollectDumps) - { - // The 'CollectDumps' option uses operating system features to collect dumps when a process crashes. We - // only enable the test executor blame feature in remaining cases, as the latter relies on ProcDump and - // interferes with automatic crash dump collection on Windows. - builder.Append(" --blame-crash"); - } + public static string GetRunSettings(WorkItemInfo workItemInfo, Options options) + { + var blameCrashSetting = !options.CollectDumps ? "" : string.Empty; // The 25 minute timeout in integration tests accounts for the fact that VSIX deployment and/or experimental hive reset and // configuration can take significant time (seems to vary from ~10 seconds to ~15 minutes), and the blame @@ -86,20 +66,56 @@ void MaybeAddSeparator(char separator = '|') // // Helix timeout is 15 minutes as helix jobs fully timeout in 30minutes. So in order to capture dumps we need the timeout // to be 2x shorter than the expected test run time (15min) in case only the last test hangs. - var timeout = options.UseHelix ? "15minutes" : "25minutes"; - - builder.Append($" --blame-hang-dump-type full --blame-hang-timeout {timeout}"); - - return builder.ToString(); + var timeout = options.UseHelix ? TimeSpan.FromMinutes(15) : TimeSpan.FromMinutes(25); + + var xunitResultsFilePath = GetResultsFilePath(workItemInfo, options, "xml"); + var htmlResultsFilePath = GetResultsFilePath(workItemInfo, options, "html"); + var includeHtmlLogger = options.IncludeHtml ? "True" : "False"; + + var runsettingsDocument = $@" + + + + 0 + {options.Architecture} + + + + + + + {blameCrashSetting} + + + + + + + + + + {xunitResultsFilePath} + + + + + {htmlResultsFilePath} + + + + +"; + + return runsettingsDocument; } - private string GetResultsFilePath(WorkItemInfo workItemInfo, Options options, string suffix = "xml") + private static string GetResultsFilePath(WorkItemInfo workItemInfo, Options options, string suffix = "xml") { var fileName = $"WorkItem_{workItemInfo.PartitionIndex}_{options.Architecture}_test_results.{suffix}"; return Path.Combine(options.TestResultsDirectory, fileName); } - public async Task RunTestAsync(WorkItemInfo workItemInfo, Options options, CancellationToken cancellationToken) + public static async Task RunTestAsync(WorkItemInfo workItemInfo, Options options, CancellationToken cancellationToken) { var result = await RunTestAsyncInternal(workItemInfo, options, isRetry: false, cancellationToken); @@ -118,15 +134,16 @@ static bool HasBuiltInRetry(WorkItemInfo workItemInfo) } } - private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, Options options, bool isRetry, CancellationToken cancellationToken) + private static async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, Options options, bool isRetry, CancellationToken cancellationToken) { try { - var commandLineArguments = GetCommandLineArguments(workItemInfo, useSingleQuotes: false, options); + var runsettingsContents = GetRunSettings(workItemInfo, options); + var filterString = GetFilterString(workItemInfo, options); + var resultsFilePath = GetResultsFilePath(workItemInfo, options); var resultsDir = Path.GetDirectoryName(resultsFilePath); var htmlResultsFilePath = options.IncludeHtml ? GetResultsFilePath(workItemInfo, options, "html") : null; - var processResultList = new List(); // NOTE: xUnit doesn't always create the log directory Directory.CreateDirectory(resultsDir!); @@ -163,25 +180,13 @@ private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, O // an empty log just in case, so our runner will still fail. File.Create(resultsFilePath).Close(); - var start = DateTime.UtcNow; - var dotnetProcessInfo = ProcessRunner.CreateProcess( - ProcessRunner.CreateProcessStartInfo( - options.DotnetFilePath, - commandLineArguments, - displayWindow: false, - captureOutput: true, - environmentVariables: environmentVariables), - lowPriority: false, - cancellationToken: cancellationToken); - Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName}"); - - var xunitProcessResult = await dotnetProcessInfo.Result; - var span = DateTime.UtcNow - start; - - Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName} with code {xunitProcessResult.ExitCode}"); - processResultList.Add(xunitProcessResult); - - if (xunitProcessResult.ExitCode != 0) + Logger.Log($"Running tests in work item {workItemInfo.DisplayName}"); + var testResult = TestPlatformWrapper.RunTests(workItemInfo.Filters.Keys.Select(a => a.AssemblyPath), filterString, runsettingsContents, options.DotnetFilePath); + + Logger.Log($"Test run finished with code {testResult.ExitCode}, ran to completion: {testResult.RanToCompletion}"); + Logger.Log($"Took {testResult.Elapsed}"); + + if (testResult.ExitCode != 0) { // On occasion we get a non-0 output but no actual data in the result file. The could happen // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance). @@ -206,23 +211,11 @@ private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, O } } - Logger.Log($"Command line {workItemInfo.DisplayName} completed in {span.TotalSeconds} seconds: {options.DotnetFilePath} {commandLineArguments}"); - var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? ""; - var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? ""; - - var testResultInfo = new TestResultInfo( - exitCode: xunitProcessResult.ExitCode, - resultsFilePath: resultsFilePath, - htmlResultsFilePath: htmlResultsFilePath, - elapsed: span, - standardOutput: standardOutput, - errorOutput: errorOutput); - return new TestResult( workItemInfo, - testResultInfo, - commandLineArguments, - processResults: ImmutableArray.CreateRange(processResultList)); + testResult, + resultsFilePath, + htmlResultsFilePath); } catch (Exception ex) { diff --git a/src/Tools/Source/RunTests/Program.cs b/src/Tools/Source/RunTests/Program.cs index 78c5c1a9218d1..1a2b7b947a190 100644 --- a/src/Tools/Source/RunTests/Program.cs +++ b/src/Tools/Source/RunTests/Program.cs @@ -129,8 +129,7 @@ private static async Task RunAsync(Options options, TimeSpan timeout, Cance private static async Task RunAsync(Options options, CancellationToken cancellationToken) { - var testExecutor = new ProcessTestExecutor(); - var testRunner = new TestRunner(options, testExecutor); + var testRunner = new TestRunner(options); var start = DateTime.Now; var workItems = await GetWorkItemsAsync(options, cancellationToken); if (workItems.Length == 0) @@ -150,7 +149,7 @@ private static async Task RunAsync(Options options, CancellationToken cance ConsoleUtil.WriteLine($"Test execution time: {elapsed}"); - LogProcessResultDetails(result.ProcessResults); + LogTestResultDetails(result.TestResults); WriteLogFile(options); DisplayResults(options.Display, result.TestResults); @@ -164,30 +163,21 @@ private static async Task RunAsync(Options options, CancellationToken cance return ExitSuccess; } - private static void LogProcessResultDetails(ImmutableArray processResults) + private static void LogTestResultDetails(ImmutableArray testResults) { Logger.Log("### Begin logging executed process details"); - foreach (var processResult in processResults) + foreach (var testResult in testResults) { - var process = processResult.Process; - var startInfo = process.StartInfo; - Logger.Log($"### Begin {process.Id}"); - Logger.Log($"### {startInfo.FileName} {startInfo.Arguments}"); - Logger.Log($"### Exit code {process.ExitCode}"); + Logger.Log($"### Start {testResult.DisplayName}"); + Logger.Log($"### Exit code {testResult.ExitCode}"); Logger.Log("### Standard Output"); - foreach (var line in processResult.OutputLines) - { - Logger.Log(line); - } + Logger.Log(testResult.StandardOutput); Logger.Log("### Standard Error"); - foreach (var line in processResult.ErrorLines) - { - Logger.Log(line); - } - Logger.Log($"### End {process.Id}"); + Logger.Log(testResult.ErrorOutput); + Logger.Log($"### End {testResult.DisplayName}"); } - Logger.Log("End logging executed process details"); + Logger.Log("End logging test result details"); } private static void WriteLogFile(Options options) diff --git a/src/Tools/Source/RunTests/RunTests.csproj b/src/Tools/Source/RunTests/RunTests.csproj index aefa3a8aa7d11..4d25aaad63561 100644 --- a/src/Tools/Source/RunTests/RunTests.csproj +++ b/src/Tools/Source/RunTests/RunTests.csproj @@ -16,6 +16,9 @@ - + + + + \ No newline at end of file diff --git a/src/Tools/Source/RunTests/TestHistoryManager.cs b/src/Tools/Source/RunTests/TestHistoryManager.cs index eb52dda439a39..539b54ab64150 100644 --- a/src/Tools/Source/RunTests/TestHistoryManager.cs +++ b/src/Tools/Source/RunTests/TestHistoryManager.cs @@ -26,7 +26,7 @@ internal class TestHistoryManager private const int MaxTestsReturnedPerRequest = 10000; /// - /// The pipeline id for roslyn-ci, see https://dev.azure.com/dnceng/public/_build?definitionId=15&_a=summary + /// The pipeline id for roslyn-ci, see https://dev.azure.com/dnceng/public/_build?definitionId=15 /// private const int RoslynCiBuildDefinitionId = 15; @@ -61,7 +61,7 @@ public static async Task> GetTestHistoryAs if (lastSuccessfulBuild == null) { // If this is a new branch we may not have any historical data for it. - ConsoleUtil.WriteLine($"##[warning]Unable to get the last successful build for definition {RoslynCiBuildDefinitionId} and branch {adoBranchName}"); + ConsoleUtil.WriteLine($"Unable to get the last successful build for definition {RoslynCiBuildDefinitionId} and branch {adoBranchName}"); return ImmutableDictionary.Empty; } @@ -76,7 +76,7 @@ public static async Task> GetTestHistoryAs if (runForThisStage == null) { // If this is a new stage, historical runs will not have any data for it. - ConsoleUtil.WriteLine($"##[warning]Unable to get a run with name {stageName} from build {lastSuccessfulBuild.Url}."); + ConsoleUtil.WriteLine($"Unable to get a run with name {stageName} from build {lastSuccessfulBuild.Url}."); return ImmutableDictionary.Empty; } diff --git a/src/Tools/Source/RunTests/TestRunner.cs b/src/Tools/Source/RunTests/TestRunner.cs index 861dadd2ab24b..eb3d29202db8a 100644 --- a/src/Tools/Source/RunTests/TestRunner.cs +++ b/src/Tools/Source/RunTests/TestRunner.cs @@ -12,10 +12,13 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Microsoft.TeamFoundation.Common; using Mono.Options; using Newtonsoft.Json; +using TestExecutor; namespace RunTests { @@ -23,24 +26,20 @@ internal struct RunAllResult { internal bool Succeeded { get; } internal ImmutableArray TestResults { get; } - internal ImmutableArray ProcessResults { get; } - internal RunAllResult(bool succeeded, ImmutableArray testResults, ImmutableArray processResults) + internal RunAllResult(bool succeeded, ImmutableArray testResults) { Succeeded = succeeded; TestResults = testResults; - ProcessResults = processResults; } } internal sealed class TestRunner { - private readonly ProcessTestExecutor _testExecutor; private readonly Options _options; - internal TestRunner(Options options, ProcessTestExecutor testExecutor) + internal TestRunner(Options options) { - _testExecutor = testExecutor; _options = options; } @@ -132,7 +131,15 @@ internal async Task RunAllOnHelixAsync(ImmutableArray.Empty, ImmutableArray.Create(result)); + if (result.ExitCode != 0) + { + foreach (var line in result.ErrorLines) + { + ConsoleUtil.WriteLine(ConsoleColor.Red, line); + } + } + + return new RunAllResult(result.ExitCode == 0, ImmutableArray.Empty); static string getGlobalJsonPath() { @@ -208,11 +215,21 @@ string MakeHelixWorkItemProject(WorkItemInfo workItemInfo) AddRehydrateTestFoldersCommand(command, workItemInfo, isUnix); - var commandLineArguments = _testExecutor.GetCommandLineArguments(workItemInfo, isUnix, options); - // XML escape the arguments as the commands are output into the helix project xml file. - commandLineArguments = SecurityElement.Escape(commandLineArguments); - var dotnetTestCommand = $"dotnet {commandLineArguments}"; - command.AppendLine(dotnetTestCommand); + // TODO - need different names per work item, otherwise stuff just gets overwritten + var runsettings = ProcessTestExecutor.GetRunSettings(workItemInfo, options); + var runsettingsFileName = "roslyn.runsettings"; + var runsettingsPath = Path.Combine(payloadDirectory, runsettingsFileName); + File.WriteAllText(runsettingsPath, runsettings); + + var filterString = ProcessTestExecutor.GetFilterString(workItemInfo, options); + var testExecutorInfoFileName = "assembly_filter.txt"; + var testExecutorInfo = new TestExecutorInfo(workItemInfo.Filters.Keys.Select(a => a.AssemblyPath).ToList(), filterString); + var testExecutorInfoPath = Path.Combine(payloadDirectory, testExecutorInfoFileName); + File.WriteAllText(testExecutorInfoPath, JsonConvert.SerializeObject(testExecutorInfo)); + + var testExecutorPath = Path.Combine("TestExecutor", options.Configuration, "net6.0", "TestExecutor.dll"); + var dotnetPath = isUnix ? "${DOTNET_ROOT}/dotnet" : "%DOTNET_ROOT%\\dotnet"; + command.AppendLine($"dotnet exec \"{testExecutorPath}\" --dotnetPath \"{dotnetPath}\" --workItemInfoPath \"{testExecutorInfoFileName}\" --runsettingsPath {runsettingsFileName}"); // We want to collect any dumps during the post command step here; these commands are ran after the // return value of the main command is captured; a Helix Job is considered to fail if the main command returns a @@ -281,13 +298,7 @@ internal async Task RunAllAsync(ImmutableArray workI } else { - foreach (var result in testResult.ProcessResults) - { - foreach (var line in result.ErrorLines) - { - ConsoleUtil.WriteLine(ConsoleColor.Red, line); - } - } + ConsoleUtil.WriteLine(ConsoleColor.Red, testResult.ErrorOutput); } } @@ -309,7 +320,7 @@ internal async Task RunAllAsync(ImmutableArray workI while (running.Count < max && waiting.Count > 0) { - var task = _testExecutor.RunTestAsync(waiting.Pop(), _options, cancellationToken); + var task = ProcessTestExecutor.RunTestAsync(waiting.Pop(), _options, cancellationToken); running.Add(task); } @@ -330,13 +341,7 @@ internal async Task RunAllAsync(ImmutableArray workI Print(completed); - var processResults = ImmutableArray.CreateBuilder(); - foreach (var c in completed) - { - processResults.AddRange(c.ProcessResults); - } - - return new RunAllResult((failures == 0), completed.ToImmutableArray(), processResults.ToImmutable()); + return new RunAllResult((failures == 0), completed.ToImmutableArray()); } private void Print(List testResults) @@ -357,19 +362,11 @@ private void Print(List testResults) line.Append($"{testResult.DisplayName,-75}"); line.Append($" {(testResult.Succeeded ? "PASSED" : "FAILED")}"); line.Append($" {testResult.Elapsed}"); - line.Append($" {(!string.IsNullOrEmpty(testResult.Diagnostics) ? "?" : "")}"); var message = line.ToString(); ConsoleUtil.WriteLine(color, message); } ConsoleUtil.WriteLine("================"); - - // Print diagnostics out last so they are cleanly visible at the end of the test summary - ConsoleUtil.WriteLine("Extra run diagnostics for logging, did not impact run results"); - foreach (var testResult in testResults.Where(x => !string.IsNullOrEmpty(x.Diagnostics))) - { - ConsoleUtil.WriteLine(testResult.Diagnostics!); - } } private void PrintFailedTestResult(TestResult testResult) @@ -378,10 +375,9 @@ private void PrintFailedTestResult(TestResult testResult) var outputLogPath = Path.Combine(_options.LogFilesDirectory, $"xUnitFailure-{testResult.DisplayName}.log"); ConsoleUtil.WriteLine($"Errors {testResult.DisplayName}"); - ConsoleUtil.WriteLine(testResult.ErrorOutput); + ConsoleUtil.WriteLine(testResult.ErrorOutput ?? string.Empty); // TODO: Put this in the log and take it off the ConsoleUtil output to keep it simple? - ConsoleUtil.WriteLine($"Command: {testResult.CommandLine}"); ConsoleUtil.WriteLine($"xUnit output log: {outputLogPath}"); File.WriteAllText(outputLogPath, testResult.StandardOutput ?? ""); @@ -397,10 +393,9 @@ private void PrintFailedTestResult(TestResult testResult) } // If the results are html, use Process.Start to open in the browser. - var htmlResultsFilePath = testResult.TestResultInfo.HtmlResultsFilePath; - if (!string.IsNullOrEmpty(htmlResultsFilePath)) + if (Path.GetExtension(testResult.ResultsDisplayFilePath) == "html") { - var startInfo = new ProcessStartInfo() { FileName = htmlResultsFilePath, UseShellExecute = true }; + var startInfo = new ProcessStartInfo() { FileName = testResult.ResultsDisplayFilePath, UseShellExecute = true }; Process.Start(startInfo); } } diff --git a/src/Tools/TestExecutor/Program.cs b/src/Tools/TestExecutor/Program.cs new file mode 100644 index 0000000000000..cbea2f63a7f7f --- /dev/null +++ b/src/Tools/TestExecutor/Program.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Mono.Options; + +namespace TestExecutor; + +public record struct TestExecutorInfo(List Assemblies, string FilterString); + +internal static class Program +{ + internal const int ExitFailure = 1; + internal const int ExitSuccess = 0; + + public static int Main(string[] args) + { + Console.WriteLine($"Running with args: {string.Join(" ", args)}"); + + string? dotnetPath = null; + string? workItemInfoPath = null; + string? runsettingsPath = null; + + var options = new OptionSet() + { + { "dotnetPath=", "Path to the dotnet executable", (string s) => dotnetPath = s }, + { "workItemInfoPath=", "File path containing the test assemblies and filters to execute", (string s) => workItemInfoPath = s }, + { "runsettingsPath=", ".runsettings file path", (string s) => runsettingsPath = s }, + }; + options.Parse(args); + + if (dotnetPath is null) + { + Console.Error.WriteLine("--dotnetPath argument must be provided"); + return ExitFailure; + } + + if (workItemInfoPath is null) + { + Console.Error.WriteLine("--workItemInfoPath argument must be provided"); + return ExitFailure; + } + + if (!File.Exists(workItemInfoPath)) + { + Console.Error.WriteLine("--workItemInfoPath must exist"); + return ExitFailure; + } + + if (runsettingsPath is null) + { + Console.Error.WriteLine("--runsettingsPath argument must be provided"); + return ExitFailure; + } + + if (!File.Exists(runsettingsPath)) + { + Console.Error.WriteLine("--runsettingsPath must exist"); + return ExitFailure; + } + + var runsettings = File.ReadAllText(runsettingsPath); + var testExecutorInfo = JsonSerializer.Deserialize(File.ReadAllText(workItemInfoPath)); + var result = TestPlatformWrapper.RunTests(testExecutorInfo.Assemblies, testExecutorInfo.FilterString, runsettings, dotnetPath); + Console.WriteLine($"Test run finished with code {result.ExitCode}, ran to completion: {result.RanToCompletion}"); + Console.WriteLine("### Standard Output ####"); + Console.WriteLine(result.StandardOutput); + Console.WriteLine("### Error Output ####"); + Console.WriteLine(result.ErrorOutput); + return result.ExitCode; + } +} diff --git a/src/Tools/TestExecutor/RunTestsHandler.cs b/src/Tools/TestExecutor/RunTestsHandler.cs new file mode 100644 index 0000000000000..1749141ca92f1 --- /dev/null +++ b/src/Tools/TestExecutor/RunTestsHandler.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics.Contracts; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace TestExecutor; + +internal class RunTestsHandler : ITestRunEventsHandler +{ + public Action, ICollection>? HandleCompletion { get; init; } + + public Action? HandleLog { get; init; } + + public void HandleLogMessage(TestMessageLevel level, string message) + { + Contract.Assert(HandleLog != null); + HandleLog(level, message); + } + + public void HandleRawMessage(string rawMessage) + { + } + + public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection runContextAttachments, ICollection executorUris) + { + Contract.Assert(HandleCompletion != null); + HandleCompletion(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris); + } + + public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs) + { + } + + public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessStartInfo) + { + // This is not used in vstest 17.2+; + return -1; + } +} diff --git a/src/Tools/TestExecutor/TestExecutor.csproj b/src/Tools/TestExecutor/TestExecutor.csproj new file mode 100644 index 0000000000000..a693bf10f7b8d --- /dev/null +++ b/src/Tools/TestExecutor/TestExecutor.csproj @@ -0,0 +1,17 @@ + + + + + Exe + net6.0 + false + false + false + + + + + + + + diff --git a/src/Tools/TestExecutor/TestPlatformWrapper.cs b/src/Tools/TestExecutor/TestPlatformWrapper.cs new file mode 100644 index 0000000000000..833ee48539747 --- /dev/null +++ b/src/Tools/TestExecutor/TestPlatformWrapper.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Contracts; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace TestExecutor; +public class TestPlatformWrapper +{ + public static TestResultInfo RunTests(IEnumerable assemblies, string filterString, string runsettingsContents, string dotnetPath) + { + var vsTestConsole = Directory.EnumerateFiles(Path.Combine(Path.GetDirectoryName(dotnetPath)!, "sdk"), "vstest.console.dll", SearchOption.AllDirectories).Last(); + var vstestConsoleWrapper = new VsTestConsoleWrapper(vsTestConsole, new ConsoleParameters + { + LogFilePath = Path.Combine(AppContext.BaseDirectory, "logs", "test_discovery_logs.txt"), + TraceLevel = TraceLevel.Error, + }); + + var exitResult = Program.ExitFailure; + TestRunCompleteEventArgs? testRunComplete = null; + var outputLines = new List(); + var errorLines = new List(); + + var runHandler = new RunTestsHandler + { + HandleCompletion = (testRunCompleteArgs, testRunChanged, _, _2) => + { + WriteSummary(testRunCompleteArgs); + + testRunComplete = testRunCompleteArgs; + if (DidPass(testRunCompleteArgs)) + { + exitResult = Program.ExitSuccess; + } + }, + HandleLog = (severity, logMessage) => + { + if (severity == TestMessageLevel.Error) + { + errorLines.Add(logMessage); + } + + outputLines.Add(logMessage); + } + }; + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + vstestConsoleWrapper.RunTests(assemblies, runsettingsContents, new TestPlatformOptions + { + TestCaseFilter = filterString, + }, runHandler); + stopwatch.Stop(); + + var ranToCompletion = testRunComplete != null ? !testRunComplete.IsCanceled && !testRunComplete.IsAborted : false; + return new TestResultInfo(exitResult, stopwatch.Elapsed, string.Join(Environment.NewLine, outputLines), string.Join(Environment.NewLine, errorLines), ranToCompletion); + } + + public record struct TestExecutionInfo(List Assemblies, List Tests); + + private static void WriteSummary(TestRunCompleteEventArgs testRunComplete) + { + if (testRunComplete.TestRunStatistics != null) + { + testRunComplete.TestRunStatistics.Stats.TryGetValue(TestOutcome.Passed, out var passed); + testRunComplete.TestRunStatistics.Stats.TryGetValue(TestOutcome.Failed, out var failed); + testRunComplete.TestRunStatistics.Stats.TryGetValue(TestOutcome.Skipped, out var skipped); + Console.WriteLine($"Summary: Passed: {passed}, Failed: {failed}, Skipped {skipped}"); + } + + if (testRunComplete.Error != null) + { + Console.WriteLine($"ERROR: {testRunComplete.Error}"); + } + } + + private static bool DidPass(TestRunCompleteEventArgs testRunComplete) + { + if (testRunComplete.Error != null || testRunComplete.IsAborted || testRunComplete.IsCanceled || testRunComplete.TestRunStatistics == null) + { + return false; + } + + testRunComplete.TestRunStatistics.Stats.TryGetValue(TestOutcome.Failed, out var failed); + return failed == 0; + } + +} diff --git a/src/Tools/TestExecutor/TestResultInfo.cs b/src/Tools/TestExecutor/TestResultInfo.cs new file mode 100644 index 0000000000000..0aedfece44641 --- /dev/null +++ b/src/Tools/TestExecutor/TestResultInfo.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +namespace TestExecutor; + +/// +/// The actual results from running the xunit tests. +/// +/// +/// The difference between and is the former +/// is specifically for the actual test execution results while the latter can contain extra metadata +/// about the results. For example whether it was cached, or had diagnostic, output, etc ... +/// +public readonly struct TestResultInfo +{ + public int ExitCode { get; } + public TimeSpan Elapsed { get; } + public string StandardOutput { get; } + public string ErrorOutput { get; } + public bool RanToCompletion { get; } + + internal TestResultInfo(int exitCode, TimeSpan elapsed, string standardOutput, string errorOutput, bool ranToCompletion) + { + ExitCode = exitCode; + Elapsed = elapsed; + StandardOutput = standardOutput; + ErrorOutput = errorOutput; + RanToCompletion = ranToCompletion; + } +} From f4df81b6f6c6de53689e72cc9d9158945d19f3fc Mon Sep 17 00:00:00 2001 From: David Barbet Date: Mon, 25 Jul 2022 10:48:33 -0700 Subject: [PATCH 09/26] Revert "Use test platform APIs for running tests" This reverts commit 04290aead60381d850e9e2e5bc64b6dc1e682f8f. --- Roslyn.sln | 7 - src/Tools/PrepareTests/MinimizeUtil.cs | 1 - .../Source/RunTests/AssemblyScheduler.cs | 26 ++-- src/Tools/Source/RunTests/ITestExecutor.cs | 76 +++++++++- src/Tools/Source/RunTests/ProcessRunner.cs | 2 + .../Source/RunTests/ProcessTestExecutor.cs | 141 +++++++++--------- src/Tools/Source/RunTests/Program.cs | 30 ++-- src/Tools/Source/RunTests/RunTests.csproj | 5 +- .../Source/RunTests/TestHistoryManager.cs | 6 +- src/Tools/Source/RunTests/TestRunner.cs | 75 +++++----- src/Tools/TestExecutor/Program.cs | 82 ---------- src/Tools/TestExecutor/RunTestsHandler.cs | 45 ------ src/Tools/TestExecutor/TestExecutor.csproj | 17 --- src/Tools/TestExecutor/TestPlatformWrapper.cs | 99 ------------ src/Tools/TestExecutor/TestResultInfo.cs | 34 ----- 15 files changed, 220 insertions(+), 426 deletions(-) delete mode 100644 src/Tools/TestExecutor/Program.cs delete mode 100644 src/Tools/TestExecutor/RunTestsHandler.cs delete mode 100644 src/Tools/TestExecutor/TestExecutor.csproj delete mode 100644 src/Tools/TestExecutor/TestPlatformWrapper.cs delete mode 100644 src/Tools/TestExecutor/TestResultInfo.cs diff --git a/Roslyn.sln b/Roslyn.sln index b3b383ea26f18..7b3342cc3ad33 100644 --- a/Roslyn.sln +++ b/Roslyn.sln @@ -511,8 +511,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Net.Compilers.Too EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests", "src\Compilers\CSharp\Test\EndToEnd\Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests.csproj", "{C247414A-8946-4BAB-BE1F-C82B90C63EF6}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestExecutor", "src\Tools\TestExecutor\TestExecutor.csproj", "{822B8815-3E3B-4EB6-945C-1E34E729D0F7}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1243,10 +1241,6 @@ Global {C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Debug|Any CPU.Build.0 = Debug|Any CPU {C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Release|Any CPU.ActiveCfg = Release|Any CPU {C247414A-8946-4BAB-BE1F-C82B90C63EF6}.Release|Any CPU.Build.0 = Release|Any CPU - {822B8815-3E3B-4EB6-945C-1E34E729D0F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {822B8815-3E3B-4EB6-945C-1E34E729D0F7}.Debug|Any CPU.Build.0 = Debug|Any CPU - {822B8815-3E3B-4EB6-945C-1E34E729D0F7}.Release|Any CPU.ActiveCfg = Release|Any CPU - {822B8815-3E3B-4EB6-945C-1E34E729D0F7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1479,7 +1473,6 @@ Global {6131713D-DFB4-49B5-8010-50071FED3E85} = {C52D8057-43AF-40E6-A01B-6CDBB7301985} {A9A8ADE5-F123-4109-9FA4-4B92F1657043} = {C52D8057-43AF-40E6-A01B-6CDBB7301985} {C247414A-8946-4BAB-BE1F-C82B90C63EF6} = {32A48625-F0AD-419D-828B-A50BDABA38EA} - {822B8815-3E3B-4EB6-945C-1E34E729D0F7} = {FD0FAF5F-1DED-485C-99FA-84B97F3A8EEC} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {604E6B91-7BC0-4126-AE07-D4D2FEFC3D29} diff --git a/src/Tools/PrepareTests/MinimizeUtil.cs b/src/Tools/PrepareTests/MinimizeUtil.cs index 6a7576caaae60..19295e8f2d485 100644 --- a/src/Tools/PrepareTests/MinimizeUtil.cs +++ b/src/Tools/PrepareTests/MinimizeUtil.cs @@ -58,7 +58,6 @@ Dictionary> initialWalk() var artifactsDir = Path.Combine(sourceDirectory, "artifacts/bin"); directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "*.UnitTests")); directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "RunTests")); - directories = directories.Concat(Directory.EnumerateDirectories(artifactsDir, "TestExecutor")); var idToFilePathMap = directories.AsParallel() .SelectMany(unitDirPath => walkDirectory(unitDirPath, sourceDirectory, destinationDirectory)) diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index 8fae26e684eca..8f9756bd52f5a 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -14,7 +14,6 @@ using System.Reflection.Metadata; using System.Reflection.Metadata.Ecma335; using System.Reflection.PortableExecutable; -using System.Security.Cryptography.X509Certificates; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -89,24 +88,23 @@ public async Task> ScheduleAsync(ImmutableArray assembly, GetTypeInfoList); ConsoleUtil.WriteLine($"Found {orderedTypeInfos.Values.SelectMany(t => t).SelectMany(t => t.Tests).Count()} tests to run in {orderedTypeInfos.Keys.Count()} assemblies"); + if (_options.Sequential) + { + Logger.Log("Building sequential work items"); + // return individual work items per assembly that contain all the tests in that assembly. + return CreateWorkItemsForFullAssemblies(orderedTypeInfos); + } + // Retrieve test runtimes from azure devops historical data. var testHistory = await TestHistoryManager.GetTestHistoryAsync(cancellationToken); if (testHistory.IsEmpty) { // We didn't have any test history from azure devops, just partition by assembly. - ConsoleUtil.WriteLine($"##[warning]Could not look up test history - building a single work item per assembly"); - return CreateWorkItemsForFullAssemblies(assemblies); + return CreateWorkItemsForFullAssemblies(orderedTypeInfos); } // Now for our current set of test methods we got from the assemblies we built, match them to tests from our test run history @@ -122,13 +120,13 @@ public async Task> ScheduleAsync(ImmutableArray CreateWorkItemsForFullAssemblies(ImmutableArray assemblies) + static ImmutableArray CreateWorkItemsForFullAssemblies(ImmutableSortedDictionary> orderedTypeInfos) { var workItems = new List(); var partitionIndex = 0; - foreach (var assembly in assemblies) + foreach (var orderedTypeInfo in orderedTypeInfos) { - var currentWorkItem = ImmutableSortedDictionary>.Empty.Add(assembly, ImmutableArray.Empty); + var currentWorkItem = ImmutableSortedDictionary>.Empty.Add(orderedTypeInfo.Key, ImmutableArray.Create((ITestFilter)new AssemblyTestFilter(orderedTypeInfo.Value))); workItems.Add(new WorkItemInfo(currentWorkItem, partitionIndex++)); } @@ -329,8 +327,6 @@ private static ImmutableArray GetTypeInfoList(AssemblyInfo assemblyInf { var assemblyDirectory = Path.GetDirectoryName(assemblyInfo.AssemblyPath); var testListPath = Path.Combine(assemblyDirectory!, "testlist.json"); - Contract.Assert(File.Exists(testListPath)); - var deserialized = JsonSerializer.Deserialize>(File.ReadAllText(testListPath)); Contract.Assert(deserialized != null); diff --git a/src/Tools/Source/RunTests/ITestExecutor.cs b/src/Tools/Source/RunTests/ITestExecutor.cs index 17b970269e64e..8860870bd9cc4 100644 --- a/src/Tools/Source/RunTests/ITestExecutor.cs +++ b/src/Tools/Source/RunTests/ITestExecutor.cs @@ -3,31 +3,93 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Generic; using System.Collections.Immutable; -using TestExecutor; namespace RunTests { + internal readonly struct TestExecutionOptions + { + internal string DotnetFilePath { get; } + internal string TestResultsDirectory { get; } + internal string? TestFilter { get; } + internal bool IncludeHtml { get; } + internal bool Retry { get; } + internal bool CollectDumps { get; } + + internal TestExecutionOptions(string dotnetFilePath, string testResultsDirectory, string? testFilter, bool includeHtml, bool retry, bool collectDumps) + { + DotnetFilePath = dotnetFilePath; + TestResultsDirectory = testResultsDirectory; + TestFilter = testFilter; + IncludeHtml = includeHtml; + Retry = retry; + CollectDumps = collectDumps; + } + } + + /// + /// The actual results from running the xunit tests. + /// + /// + /// The difference between and is the former + /// is specifically for the actual test execution results while the latter can contain extra metadata + /// about the results. For example whether it was cached, or had diagnostic, output, etc ... + /// + internal readonly struct TestResultInfo + { + internal int ExitCode { get; } + internal TimeSpan Elapsed { get; } + internal string StandardOutput { get; } + internal string ErrorOutput { get; } + + /// + /// Path to the XML results file. + /// + internal string? ResultsFilePath { get; } + + /// + /// Path to the HTML results file if HTML output is enabled, otherwise, . + /// + internal string? HtmlResultsFilePath { get; } + + internal TestResultInfo(int exitCode, string? resultsFilePath, string? htmlResultsFilePath, TimeSpan elapsed, string standardOutput, string errorOutput) + { + ExitCode = exitCode; + ResultsFilePath = resultsFilePath; + HtmlResultsFilePath = htmlResultsFilePath; + Elapsed = elapsed; + StandardOutput = standardOutput; + ErrorOutput = errorOutput; + } + } + internal readonly struct TestResult { internal TestResultInfo TestResultInfo { get; } internal WorkItemInfo WorkItemInfo { get; } + internal string CommandLine { get; } + internal string? Diagnostics { get; } + + /// + /// Collection of processes the runner explicitly ran to get the result. + /// + internal ImmutableArray ProcessResults { get; } + internal string DisplayName => WorkItemInfo.DisplayName; internal bool Succeeded => ExitCode == 0; internal int ExitCode => TestResultInfo.ExitCode; internal TimeSpan Elapsed => TestResultInfo.Elapsed; internal string StandardOutput => TestResultInfo.StandardOutput; internal string ErrorOutput => TestResultInfo.ErrorOutput; - internal string? ResultsDisplayFilePath { get; } - internal string? HtmlResultsFilePath { get; } + internal string? ResultsDisplayFilePath => TestResultInfo.HtmlResultsFilePath ?? TestResultInfo.ResultsFilePath; - internal TestResult(WorkItemInfo workItemInfo, TestResultInfo testResultInfo, string? resultsFilePath, string? htmlResultsFilePath) + internal TestResult(WorkItemInfo workItemInfo, TestResultInfo testResultInfo, string commandLine, ImmutableArray processResults = default, string? diagnostics = null) { WorkItemInfo = workItemInfo; TestResultInfo = testResultInfo; - ResultsDisplayFilePath = resultsFilePath; - HtmlResultsFilePath = htmlResultsFilePath; + CommandLine = commandLine; + ProcessResults = processResults.IsDefault ? ImmutableArray.Empty : processResults; + Diagnostics = diagnostics; } } } diff --git a/src/Tools/Source/RunTests/ProcessRunner.cs b/src/Tools/Source/RunTests/ProcessRunner.cs index 67c736ba296bd..e110c4ebc7e11 100644 --- a/src/Tools/Source/RunTests/ProcessRunner.cs +++ b/src/Tools/Source/RunTests/ProcessRunner.cs @@ -106,6 +106,7 @@ public static ProcessInfo CreateProcess( }; process.Exited += (s, e) => + { // We must call WaitForExit to make sure we've received all OutputDataReceived/ErrorDataReceived calls // or else we'll be returning a list we're still modifying. For paranoia, we'll start a task here rather // than enter right back into the Process type and start a wait which isn't guaranteed to be safe. @@ -119,6 +120,7 @@ public static ProcessInfo CreateProcess( new ReadOnlyCollection(errorLines)); tcs.TrySetResult(result); }, cancellationToken); + }; var registration = cancellationToken.Register(() => { diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index 0b40624cd319d..3075ec0130bde 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -13,27 +13,38 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using System.Xml; using System.Xml.Linq; using System.Xml.XPath; -using TestExecutor; namespace RunTests { internal sealed class ProcessTestExecutor { - public static string GetFilterString(WorkItemInfo workItemInfo, Options options) + public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuotes, Options options) { + // http://www.gnu.org/software/bash/manual/html_node/Single-Quotes.html + // Single quotes are needed in bash to avoid the need to escape characters such as backtick (`) which are found in metadata names. + // Batch scripts don't need to worry about escaping backticks, but they don't support single quoted strings, so we have to use double quotes. + // We also need double quotes when building an arguments string for Process.Start in .NET Core so that splitting/unquoting works as expected. + var sep = useSingleQuotes ? "'" : @""""; + var builder = new StringBuilder(); - var filters = workItemInfo.Filters.Values.SelectMany(filter => filter); - if (filters.Any() || !string.IsNullOrWhiteSpace(options.TestFilter)) + builder.Append($@"test"); + + var escapedAssemblyPaths = workItem.Filters.Keys.Select(assembly => $"{sep}{assembly.AssemblyPath}{sep}"); + builder.Append($@" {string.Join(" ", escapedAssemblyPaths)}"); + + var filters = workItem.Filters.Values.SelectMany(filter => filter).Where(filter => !string.IsNullOrEmpty(filter.GetFilterString())).ToImmutableArray(); + if (filters.Length > 0 || !string.IsNullOrWhiteSpace(options.TestFilter)) { + builder.Append($@" --filter {sep}"); var any = false; foreach (var filter in filters) { MaybeAddSeparator(); builder.Append(filter.GetFilterString()); } + builder.Append(sep); if (options.TestFilter is object) { @@ -52,12 +63,21 @@ void MaybeAddSeparator(char separator = '|') } } - return builder.ToString(); - } + builder.Append($@" --arch {options.Architecture}"); + builder.Append($@" --logger {sep}xunit;LogFilePath={GetResultsFilePath(workItem, options, "xml")}{sep}"); - public static string GetRunSettings(WorkItemInfo workItemInfo, Options options) - { - var blameCrashSetting = !options.CollectDumps ? "" : string.Empty; + if (options.IncludeHtml) + { + builder.AppendFormat($@" --logger {sep}html;LogFileName={GetResultsFilePath(workItem, options, "html")}{sep}"); + } + + if (!options.CollectDumps) + { + // The 'CollectDumps' option uses operating system features to collect dumps when a process crashes. We + // only enable the test executor blame feature in remaining cases, as the latter relies on ProcDump and + // interferes with automatic crash dump collection on Windows. + builder.Append(" --blame-crash"); + } // The 25 minute timeout in integration tests accounts for the fact that VSIX deployment and/or experimental hive reset and // configuration can take significant time (seems to vary from ~10 seconds to ~15 minutes), and the blame @@ -66,56 +86,20 @@ public static string GetRunSettings(WorkItemInfo workItemInfo, Options options) // // Helix timeout is 15 minutes as helix jobs fully timeout in 30minutes. So in order to capture dumps we need the timeout // to be 2x shorter than the expected test run time (15min) in case only the last test hangs. - var timeout = options.UseHelix ? TimeSpan.FromMinutes(15) : TimeSpan.FromMinutes(25); - - var xunitResultsFilePath = GetResultsFilePath(workItemInfo, options, "xml"); - var htmlResultsFilePath = GetResultsFilePath(workItemInfo, options, "html"); - var includeHtmlLogger = options.IncludeHtml ? "True" : "False"; - - var runsettingsDocument = $@" - - - - 0 - {options.Architecture} - - - - - - - {blameCrashSetting} - - - - - - - - - - {xunitResultsFilePath} - - - - - {htmlResultsFilePath} - - - - -"; - - return runsettingsDocument; + var timeout = options.UseHelix ? "15minutes" : "25minutes"; + + builder.Append($" --blame-hang-dump-type full --blame-hang-timeout {timeout}"); + + return builder.ToString(); } - private static string GetResultsFilePath(WorkItemInfo workItemInfo, Options options, string suffix = "xml") + private string GetResultsFilePath(WorkItemInfo workItemInfo, Options options, string suffix = "xml") { var fileName = $"WorkItem_{workItemInfo.PartitionIndex}_{options.Architecture}_test_results.{suffix}"; return Path.Combine(options.TestResultsDirectory, fileName); } - public static async Task RunTestAsync(WorkItemInfo workItemInfo, Options options, CancellationToken cancellationToken) + public async Task RunTestAsync(WorkItemInfo workItemInfo, Options options, CancellationToken cancellationToken) { var result = await RunTestAsyncInternal(workItemInfo, options, isRetry: false, cancellationToken); @@ -134,16 +118,15 @@ static bool HasBuiltInRetry(WorkItemInfo workItemInfo) } } - private static async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, Options options, bool isRetry, CancellationToken cancellationToken) + private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, Options options, bool isRetry, CancellationToken cancellationToken) { try { - var runsettingsContents = GetRunSettings(workItemInfo, options); - var filterString = GetFilterString(workItemInfo, options); - + var commandLineArguments = GetCommandLineArguments(workItemInfo, useSingleQuotes: false, options); var resultsFilePath = GetResultsFilePath(workItemInfo, options); var resultsDir = Path.GetDirectoryName(resultsFilePath); var htmlResultsFilePath = options.IncludeHtml ? GetResultsFilePath(workItemInfo, options, "html") : null; + var processResultList = new List(); // NOTE: xUnit doesn't always create the log directory Directory.CreateDirectory(resultsDir!); @@ -180,13 +163,25 @@ private static async Task RunTestAsyncInternal(WorkItemInfo workItem // an empty log just in case, so our runner will still fail. File.Create(resultsFilePath).Close(); - Logger.Log($"Running tests in work item {workItemInfo.DisplayName}"); - var testResult = TestPlatformWrapper.RunTests(workItemInfo.Filters.Keys.Select(a => a.AssemblyPath), filterString, runsettingsContents, options.DotnetFilePath); - - Logger.Log($"Test run finished with code {testResult.ExitCode}, ran to completion: {testResult.RanToCompletion}"); - Logger.Log($"Took {testResult.Elapsed}"); - - if (testResult.ExitCode != 0) + var start = DateTime.UtcNow; + var dotnetProcessInfo = ProcessRunner.CreateProcess( + ProcessRunner.CreateProcessStartInfo( + options.DotnetFilePath, + commandLineArguments, + displayWindow: false, + captureOutput: true, + environmentVariables: environmentVariables), + lowPriority: false, + cancellationToken: cancellationToken); + Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName}"); + + var xunitProcessResult = await dotnetProcessInfo.Result; + var span = DateTime.UtcNow - start; + + Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {workItemInfo.DisplayName} with code {xunitProcessResult.ExitCode}"); + processResultList.Add(xunitProcessResult); + + if (xunitProcessResult.ExitCode != 0) { // On occasion we get a non-0 output but no actual data in the result file. The could happen // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance). @@ -211,11 +206,23 @@ private static async Task RunTestAsyncInternal(WorkItemInfo workItem } } + Logger.Log($"Command line {workItemInfo.DisplayName} completed in {span.TotalSeconds} seconds: {options.DotnetFilePath} {commandLineArguments}"); + var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? ""; + var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? ""; + + var testResultInfo = new TestResultInfo( + exitCode: xunitProcessResult.ExitCode, + resultsFilePath: resultsFilePath, + htmlResultsFilePath: htmlResultsFilePath, + elapsed: span, + standardOutput: standardOutput, + errorOutput: errorOutput); + return new TestResult( workItemInfo, - testResult, - resultsFilePath, - htmlResultsFilePath); + testResultInfo, + commandLineArguments, + processResults: ImmutableArray.CreateRange(processResultList)); } catch (Exception ex) { diff --git a/src/Tools/Source/RunTests/Program.cs b/src/Tools/Source/RunTests/Program.cs index 1a2b7b947a190..78c5c1a9218d1 100644 --- a/src/Tools/Source/RunTests/Program.cs +++ b/src/Tools/Source/RunTests/Program.cs @@ -129,7 +129,8 @@ private static async Task RunAsync(Options options, TimeSpan timeout, Cance private static async Task RunAsync(Options options, CancellationToken cancellationToken) { - var testRunner = new TestRunner(options); + var testExecutor = new ProcessTestExecutor(); + var testRunner = new TestRunner(options, testExecutor); var start = DateTime.Now; var workItems = await GetWorkItemsAsync(options, cancellationToken); if (workItems.Length == 0) @@ -149,7 +150,7 @@ private static async Task RunAsync(Options options, CancellationToken cance ConsoleUtil.WriteLine($"Test execution time: {elapsed}"); - LogTestResultDetails(result.TestResults); + LogProcessResultDetails(result.ProcessResults); WriteLogFile(options); DisplayResults(options.Display, result.TestResults); @@ -163,21 +164,30 @@ private static async Task RunAsync(Options options, CancellationToken cance return ExitSuccess; } - private static void LogTestResultDetails(ImmutableArray testResults) + private static void LogProcessResultDetails(ImmutableArray processResults) { Logger.Log("### Begin logging executed process details"); - foreach (var testResult in testResults) + foreach (var processResult in processResults) { - Logger.Log($"### Start {testResult.DisplayName}"); - Logger.Log($"### Exit code {testResult.ExitCode}"); + var process = processResult.Process; + var startInfo = process.StartInfo; + Logger.Log($"### Begin {process.Id}"); + Logger.Log($"### {startInfo.FileName} {startInfo.Arguments}"); + Logger.Log($"### Exit code {process.ExitCode}"); Logger.Log("### Standard Output"); - Logger.Log(testResult.StandardOutput); + foreach (var line in processResult.OutputLines) + { + Logger.Log(line); + } Logger.Log("### Standard Error"); - Logger.Log(testResult.ErrorOutput); - Logger.Log($"### End {testResult.DisplayName}"); + foreach (var line in processResult.ErrorLines) + { + Logger.Log(line); + } + Logger.Log($"### End {process.Id}"); } - Logger.Log("End logging test result details"); + Logger.Log("End logging executed process details"); } private static void WriteLogFile(Options options) diff --git a/src/Tools/Source/RunTests/RunTests.csproj b/src/Tools/Source/RunTests/RunTests.csproj index 4d25aaad63561..aefa3a8aa7d11 100644 --- a/src/Tools/Source/RunTests/RunTests.csproj +++ b/src/Tools/Source/RunTests/RunTests.csproj @@ -16,9 +16,6 @@ - - - - + \ No newline at end of file diff --git a/src/Tools/Source/RunTests/TestHistoryManager.cs b/src/Tools/Source/RunTests/TestHistoryManager.cs index 539b54ab64150..eb52dda439a39 100644 --- a/src/Tools/Source/RunTests/TestHistoryManager.cs +++ b/src/Tools/Source/RunTests/TestHistoryManager.cs @@ -26,7 +26,7 @@ internal class TestHistoryManager private const int MaxTestsReturnedPerRequest = 10000; /// - /// The pipeline id for roslyn-ci, see https://dev.azure.com/dnceng/public/_build?definitionId=15 + /// The pipeline id for roslyn-ci, see https://dev.azure.com/dnceng/public/_build?definitionId=15&_a=summary /// private const int RoslynCiBuildDefinitionId = 15; @@ -61,7 +61,7 @@ public static async Task> GetTestHistoryAs if (lastSuccessfulBuild == null) { // If this is a new branch we may not have any historical data for it. - ConsoleUtil.WriteLine($"Unable to get the last successful build for definition {RoslynCiBuildDefinitionId} and branch {adoBranchName}"); + ConsoleUtil.WriteLine($"##[warning]Unable to get the last successful build for definition {RoslynCiBuildDefinitionId} and branch {adoBranchName}"); return ImmutableDictionary.Empty; } @@ -76,7 +76,7 @@ public static async Task> GetTestHistoryAs if (runForThisStage == null) { // If this is a new stage, historical runs will not have any data for it. - ConsoleUtil.WriteLine($"Unable to get a run with name {stageName} from build {lastSuccessfulBuild.Url}."); + ConsoleUtil.WriteLine($"##[warning]Unable to get a run with name {stageName} from build {lastSuccessfulBuild.Url}."); return ImmutableDictionary.Empty; } diff --git a/src/Tools/Source/RunTests/TestRunner.cs b/src/Tools/Source/RunTests/TestRunner.cs index eb3d29202db8a..861dadd2ab24b 100644 --- a/src/Tools/Source/RunTests/TestRunner.cs +++ b/src/Tools/Source/RunTests/TestRunner.cs @@ -12,13 +12,10 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Microsoft.TeamFoundation.Common; using Mono.Options; using Newtonsoft.Json; -using TestExecutor; namespace RunTests { @@ -26,20 +23,24 @@ internal struct RunAllResult { internal bool Succeeded { get; } internal ImmutableArray TestResults { get; } + internal ImmutableArray ProcessResults { get; } - internal RunAllResult(bool succeeded, ImmutableArray testResults) + internal RunAllResult(bool succeeded, ImmutableArray testResults, ImmutableArray processResults) { Succeeded = succeeded; TestResults = testResults; + ProcessResults = processResults; } } internal sealed class TestRunner { + private readonly ProcessTestExecutor _testExecutor; private readonly Options _options; - internal TestRunner(Options options) + internal TestRunner(Options options, ProcessTestExecutor testExecutor) { + _testExecutor = testExecutor; _options = options; } @@ -131,15 +132,7 @@ internal async Task RunAllOnHelixAsync(ImmutableArray.Empty); + return new RunAllResult(result.ExitCode == 0, ImmutableArray.Empty, ImmutableArray.Create(result)); static string getGlobalJsonPath() { @@ -215,21 +208,11 @@ string MakeHelixWorkItemProject(WorkItemInfo workItemInfo) AddRehydrateTestFoldersCommand(command, workItemInfo, isUnix); - // TODO - need different names per work item, otherwise stuff just gets overwritten - var runsettings = ProcessTestExecutor.GetRunSettings(workItemInfo, options); - var runsettingsFileName = "roslyn.runsettings"; - var runsettingsPath = Path.Combine(payloadDirectory, runsettingsFileName); - File.WriteAllText(runsettingsPath, runsettings); - - var filterString = ProcessTestExecutor.GetFilterString(workItemInfo, options); - var testExecutorInfoFileName = "assembly_filter.txt"; - var testExecutorInfo = new TestExecutorInfo(workItemInfo.Filters.Keys.Select(a => a.AssemblyPath).ToList(), filterString); - var testExecutorInfoPath = Path.Combine(payloadDirectory, testExecutorInfoFileName); - File.WriteAllText(testExecutorInfoPath, JsonConvert.SerializeObject(testExecutorInfo)); - - var testExecutorPath = Path.Combine("TestExecutor", options.Configuration, "net6.0", "TestExecutor.dll"); - var dotnetPath = isUnix ? "${DOTNET_ROOT}/dotnet" : "%DOTNET_ROOT%\\dotnet"; - command.AppendLine($"dotnet exec \"{testExecutorPath}\" --dotnetPath \"{dotnetPath}\" --workItemInfoPath \"{testExecutorInfoFileName}\" --runsettingsPath {runsettingsFileName}"); + var commandLineArguments = _testExecutor.GetCommandLineArguments(workItemInfo, isUnix, options); + // XML escape the arguments as the commands are output into the helix project xml file. + commandLineArguments = SecurityElement.Escape(commandLineArguments); + var dotnetTestCommand = $"dotnet {commandLineArguments}"; + command.AppendLine(dotnetTestCommand); // We want to collect any dumps during the post command step here; these commands are ran after the // return value of the main command is captured; a Helix Job is considered to fail if the main command returns a @@ -298,7 +281,13 @@ internal async Task RunAllAsync(ImmutableArray workI } else { - ConsoleUtil.WriteLine(ConsoleColor.Red, testResult.ErrorOutput); + foreach (var result in testResult.ProcessResults) + { + foreach (var line in result.ErrorLines) + { + ConsoleUtil.WriteLine(ConsoleColor.Red, line); + } + } } } @@ -320,7 +309,7 @@ internal async Task RunAllAsync(ImmutableArray workI while (running.Count < max && waiting.Count > 0) { - var task = ProcessTestExecutor.RunTestAsync(waiting.Pop(), _options, cancellationToken); + var task = _testExecutor.RunTestAsync(waiting.Pop(), _options, cancellationToken); running.Add(task); } @@ -341,7 +330,13 @@ internal async Task RunAllAsync(ImmutableArray workI Print(completed); - return new RunAllResult((failures == 0), completed.ToImmutableArray()); + var processResults = ImmutableArray.CreateBuilder(); + foreach (var c in completed) + { + processResults.AddRange(c.ProcessResults); + } + + return new RunAllResult((failures == 0), completed.ToImmutableArray(), processResults.ToImmutable()); } private void Print(List testResults) @@ -362,11 +357,19 @@ private void Print(List testResults) line.Append($"{testResult.DisplayName,-75}"); line.Append($" {(testResult.Succeeded ? "PASSED" : "FAILED")}"); line.Append($" {testResult.Elapsed}"); + line.Append($" {(!string.IsNullOrEmpty(testResult.Diagnostics) ? "?" : "")}"); var message = line.ToString(); ConsoleUtil.WriteLine(color, message); } ConsoleUtil.WriteLine("================"); + + // Print diagnostics out last so they are cleanly visible at the end of the test summary + ConsoleUtil.WriteLine("Extra run diagnostics for logging, did not impact run results"); + foreach (var testResult in testResults.Where(x => !string.IsNullOrEmpty(x.Diagnostics))) + { + ConsoleUtil.WriteLine(testResult.Diagnostics!); + } } private void PrintFailedTestResult(TestResult testResult) @@ -375,9 +378,10 @@ private void PrintFailedTestResult(TestResult testResult) var outputLogPath = Path.Combine(_options.LogFilesDirectory, $"xUnitFailure-{testResult.DisplayName}.log"); ConsoleUtil.WriteLine($"Errors {testResult.DisplayName}"); - ConsoleUtil.WriteLine(testResult.ErrorOutput ?? string.Empty); + ConsoleUtil.WriteLine(testResult.ErrorOutput); // TODO: Put this in the log and take it off the ConsoleUtil output to keep it simple? + ConsoleUtil.WriteLine($"Command: {testResult.CommandLine}"); ConsoleUtil.WriteLine($"xUnit output log: {outputLogPath}"); File.WriteAllText(outputLogPath, testResult.StandardOutput ?? ""); @@ -393,9 +397,10 @@ private void PrintFailedTestResult(TestResult testResult) } // If the results are html, use Process.Start to open in the browser. - if (Path.GetExtension(testResult.ResultsDisplayFilePath) == "html") + var htmlResultsFilePath = testResult.TestResultInfo.HtmlResultsFilePath; + if (!string.IsNullOrEmpty(htmlResultsFilePath)) { - var startInfo = new ProcessStartInfo() { FileName = testResult.ResultsDisplayFilePath, UseShellExecute = true }; + var startInfo = new ProcessStartInfo() { FileName = htmlResultsFilePath, UseShellExecute = true }; Process.Start(startInfo); } } diff --git a/src/Tools/TestExecutor/Program.cs b/src/Tools/TestExecutor/Program.cs deleted file mode 100644 index cbea2f63a7f7f..0000000000000 --- a/src/Tools/TestExecutor/Program.cs +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Mono.Options; - -namespace TestExecutor; - -public record struct TestExecutorInfo(List Assemblies, string FilterString); - -internal static class Program -{ - internal const int ExitFailure = 1; - internal const int ExitSuccess = 0; - - public static int Main(string[] args) - { - Console.WriteLine($"Running with args: {string.Join(" ", args)}"); - - string? dotnetPath = null; - string? workItemInfoPath = null; - string? runsettingsPath = null; - - var options = new OptionSet() - { - { "dotnetPath=", "Path to the dotnet executable", (string s) => dotnetPath = s }, - { "workItemInfoPath=", "File path containing the test assemblies and filters to execute", (string s) => workItemInfoPath = s }, - { "runsettingsPath=", ".runsettings file path", (string s) => runsettingsPath = s }, - }; - options.Parse(args); - - if (dotnetPath is null) - { - Console.Error.WriteLine("--dotnetPath argument must be provided"); - return ExitFailure; - } - - if (workItemInfoPath is null) - { - Console.Error.WriteLine("--workItemInfoPath argument must be provided"); - return ExitFailure; - } - - if (!File.Exists(workItemInfoPath)) - { - Console.Error.WriteLine("--workItemInfoPath must exist"); - return ExitFailure; - } - - if (runsettingsPath is null) - { - Console.Error.WriteLine("--runsettingsPath argument must be provided"); - return ExitFailure; - } - - if (!File.Exists(runsettingsPath)) - { - Console.Error.WriteLine("--runsettingsPath must exist"); - return ExitFailure; - } - - var runsettings = File.ReadAllText(runsettingsPath); - var testExecutorInfo = JsonSerializer.Deserialize(File.ReadAllText(workItemInfoPath)); - var result = TestPlatformWrapper.RunTests(testExecutorInfo.Assemblies, testExecutorInfo.FilterString, runsettings, dotnetPath); - Console.WriteLine($"Test run finished with code {result.ExitCode}, ran to completion: {result.RanToCompletion}"); - Console.WriteLine("### Standard Output ####"); - Console.WriteLine(result.StandardOutput); - Console.WriteLine("### Error Output ####"); - Console.WriteLine(result.ErrorOutput); - return result.ExitCode; - } -} diff --git a/src/Tools/TestExecutor/RunTestsHandler.cs b/src/Tools/TestExecutor/RunTestsHandler.cs deleted file mode 100644 index 1749141ca92f1..0000000000000 --- a/src/Tools/TestExecutor/RunTestsHandler.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Diagnostics.Contracts; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; - -namespace TestExecutor; - -internal class RunTestsHandler : ITestRunEventsHandler -{ - public Action, ICollection>? HandleCompletion { get; init; } - - public Action? HandleLog { get; init; } - - public void HandleLogMessage(TestMessageLevel level, string message) - { - Contract.Assert(HandleLog != null); - HandleLog(level, message); - } - - public void HandleRawMessage(string rawMessage) - { - } - - public void HandleTestRunComplete(TestRunCompleteEventArgs testRunCompleteArgs, TestRunChangedEventArgs lastChunkArgs, ICollection runContextAttachments, ICollection executorUris) - { - Contract.Assert(HandleCompletion != null); - HandleCompletion(testRunCompleteArgs, lastChunkArgs, runContextAttachments, executorUris); - } - - public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs) - { - } - - public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessStartInfo) - { - // This is not used in vstest 17.2+; - return -1; - } -} diff --git a/src/Tools/TestExecutor/TestExecutor.csproj b/src/Tools/TestExecutor/TestExecutor.csproj deleted file mode 100644 index a693bf10f7b8d..0000000000000 --- a/src/Tools/TestExecutor/TestExecutor.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - - Exe - net6.0 - false - false - false - - - - - - - - diff --git a/src/Tools/TestExecutor/TestPlatformWrapper.cs b/src/Tools/TestExecutor/TestPlatformWrapper.cs deleted file mode 100644 index 833ee48539747..0000000000000 --- a/src/Tools/TestExecutor/TestPlatformWrapper.cs +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.Contracts; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; - -namespace TestExecutor; -public class TestPlatformWrapper -{ - public static TestResultInfo RunTests(IEnumerable assemblies, string filterString, string runsettingsContents, string dotnetPath) - { - var vsTestConsole = Directory.EnumerateFiles(Path.Combine(Path.GetDirectoryName(dotnetPath)!, "sdk"), "vstest.console.dll", SearchOption.AllDirectories).Last(); - var vstestConsoleWrapper = new VsTestConsoleWrapper(vsTestConsole, new ConsoleParameters - { - LogFilePath = Path.Combine(AppContext.BaseDirectory, "logs", "test_discovery_logs.txt"), - TraceLevel = TraceLevel.Error, - }); - - var exitResult = Program.ExitFailure; - TestRunCompleteEventArgs? testRunComplete = null; - var outputLines = new List(); - var errorLines = new List(); - - var runHandler = new RunTestsHandler - { - HandleCompletion = (testRunCompleteArgs, testRunChanged, _, _2) => - { - WriteSummary(testRunCompleteArgs); - - testRunComplete = testRunCompleteArgs; - if (DidPass(testRunCompleteArgs)) - { - exitResult = Program.ExitSuccess; - } - }, - HandleLog = (severity, logMessage) => - { - if (severity == TestMessageLevel.Error) - { - errorLines.Add(logMessage); - } - - outputLines.Add(logMessage); - } - }; - - var stopwatch = new Stopwatch(); - stopwatch.Start(); - vstestConsoleWrapper.RunTests(assemblies, runsettingsContents, new TestPlatformOptions - { - TestCaseFilter = filterString, - }, runHandler); - stopwatch.Stop(); - - var ranToCompletion = testRunComplete != null ? !testRunComplete.IsCanceled && !testRunComplete.IsAborted : false; - return new TestResultInfo(exitResult, stopwatch.Elapsed, string.Join(Environment.NewLine, outputLines), string.Join(Environment.NewLine, errorLines), ranToCompletion); - } - - public record struct TestExecutionInfo(List Assemblies, List Tests); - - private static void WriteSummary(TestRunCompleteEventArgs testRunComplete) - { - if (testRunComplete.TestRunStatistics != null) - { - testRunComplete.TestRunStatistics.Stats.TryGetValue(TestOutcome.Passed, out var passed); - testRunComplete.TestRunStatistics.Stats.TryGetValue(TestOutcome.Failed, out var failed); - testRunComplete.TestRunStatistics.Stats.TryGetValue(TestOutcome.Skipped, out var skipped); - Console.WriteLine($"Summary: Passed: {passed}, Failed: {failed}, Skipped {skipped}"); - } - - if (testRunComplete.Error != null) - { - Console.WriteLine($"ERROR: {testRunComplete.Error}"); - } - } - - private static bool DidPass(TestRunCompleteEventArgs testRunComplete) - { - if (testRunComplete.Error != null || testRunComplete.IsAborted || testRunComplete.IsCanceled || testRunComplete.TestRunStatistics == null) - { - return false; - } - - testRunComplete.TestRunStatistics.Stats.TryGetValue(TestOutcome.Failed, out var failed); - return failed == 0; - } - -} diff --git a/src/Tools/TestExecutor/TestResultInfo.cs b/src/Tools/TestExecutor/TestResultInfo.cs deleted file mode 100644 index 0aedfece44641..0000000000000 --- a/src/Tools/TestExecutor/TestResultInfo.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; - -namespace TestExecutor; - -/// -/// The actual results from running the xunit tests. -/// -/// -/// The difference between and is the former -/// is specifically for the actual test execution results while the latter can contain extra metadata -/// about the results. For example whether it was cached, or had diagnostic, output, etc ... -/// -public readonly struct TestResultInfo -{ - public int ExitCode { get; } - public TimeSpan Elapsed { get; } - public string StandardOutput { get; } - public string ErrorOutput { get; } - public bool RanToCompletion { get; } - - internal TestResultInfo(int exitCode, TimeSpan elapsed, string standardOutput, string errorOutput, bool ranToCompletion) - { - ExitCode = exitCode; - Elapsed = elapsed; - StandardOutput = standardOutput; - ErrorOutput = errorOutput; - RanToCompletion = ranToCompletion; - } -} From 8ffd275530ff7958538489b522a4de2fe81ff0e8 Mon Sep 17 00:00:00 2001 From: David Barbet Date: Mon, 25 Jul 2022 17:52:14 -0700 Subject: [PATCH 10/26] Use rsp files and vstestconsole to run tests --- src/Tools/PrepareTests/TestDiscovery.cs | 8 + .../Source/RunTests/AssemblyScheduler.cs | 140 +++++------------- .../Source/RunTests/ProcessTestExecutor.cs | 105 +++++++------ .../Source/RunTests/TestHistoryManager.cs | 60 +++++--- src/Tools/Source/RunTests/TestRunner.cs | 41 ++++- 5 files changed, 178 insertions(+), 176 deletions(-) diff --git a/src/Tools/PrepareTests/TestDiscovery.cs b/src/Tools/PrepareTests/TestDiscovery.cs index b46ca8e727d40..3654654864f65 100644 --- a/src/Tools/PrepareTests/TestDiscovery.cs +++ b/src/Tools/PrepareTests/TestDiscovery.cs @@ -79,6 +79,14 @@ public void HandleDiscoveredTests(IEnumerable discoveredTestCases) public void HandleDiscoveryComplete(long totalTests, IEnumerable lastChunk, bool isAborted) { + if (lastChunk != null) + { + foreach (var test in lastChunk) + { + _tests.Add(test); + } + } + _isComplete = true; } diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index 8f9756bd52f5a..854603c40db22 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -22,50 +22,11 @@ namespace RunTests { - internal record struct WorkItemInfo(ImmutableSortedDictionary> Filters, int PartitionIndex) + internal record struct WorkItemInfo(ImmutableSortedDictionary> Filters, int PartitionIndex) { internal string DisplayName => $"{string.Join("_", Filters.Keys.Select(a => Path.GetFileNameWithoutExtension(a.AssemblyName)))}_{PartitionIndex}"; } - internal interface ITestFilter - { - internal string GetFilterString(); - - internal TimeSpan GetExecutionTime(); - } - - internal record struct AssemblyTestFilter(ImmutableArray TypesInAssembly) : ITestFilter - { - TimeSpan ITestFilter.GetExecutionTime() => TimeSpan.FromMilliseconds(TypesInAssembly.SelectMany(type => type.Tests).Sum(test => test.ExecutionTime.TotalMilliseconds)); - - /// - /// TODO - NOT FINE IF THERE ARE OTHER FILTERS - - /// - /// - string ITestFilter.GetFilterString() => string.Empty; - } - - internal record struct TypeTestFilter(TypeInfo Type) : ITestFilter - { - TimeSpan ITestFilter.GetExecutionTime() => TimeSpan.FromMilliseconds(Type.Tests.Sum(test => test.ExecutionTime.TotalMilliseconds)); - - string ITestFilter.GetFilterString() - { - // https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests?pivots=mstest#syntax - // We want to avoid matching other test classes whose names are prefixed with this test class's name. - // For example, avoid running 'AttributeTests_WellKnownMember', when the request here is to run 'AttributeTests'. - // We append a '.', assuming that all test methods in the class *will* match it, but not methods in other classes. - return $"{Type.Name}."; - } - } - - internal record struct MethodTestFilter(TestMethodInfo Test) : ITestFilter - { - TimeSpan ITestFilter.GetExecutionTime() => Test.ExecutionTime; - - string ITestFilter.GetFilterString() => Test.FullyQualifiedName; - } - internal sealed class AssemblyScheduler { /// @@ -88,23 +49,24 @@ public async Task> ScheduleAsync(ImmutableArray assembly, GetTypeInfoList); - - ConsoleUtil.WriteLine($"Found {orderedTypeInfos.Values.SelectMany(t => t).SelectMany(t => t.Tests).Count()} tests to run in {orderedTypeInfos.Keys.Count()} assemblies"); - - if (_options.Sequential) + if (_options.Sequential || !_options.UseHelix) { - Logger.Log("Building sequential work items"); + Logger.Log("Building work items with one assembly each."); // return individual work items per assembly that contain all the tests in that assembly. - return CreateWorkItemsForFullAssemblies(orderedTypeInfos); + return CreateWorkItemsForFullAssemblies(assemblies); } + var orderedTypeInfos = assemblies.ToImmutableSortedDictionary(assembly => assembly, GetTypeInfoList); + + ConsoleUtil.WriteLine($"Found {orderedTypeInfos.Values.SelectMany(t => t).SelectMany(t => t.Tests).Count()} tests to run in {orderedTypeInfos.Keys.Count()} assemblies"); + // Retrieve test runtimes from azure devops historical data. var testHistory = await TestHistoryManager.GetTestHistoryAsync(cancellationToken); if (testHistory.IsEmpty) { // We didn't have any test history from azure devops, just partition by assembly. - return CreateWorkItemsForFullAssemblies(orderedTypeInfos); + ConsoleUtil.WriteLine($"##[warning]Could not look up test history - building a single work item per assembly"); + return CreateWorkItemsForFullAssemblies(assemblies); } // Now for our current set of test methods we got from the assemblies we built, match them to tests from our test run history @@ -120,13 +82,13 @@ public async Task> ScheduleAsync(ImmutableArray CreateWorkItemsForFullAssemblies(ImmutableSortedDictionary> orderedTypeInfos) + static ImmutableArray CreateWorkItemsForFullAssemblies(ImmutableArray assemblies) { var workItems = new List(); var partitionIndex = 0; - foreach (var orderedTypeInfo in orderedTypeInfos) + foreach (var assembly in assemblies) { - var currentWorkItem = ImmutableSortedDictionary>.Empty.Add(orderedTypeInfo.Key, ImmutableArray.Create((ITestFilter)new AssemblyTestFilter(orderedTypeInfo.Value))); + var currentWorkItem = ImmutableSortedDictionary>.Empty.Add(assembly, ImmutableArray.Empty); workItems.Add(new WorkItemInfo(currentWorkItem, partitionIndex++)); } @@ -208,55 +170,29 @@ private static ImmutableArray BuildWorkItems(ImmutableSortedDictio var currentExecutionTime = TimeSpan.Zero; // Keep track of the types we're planning to add to the current work item. - var currentFilters = new SortedDictionary>(); + var currentFilters = new SortedDictionary>(); // Iterate through each assembly and type and build up the work items to run. // We add types from assemblies one by one until we hit our execution time limit, // at which point we create a new work item with the current types and start a new one. foreach (var (assembly, types) in typeInfos) { - // See if we can just add all types from this assembly to the current work item without going over our execution time limit. - var executionTimeForAllTypesInAssembly = TimeSpan.FromMilliseconds(types.SelectMany(type => type.Tests).Sum(t => t.ExecutionTime.TotalMilliseconds)); - if (executionTimeForAllTypesInAssembly + currentExecutionTime >= executionTimeLimit) + foreach (var type in types) { - // We can't add every type - go type by type to add what we can and end the work item where we need to. - foreach (var type in types) + foreach (var test in type.Tests) { - // See if we can add every test in this type to the current work item without going over our execution time limit. - var executionTimeForAllTestsInType = TimeSpan.FromMilliseconds(type.Tests.Sum(method => method.ExecutionTime.TotalMilliseconds)); - if (executionTimeForAllTestsInType + currentExecutionTime >= executionTimeLimit) + if (test.ExecutionTime + currentExecutionTime >= executionTimeLimit) { - // We can't add every test, go test by test to add what we can and end the work item when we hit the limit. - foreach (var test in type.Tests) - { - if (test.ExecutionTime + currentExecutionTime >= executionTimeLimit) - { - // Adding this type would put us over the time limit for this partition. - // Add the current work item to our list and start a new one. - AddCurrentWorkItem(); - } - - // Update the current group in the work item with this new type. - AddFilter(assembly, new MethodTestFilter(test)); - currentExecutionTime += test.ExecutionTime; - } - } - else - { - // All the tests in this type can be safely added to the current work item. - // Add them and update our work item execution time with the total execution time of tests in the type. - AddFilter(assembly, new TypeTestFilter(type)); - currentExecutionTime += executionTimeForAllTestsInType; + // Adding this type would put us over the time limit for this partition. + // Add the current work item to our list and start a new one. + AddCurrentWorkItem(); } + + // Update the current group in the work item with this new type. + AddFilter(assembly, test); + currentExecutionTime += test.ExecutionTime; } } - else - { - // All the types in this assembly can safely be added to the current work item. - // Add them and update our work item execution time with the total execution time of tests in the assembly. - AddFilter(assembly, new AssemblyTestFilter(types)); - currentExecutionTime += executionTimeForAllTypesInAssembly; - } } // Add any remaining tests to the work item. @@ -275,16 +211,18 @@ void AddCurrentWorkItem() currentExecutionTime = TimeSpan.Zero; } - void AddFilter(AssemblyInfo assembly, ITestFilter filter) + void AddFilter(AssemblyInfo assembly, TestMethodInfo test) { if (currentFilters.TryGetValue(assembly, out var assemblyFilters)) { - assemblyFilters.Add(filter); + assemblyFilters.Add(test); } else { - var filterList = new List(); - filterList.Add(filter); + var filterList = new List + { + test + }; currentFilters.Add(assembly, filterList); } } @@ -296,7 +234,7 @@ private static void LogWorkItems(ImmutableArray workItems) Logger.Log("==== Work Item List ===="); foreach (var workItem in workItems) { - var totalExecutionTime = TimeSpan.FromMilliseconds(workItem.Filters.Values.SelectMany(f => f).Sum(f => f.GetExecutionTime().TotalMilliseconds)); + var totalExecutionTime = TimeSpan.FromMilliseconds(workItem.Filters.Values.SelectMany(f => f).Sum(f => f.ExecutionTime.TotalMilliseconds)); Logger.Log($"- Work Item {workItem.PartitionIndex} (Execution time {totalExecutionTime})"); if (totalExecutionTime > s_maxExecutionTime) { @@ -305,19 +243,12 @@ private static void LogWorkItems(ImmutableArray workItems) foreach (var assembly in workItem.Filters) { - var assemblyRuntime = TimeSpan.FromMilliseconds(assembly.Value.Sum(f => f.GetExecutionTime().TotalMilliseconds)); + var assemblyRuntime = TimeSpan.FromMilliseconds(assembly.Value.Sum(f => f.ExecutionTime.TotalMilliseconds)); Logger.Log($" - {assembly.Key.AssemblyName} with execution time {assemblyRuntime}"); - - var typeFilters = assembly.Value.Where(f => f is TypeTestFilter); - if (typeFilters.Count() > 0) - { - Logger.Log($" - {typeFilters.Count()} types: {string.Join(",", typeFilters)}"); - } - - var testFilters = assembly.Value.Where(f => f is MethodTestFilter); + var testFilters = assembly.Value; if (testFilters.Count() > 0) { - Logger.Log($" - {testFilters.Count()} tests: {string.Join(",", testFilters)}"); + Logger.Log($" - {testFilters.Length} tests: {string.Join(",", testFilters.Select(t => t.FullyQualifiedName))}"); } } } @@ -327,8 +258,9 @@ private static ImmutableArray GetTypeInfoList(AssemblyInfo assemblyInf { var assemblyDirectory = Path.GetDirectoryName(assemblyInfo.AssemblyPath); var testListPath = Path.Combine(assemblyDirectory!, "testlist.json"); - var deserialized = JsonSerializer.Deserialize>(File.ReadAllText(testListPath)); + Contract.Assert(File.Exists(testListPath)); + var deserialized = JsonSerializer.Deserialize>(File.ReadAllText(testListPath)); Contract.Assert(deserialized != null); var tests = deserialized.GroupBy(GetTypeName) .Select(group => new TypeInfo(GetName(group.Key), group.Key, group.Select(test => new TestMethodInfo(GetName(test), test, TimeSpan.Zero)).ToImmutableArray())) diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index 3075ec0130bde..8dba232f7c734 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -20,80 +20,86 @@ namespace RunTests { internal sealed class ProcessTestExecutor { - public string GetCommandLineArguments(WorkItemInfo workItem, bool useSingleQuotes, Options options) + public static string BuildRspFileContents(WorkItemInfo workItem, Options options) { - // http://www.gnu.org/software/bash/manual/html_node/Single-Quotes.html - // Single quotes are needed in bash to avoid the need to escape characters such as backtick (`) which are found in metadata names. - // Batch scripts don't need to worry about escaping backticks, but they don't support single quoted strings, so we have to use double quotes. - // We also need double quotes when building an arguments string for Process.Start in .NET Core so that splitting/unquoting works as expected. - var sep = useSingleQuotes ? "'" : @""""; + var fileContentsBuilder = new StringBuilder(); - var builder = new StringBuilder(); - builder.Append($@"test"); + // Add each assembly we want to test on a new line. + var assemblyPaths = workItem.Filters.Keys.Select(assembly => assembly.AssemblyPath); + foreach (var path in assemblyPaths) + { + fileContentsBuilder.AppendLine($"\"{path}\""); + } + + fileContentsBuilder.AppendLine($@"/Platform:{options.Architecture}"); + fileContentsBuilder.AppendLine($@"/Logger:xunit;LogFilePath={GetResultsFilePath(workItem, options, "xml")}"); + if (options.IncludeHtml) + { + fileContentsBuilder.AppendLine($@"/Logger:html;LogFileName={GetResultsFilePath(workItem, options, "html")}"); + } + + var blameOption = "CollectHangDump"; + if (!options.CollectDumps) + { + // The 'CollectDumps' option uses operating system features to collect dumps when a process crashes. We + // only enable the test executor blame feature in remaining cases, as the latter relies on ProcDump and + // interferes with automatic crash dump collection on Windows. + blameOption = "CollectDump;CollectHangDump"; + } - var escapedAssemblyPaths = workItem.Filters.Keys.Select(assembly => $"{sep}{assembly.AssemblyPath}{sep}"); - builder.Append($@" {string.Join(" ", escapedAssemblyPaths)}"); + // The 25 minute timeout in integration tests accounts for the fact that VSIX deployment and/or experimental hive reset and + // configuration can take significant time (seems to vary from ~10 seconds to ~15 minutes), and the blame + // functionality cannot separate this configuration overhead from the first test which will eventually run. + // https://github.com/dotnet/roslyn/issues/59851 + // + // Helix timeout is 15 minutes as helix jobs fully timeout in 30minutes. So in order to capture dumps we need the timeout + // to be 2x shorter than the expected test run time (15min) in case only the last test hangs. + var timeout = options.UseHelix ? "15minutes" : "25minutes"; + fileContentsBuilder.AppendLine($"/Blame:{blameOption};TestTimeout=15minutes;DumpType=full"); - var filters = workItem.Filters.Values.SelectMany(filter => filter).Where(filter => !string.IsNullOrEmpty(filter.GetFilterString())).ToImmutableArray(); + // Build the filter string + var filterStringBuilder = new StringBuilder(); + var filters = workItem.Filters.Values.SelectMany(filter => filter).Where(filter => !string.IsNullOrEmpty(filter.FullyQualifiedName)).ToImmutableArray(); if (filters.Length > 0 || !string.IsNullOrWhiteSpace(options.TestFilter)) { - builder.Append($@" --filter {sep}"); + filterStringBuilder.Append("/TestCaseFilter:\""); var any = false; foreach (var filter in filters) { MaybeAddSeparator(); - builder.Append(filter.GetFilterString()); + filterStringBuilder.Append(filter.FullyQualifiedName); } - builder.Append(sep); - if (options.TestFilter is object) + if (options.TestFilter is not null) { MaybeAddSeparator(); - builder.Append(options.TestFilter); + filterStringBuilder.Append(options.TestFilter); } void MaybeAddSeparator(char separator = '|') { if (any) { - builder.Append(separator); + filterStringBuilder.Append(separator); } any = true; } } - builder.Append($@" --arch {options.Architecture}"); - builder.Append($@" --logger {sep}xunit;LogFilePath={GetResultsFilePath(workItem, options, "xml")}{sep}"); - - if (options.IncludeHtml) - { - builder.AppendFormat($@" --logger {sep}html;LogFileName={GetResultsFilePath(workItem, options, "html")}{sep}"); - } - - if (!options.CollectDumps) - { - // The 'CollectDumps' option uses operating system features to collect dumps when a process crashes. We - // only enable the test executor blame feature in remaining cases, as the latter relies on ProcDump and - // interferes with automatic crash dump collection on Windows. - builder.Append(" --blame-crash"); - } - - // The 25 minute timeout in integration tests accounts for the fact that VSIX deployment and/or experimental hive reset and - // configuration can take significant time (seems to vary from ~10 seconds to ~15 minutes), and the blame - // functionality cannot separate this configuration overhead from the first test which will eventually run. - // https://github.com/dotnet/roslyn/issues/59851 - // - // Helix timeout is 15 minutes as helix jobs fully timeout in 30minutes. So in order to capture dumps we need the timeout - // to be 2x shorter than the expected test run time (15min) in case only the last test hangs. - var timeout = options.UseHelix ? "15minutes" : "25minutes"; - - builder.Append($" --blame-hang-dump-type full --blame-hang-timeout {timeout}"); + fileContentsBuilder.AppendLine(filterStringBuilder.ToString()); + return fileContentsBuilder.ToString(); + } - return builder.ToString(); + private static string GetVsTestConsolePath(string dotnetPath) + { + var dotnetDir = Path.GetDirectoryName(dotnetPath)!; + var sdkDir = Path.Combine(dotnetDir, "sdk"); + var vsTestConsolePath = Directory.EnumerateFiles(sdkDir, "vstest.console.dll", SearchOption.AllDirectories).Last(); + return vsTestConsolePath; } - private string GetResultsFilePath(WorkItemInfo workItemInfo, Options options, string suffix = "xml") + private static string GetResultsFilePath(WorkItemInfo workItemInfo, Options options, string suffix = "xml") { var fileName = $"WorkItem_{workItemInfo.PartitionIndex}_{options.Architecture}_test_results.{suffix}"; return Path.Combine(options.TestResultsDirectory, fileName); @@ -122,7 +128,14 @@ private async Task RunTestAsyncInternal(WorkItemInfo workItemInfo, O { try { - var commandLineArguments = GetCommandLineArguments(workItemInfo, useSingleQuotes: false, options); + var rspFileContents = BuildRspFileContents(workItemInfo, options); + var rspFilePath = Path.Combine(Directory.GetCurrentDirectory(), $"vstest_{workItemInfo.PartitionIndex}.rsp"); + File.WriteAllText(rspFilePath, rspFileContents); + + var vsTestConsolePath = GetVsTestConsolePath(options.DotnetFilePath); + + var commandLineArguments = $"exec \"{vsTestConsolePath}\" @\"{rspFilePath}\""; + var resultsFilePath = GetResultsFilePath(workItemInfo, options); var resultsDir = Path.GetDirectoryName(resultsFilePath); var htmlResultsFilePath = options.IncludeHtml ? GetResultsFilePath(workItemInfo, options, "html") : null; diff --git a/src/Tools/Source/RunTests/TestHistoryManager.cs b/src/Tools/Source/RunTests/TestHistoryManager.cs index eb52dda439a39..e82a2b4947bf6 100644 --- a/src/Tools/Source/RunTests/TestHistoryManager.cs +++ b/src/Tools/Source/RunTests/TestHistoryManager.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Contracts; using System.Linq; using System.Net.Http; using System.Text; @@ -26,7 +28,7 @@ internal class TestHistoryManager private const int MaxTestsReturnedPerRequest = 10000; /// - /// The pipeline id for roslyn-ci, see https://dev.azure.com/dnceng/public/_build?definitionId=15&_a=summary + /// The pipeline id for roslyn-ci, see https://dev.azure.com/dnceng/public/_build?definitionId=15 /// private const int RoslynCiBuildDefinitionId = 15; @@ -36,32 +38,42 @@ internal class TestHistoryManager private static readonly Uri s_projectUri = new(@"https://dev.azure.com/dnceng"); /// - /// Todo - build definition comes from System.DefinitionId - /// Todo - stageName comes from System.StageName to identify which test runs to get from the build. + /// Looks up the last passing test run for the current build and stage to estimate execution times for each test. /// - public static async Task> GetTestHistoryAsync(/*int buildDefinitionId, string stageName, string branchName*/CancellationToken cancellationToken) + public static async Task> GetTestHistoryAsync(CancellationToken cancellationToken) { - var pat = Environment.GetEnvironmentVariable("TEST_PAT"); - var credentials = new Microsoft.VisualStudio.Services.Common.VssBasicCredential(string.Empty, pat); + // Gets environment variables set by our test yaml templates. + // The access token is required to lookup test histories. + // We use the target branch of the current build to lookup the last successful build for the same branch. + // The stage name is used to filter the tests on the last passing build to only those that apply to the currently running stage. + if (!TryGetEnvironmentVariable("SYSTEM_ACCESSTOKEN", out var accessToken) + || !TryGetEnvironmentVariable("SYSTEM_STAGENAME", out var stageName)) + { + Console.WriteLine("Missing required environment variables - skipping test history lookup"); + return ImmutableDictionary.Empty; + } - var connection = new VssConnection(s_projectUri, credentials); + // Use the target branch (in the case of PRs) or source branch to find the last successful build. + var targetBranch = Environment.GetEnvironmentVariable("SYSTEM_PULLREQUESTTARGETBRANCH") ?? Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH"); + if (string.IsNullOrEmpty(targetBranch)) + { + Console.WriteLine("Missing both PR target branch and build source branch environment variables - skipping test history lookup"); + return ImmutableDictionary.Empty; + } - var stageName = "Test_Windows_Desktop_Release_64"; - var targetBranchName = "main"; + var credentials = new Microsoft.VisualStudio.Services.Common.VssBasicCredential(string.Empty, accessToken); - using var buildClient = connection.GetClient(); + var connection = new VssConnection(s_projectUri, credentials); - // Tries to get the latest succeeded build from the input build definition (pipeline) from dnceng/public with the specified branch. - Logger.Log($"Branch name: {targetBranchName}"); - Logger.Log($"Stage name: {stageName}"); + using var buildClient = connection.GetClient(); - var adoBranchName = $"refs/heads/{targetBranchName}"; - var builds = await buildClient.GetBuildsAsync2(project: "public", new int[] { RoslynCiBuildDefinitionId }, resultFilter: BuildResult.Succeeded, queryOrder: BuildQueryOrder.FinishTimeDescending, maxBuildsPerDefinition: 1, reasonFilter: BuildReason.IndividualCI, branchName: adoBranchName, cancellationToken: cancellationToken); + Console.WriteLine($"Getting last successful build for branch {targetBranch} and stage {stageName}"); + var builds = await buildClient.GetBuildsAsync2(project: "public", new int[] { RoslynCiBuildDefinitionId }, resultFilter: BuildResult.Succeeded, queryOrder: BuildQueryOrder.FinishTimeDescending, maxBuildsPerDefinition: 1, reasonFilter: BuildReason.IndividualCI, branchName: targetBranch, cancellationToken: cancellationToken); var lastSuccessfulBuild = builds?.FirstOrDefault(); if (lastSuccessfulBuild == null) { // If this is a new branch we may not have any historical data for it. - ConsoleUtil.WriteLine($"##[warning]Unable to get the last successful build for definition {RoslynCiBuildDefinitionId} and branch {adoBranchName}"); + ConsoleUtil.WriteLine($"Unable to get the last successful build for definition {RoslynCiBuildDefinitionId} and branch {targetBranch}"); return ImmutableDictionary.Empty; } @@ -76,11 +88,11 @@ public static async Task> GetTestHistoryAs if (runForThisStage == null) { // If this is a new stage, historical runs will not have any data for it. - ConsoleUtil.WriteLine($"##[warning]Unable to get a run with name {stageName} from build {lastSuccessfulBuild.Url}."); + ConsoleUtil.WriteLine($"Unable to get a run with name {stageName} from build {lastSuccessfulBuild.Url}."); return ImmutableDictionary.Empty; } - ConsoleUtil.WriteLine($"Looking up test execution data from last successful build {lastSuccessfulBuild.Id} on branch {targetBranchName} and stage {stageName}"); + ConsoleUtil.WriteLine($"Looking up test execution data from last successful build {lastSuccessfulBuild.Id} on branch {targetBranch} and stage {stageName}"); var totalTests = runForThisStage.TotalTests; @@ -136,4 +148,16 @@ private static string CleanTestName(string fullyQualifiedTestName) return beforeMethodArgs; } + private static bool TryGetEnvironmentVariable(string envVarName, [NotNullWhen(true)] out string? envVar) + { + envVar = Environment.GetEnvironmentVariable(envVarName); + if (string.IsNullOrEmpty(envVar)) + { + Console.WriteLine($"Required environment variable {envVarName} is not set"); + return false; + } + + return true; + } + } diff --git a/src/Tools/Source/RunTests/TestRunner.cs b/src/Tools/Source/RunTests/TestRunner.cs index 861dadd2ab24b..cde681ee77392 100644 --- a/src/Tools/Source/RunTests/TestRunner.cs +++ b/src/Tools/Source/RunTests/TestRunner.cs @@ -60,10 +60,10 @@ internal async Task RunAllOnHelixAsync(ImmutableArray temp.txt"); + command.AppendLine("set /p vstestConsolePath= {payloadDirectory} - {command} + {escapedCommand} {postCommands} From a8c75f3a87ffdf9607ca3462edf59718da6177dc Mon Sep 17 00:00:00 2001 From: David Barbet Date: Tue, 26 Jul 2022 11:51:44 -0700 Subject: [PATCH 11/26] more testing --- eng/Versions.props | 1 + eng/build.sh | 2 +- src/Tools/PrepareTests/PrepareTests.csproj | 6 +- src/Tools/PrepareTests/Program.cs | 1 - src/Tools/PrepareTests/TestDiscovery.cs | 13 +++-- .../Source/RunTests/AssemblyScheduler.cs | 55 +++++++++++++------ .../Source/RunTests/ProcessTestExecutor.cs | 2 + .../Source/RunTests/TestHistoryManager.cs | 20 ++++--- src/Tools/Source/RunTests/TestRunner.cs | 11 +++- 9 files changed, 73 insertions(+), 38 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index d99e6129370de..d04f0829c5296 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -125,6 +125,7 @@ $(MicrosoftServiceHubVersion) $(MicrosoftServiceHubVersion) 1.1.1-beta-21566-01 + 17.4.0-preview-20220707-01 10.1.0 17.2.41-alpha 15.8.27812-alpha diff --git a/eng/build.sh b/eng/build.sh index 9ec073ab72113..f83e4229054fe 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -320,6 +320,6 @@ if [[ "$test_core_clr" == true ]]; then if [[ "$ci" != true ]]; then runtests_args="$runtests_args --html" fi - dotnet exec "$scriptroot/../artifacts/bin/RunTests/${configuration}/net6.0/RunTests.dll" --tfm net6.0 --configuration ${configuration} --dotnet ${_InitializeDotNetCli}/dotnet $runtests_args + dotnet exec "$scriptroot/../artifacts/bin/RunTests/${configuration}/net6.0/RunTests.dll" --tfm net6.0 --configuration ${configuration} --logs ${log_dir} --dotnet ${_InitializeDotNetCli}/dotnet $runtests_args fi ExitWithExitCode 0 diff --git a/src/Tools/PrepareTests/PrepareTests.csproj b/src/Tools/PrepareTests/PrepareTests.csproj index 542840d45ce49..861ad52f031cd 100644 --- a/src/Tools/PrepareTests/PrepareTests.csproj +++ b/src/Tools/PrepareTests/PrepareTests.csproj @@ -10,8 +10,8 @@ - - - + + + diff --git a/src/Tools/PrepareTests/Program.cs b/src/Tools/PrepareTests/Program.cs index 0438a43548ad6..af68612d4b22b 100644 --- a/src/Tools/PrepareTests/Program.cs +++ b/src/Tools/PrepareTests/Program.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using System; -using System.Threading.Tasks; using Mono.Options; namespace PrepareTests; diff --git a/src/Tools/PrepareTests/TestDiscovery.cs b/src/Tools/PrepareTests/TestDiscovery.cs index 3654654864f65..f38906a930522 100644 --- a/src/Tools/PrepareTests/TestDiscovery.cs +++ b/src/Tools/PrepareTests/TestDiscovery.cs @@ -69,15 +69,18 @@ private class DiscoveryHandler : ITestDiscoveryEventsHandler private readonly ConcurrentBag _tests = new(); private bool _isComplete = false; - public void HandleDiscoveredTests(IEnumerable discoveredTestCases) + public void HandleDiscoveredTests(IEnumerable? discoveredTestCases) { - foreach (var test in discoveredTestCases) + if (discoveredTestCases != null) { - _tests.Add(test); + foreach (var test in discoveredTestCases) + { + _tests.Add(test); + } } } - public void HandleDiscoveryComplete(long totalTests, IEnumerable lastChunk, bool isAborted) + public void HandleDiscoveryComplete(long totalTests, IEnumerable? lastChunk, bool isAborted) { if (lastChunk != null) { @@ -90,7 +93,7 @@ public void HandleDiscoveryComplete(long totalTests, IEnumerable lastC _isComplete = true; } - public void HandleLogMessage(TestMessageLevel level, string message) + public void HandleLogMessage(TestMessageLevel level, string? message) { Console.WriteLine(message); } diff --git a/src/Tools/Source/RunTests/AssemblyScheduler.cs b/src/Tools/Source/RunTests/AssemblyScheduler.cs index 854603c40db22..9bc09ce161203 100644 --- a/src/Tools/Source/RunTests/AssemblyScheduler.cs +++ b/src/Tools/Source/RunTests/AssemblyScheduler.cs @@ -30,13 +30,18 @@ internal record struct WorkItemInfo(ImmutableSortedDictionary - /// Our execution time limit is 3 minutes. We really want to run tests under 5 minutes, but we need to limit test execution time - /// to 3 minutes to account for overhead elsewhere in setting up the test run, for example + /// Our execution time limit is 2m30s. We really want to run tests under 5 minutes, but we need to limit test execution time + /// to 2m30s to account for overhead elsewhere in setting up the test run, for example /// 1. Test discovery. /// 2. Downloading assets to the helix machine. /// 3. Setting up the test host for each assembly. /// - private static readonly TimeSpan s_maxExecutionTime = TimeSpan.FromMinutes(3); + private static readonly TimeSpan s_maxExecutionTime = TimeSpan.FromSeconds(150); + + /// + /// If we were unable to find the test execution history, we fall back to partitioning by just method count. + /// + private static readonly int s_maxMethodCount = 500; private readonly Options _options; @@ -64,9 +69,15 @@ public async Task> ScheduleAsync(ImmutableArray( + orderedTypeInfos, + isOverLimitFunc: (accumulatedMethodCount) => accumulatedMethodCount >= s_maxMethodCount, + addFunc: (currentTest, accumulatedMethodCount) => accumulatedMethodCount + 1); + + LogWorkItems(workItemsByMethodCount); + return workItemsByMethodCount; } // Now for our current set of test methods we got from the assemblies we built, match them to tests from our test run history @@ -76,9 +87,10 @@ public async Task> ScheduleAsync(ImmutableArray( + orderedTypeInfos, + isOverLimitFunc: (accumulatedExecutionTime) => accumulatedExecutionTime >= s_maxExecutionTime, + addFunc: (currentTest, accumulatedExecutionTime) => currentTest.ExecutionTime + accumulatedExecutionTime); LogWorkItems(workItems); return workItems; @@ -159,29 +171,36 @@ void LogResults() } } - private static ImmutableArray BuildWorkItems(ImmutableSortedDictionary> typeInfos, TimeSpan executionTimeLimit) + private static ImmutableArray BuildWorkItems( + ImmutableSortedDictionary> typeInfos, + Func isOverLimitFunc, + Func addFunc) where T : struct { var workItems = new List(); // Keep track of which work item we are creating - used to identify work items in names. var workItemIndex = 0; - // Keep track of the execution time of the current work item we are adding to. - var currentExecutionTime = TimeSpan.Zero; + // Keep track of the limit of the current work item we are adding to. + var accumulatedValue = default(T); // Keep track of the types we're planning to add to the current work item. var currentFilters = new SortedDictionary>(); // Iterate through each assembly and type and build up the work items to run. - // We add types from assemblies one by one until we hit our execution time limit, - // at which point we create a new work item with the current types and start a new one. + // We add types from assemblies one by one until we hit our limit, + // at which point we create a work item with the current types and start a new one. foreach (var (assembly, types) in typeInfos) { foreach (var type in types) { foreach (var test in type.Tests) { - if (test.ExecutionTime + currentExecutionTime >= executionTimeLimit) + // Get a new value representing the value from the test plus the accumulated value in the work item. + var newAccumulatedValue = addFunc(test, accumulatedValue); + + // If the new accumulated value is greater than the limit + if (isOverLimitFunc(newAccumulatedValue)) { // Adding this type would put us over the time limit for this partition. // Add the current work item to our list and start a new one. @@ -190,7 +209,6 @@ private static ImmutableArray BuildWorkItems(ImmutableSortedDictio // Update the current group in the work item with this new type. AddFilter(assembly, test); - currentExecutionTime += test.ExecutionTime; } } } @@ -208,7 +226,7 @@ void AddCurrentWorkItem() } currentFilters = new(); - currentExecutionTime = TimeSpan.Zero; + accumulatedValue = default; } void AddFilter(AssemblyInfo assembly, TestMethodInfo test) @@ -225,12 +243,15 @@ void AddFilter(AssemblyInfo assembly, TestMethodInfo test) }; currentFilters.Add(assembly, filterList); } + + accumulatedValue = addFunc(test, accumulatedValue); } } private static void LogWorkItems(ImmutableArray workItems) { + ConsoleUtil.WriteLine($"Built {workItems.Length} work items"); Logger.Log("==== Work Item List ===="); foreach (var workItem in workItems) { diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index 8dba232f7c734..c47ae330e252e 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -76,6 +76,8 @@ public static string BuildRspFileContents(WorkItemInfo workItem, Options options filterStringBuilder.Append(options.TestFilter); } + filterStringBuilder.Append('"'); + void MaybeAddSeparator(char separator = '|') { if (any) diff --git a/src/Tools/Source/RunTests/TestHistoryManager.cs b/src/Tools/Source/RunTests/TestHistoryManager.cs index e82a2b4947bf6..28b8bace08c86 100644 --- a/src/Tools/Source/RunTests/TestHistoryManager.cs +++ b/src/Tools/Source/RunTests/TestHistoryManager.cs @@ -45,16 +45,19 @@ public static async Task> GetTestHistoryAs // Gets environment variables set by our test yaml templates. // The access token is required to lookup test histories. // We use the target branch of the current build to lookup the last successful build for the same branch. - // The stage name is used to filter the tests on the last passing build to only those that apply to the currently running stage. + // + // The phase name is used to filter the tests on the last passing build to only those that apply to the currently running phase. + // Note here that 'phaseName' corresponds to the 'jobName' defined in our pipeline yaml file and the job name env var is not correct. + // See https://developercommunity.visualstudio.com/t/systemjobname-seems-to-be-incorrectly-assigned-and/1209736 if (!TryGetEnvironmentVariable("SYSTEM_ACCESSTOKEN", out var accessToken) - || !TryGetEnvironmentVariable("SYSTEM_STAGENAME", out var stageName)) + || !TryGetEnvironmentVariable("SYSTEM_PHASENAME", out var phaseName)) { Console.WriteLine("Missing required environment variables - skipping test history lookup"); return ImmutableDictionary.Empty; } // Use the target branch (in the case of PRs) or source branch to find the last successful build. - var targetBranch = Environment.GetEnvironmentVariable("SYSTEM_PULLREQUESTTARGETBRANCH") ?? Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH"); + var targetBranch = Environment.GetEnvironmentVariable("SYSTEM_PULLREQUEST_TARGETBRANCH") ?? Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCHNAME"); if (string.IsNullOrEmpty(targetBranch)) { Console.WriteLine("Missing both PR target branch and build source branch environment variables - skipping test history lookup"); @@ -67,8 +70,9 @@ public static async Task> GetTestHistoryAs using var buildClient = connection.GetClient(); - Console.WriteLine($"Getting last successful build for branch {targetBranch} and stage {stageName}"); - var builds = await buildClient.GetBuildsAsync2(project: "public", new int[] { RoslynCiBuildDefinitionId }, resultFilter: BuildResult.Succeeded, queryOrder: BuildQueryOrder.FinishTimeDescending, maxBuildsPerDefinition: 1, reasonFilter: BuildReason.IndividualCI, branchName: targetBranch, cancellationToken: cancellationToken); + Console.WriteLine($"Getting last successful build for branch {targetBranch}"); + var adoBranch = $"refs/heads/{targetBranch}"; + var builds = await buildClient.GetBuildsAsync2(project: "public", new int[] { RoslynCiBuildDefinitionId }, resultFilter: BuildResult.Succeeded, queryOrder: BuildQueryOrder.FinishTimeDescending, maxBuildsPerDefinition: 1, reasonFilter: BuildReason.IndividualCI, branchName: adoBranch, cancellationToken: cancellationToken); var lastSuccessfulBuild = builds?.FirstOrDefault(); if (lastSuccessfulBuild == null) { @@ -84,15 +88,15 @@ public static async Task> GetTestHistoryAs var maxTime = lastSuccessfulBuild.FinishTime!.Value; var runsInBuild = await testClient.QueryTestRunsAsync2("public", minTime, maxTime, buildIds: new int[] { lastSuccessfulBuild.Id }, cancellationToken: cancellationToken); - var runForThisStage = runsInBuild.SingleOrDefault(r => r.Name.Contains(stageName)); + var runForThisStage = runsInBuild.SingleOrDefault(r => r.Name.Contains(phaseName)); if (runForThisStage == null) { // If this is a new stage, historical runs will not have any data for it. - ConsoleUtil.WriteLine($"Unable to get a run with name {stageName} from build {lastSuccessfulBuild.Url}."); + ConsoleUtil.WriteLine($"Unable to get a run with name {phaseName} from build {lastSuccessfulBuild.Url}."); return ImmutableDictionary.Empty; } - ConsoleUtil.WriteLine($"Looking up test execution data from last successful build {lastSuccessfulBuild.Id} on branch {targetBranch} and stage {stageName}"); + ConsoleUtil.WriteLine($"Looking up test execution data for build {lastSuccessfulBuild.Id} on branch {targetBranch} and stage {phaseName}"); var totalTests = runForThisStage.TotalTests; diff --git a/src/Tools/Source/RunTests/TestRunner.cs b/src/Tools/Source/RunTests/TestRunner.cs index cde681ee77392..a57f04b053448 100644 --- a/src/Tools/Source/RunTests/TestRunner.cs +++ b/src/Tools/Source/RunTests/TestRunner.cs @@ -222,9 +222,11 @@ string MakeHelixWorkItemProject(WorkItemInfo workItemInfo) // DOTNET_ROOT environment variable set by helix. if (isUnix) { - command.AppendLine("vstestConsolePath=$(find ${DOTNET_ROOT} -name \"vstest.console.dll\")"); - command.AppendLine("echo ${vstestConsolePath}"); - command.AppendLine($"dotnet exec \"${{vstestConsolePath}}\" @{rspFileName}"); + // $ is a special character in msbuild so we replace it with %24 in the helix project. + // https://docs.microsoft.com/en-us/visualstudio/msbuild/msbuild-special-characters?view=vs-2022 + command.AppendLine("vstestConsolePath=%24(find %24{DOTNET_ROOT} -name \"vstest.console.dll\")"); + command.AppendLine("echo %24{vstestConsolePath}"); + command.AppendLine($"dotnet exec \"%24{{vstestConsolePath}}\" @{rspFileName}"); } else { @@ -236,6 +238,9 @@ string MakeHelixWorkItemProject(WorkItemInfo workItemInfo) command.AppendLine($"dotnet exec \"%vstestConsolePath%\" @{rspFileName}"); } + // TODO remove false exit code, setting to get test run logs..... + command.AppendLine(isUnix ? "false" : "cd invaliddir"); + // The command string contains characters like % which are not valid XML to pass into the helix csproj. var escapedCommand = SecurityElement.Escape(command.ToString()); From 4e120ec538ee26522a63e5a531471a698666313b Mon Sep 17 00:00:00 2001 From: David Barbet Date: Tue, 26 Jul 2022 13:24:37 -0700 Subject: [PATCH 12/26] Fix integration tests assembly lookup --- src/Tools/PrepareTests/Program.cs | 2 +- src/Tools/Source/RunTests/ProcessTestExecutor.cs | 2 ++ src/Tools/Source/RunTests/Program.cs | 4 +--- src/Tools/Source/RunTests/TestHistoryManager.cs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Tools/PrepareTests/Program.cs b/src/Tools/PrepareTests/Program.cs index af68612d4b22b..b5b34559c79d7 100644 --- a/src/Tools/PrepareTests/Program.cs +++ b/src/Tools/PrepareTests/Program.cs @@ -24,7 +24,7 @@ public static int Main(string[] args) { "source=", "Path to binaries", (string s) => source = s }, { "destination=", "Output path", (string s) => destination = s }, { "unix", "If true, prepares tests for unix environment instead of Windows", o => isUnix = o is object }, - { "dotnetPath=", "Output path", (string s) => dotnetPath = s }, + { "dotnetPath=", "Path to the dotnet CLI", (string s) => dotnetPath = s }, }; options.Parse(args); diff --git a/src/Tools/Source/RunTests/ProcessTestExecutor.cs b/src/Tools/Source/RunTests/ProcessTestExecutor.cs index c47ae330e252e..dc471761a8ee0 100644 --- a/src/Tools/Source/RunTests/ProcessTestExecutor.cs +++ b/src/Tools/Source/RunTests/ProcessTestExecutor.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.InteropServices; @@ -60,6 +61,7 @@ public static string BuildRspFileContents(WorkItemInfo workItem, Options options // Build the filter string var filterStringBuilder = new StringBuilder(); var filters = workItem.Filters.Values.SelectMany(filter => filter).Where(filter => !string.IsNullOrEmpty(filter.FullyQualifiedName)).ToImmutableArray(); + if (filters.Length > 0 || !string.IsNullOrWhiteSpace(options.TestFilter)) { filterStringBuilder.Append("/TestCaseFilter:\""); diff --git a/src/Tools/Source/RunTests/Program.cs b/src/Tools/Source/RunTests/Program.cs index 78c5c1a9218d1..6f98ca34a60e7 100644 --- a/src/Tools/Source/RunTests/Program.cs +++ b/src/Tools/Source/RunTests/Program.cs @@ -285,9 +285,7 @@ private static ImmutableArray GetAssemblyFilePaths(Options options { var list = new List(); var binDirectory = Path.Combine(options.ArtifactsDirectory, "bin"); - - // Find all the project folders that fit our naming scheme for unit tests. - foreach (var project in Directory.EnumerateDirectories(binDirectory, "*.UnitTests", SearchOption.TopDirectoryOnly)) + foreach (var project in Directory.EnumerateDirectories(binDirectory, "*", SearchOption.TopDirectoryOnly)) { var name = Path.GetFileName(project); if (!shouldInclude(name, options) || shouldExclude(name, options)) diff --git a/src/Tools/Source/RunTests/TestHistoryManager.cs b/src/Tools/Source/RunTests/TestHistoryManager.cs index 28b8bace08c86..e5d3d826b3e1b 100644 --- a/src/Tools/Source/RunTests/TestHistoryManager.cs +++ b/src/Tools/Source/RunTests/TestHistoryManager.cs @@ -23,7 +23,7 @@ namespace RunTests; internal class TestHistoryManager { /// - /// Azure devops limits the number of tests returned per request to 1000. + /// Azure devops limits the number of tests returned per request to 10000. /// private const int MaxTestsReturnedPerRequest = 10000; From 29ba5cf7b7716a549eb855bbfac800e9df8ef6ad Mon Sep 17 00:00:00 2001 From: David Barbet Date: Tue, 26 Jul 2022 14:13:40 -0700 Subject: [PATCH 13/26] cleanup --- eng/Versions.props | 4 +++- src/Tools/PrepareTests/PrepareTests.csproj | 4 ++-- src/Tools/Source/RunTests/AssemblyScheduler.cs | 18 ++++++++++++++---- src/Tools/Source/RunTests/TestRunner.cs | 3 --- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index d04f0829c5296..ddc526d4273a8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -43,6 +43,7 @@ 7.0.0-alpha.1.22060.1 3.1.4097 17.2.32 + 17.4.0-preview-20220707-01