Partition helix test runs using historical execution time data

eng/Versions.props

@@ -43,6 +43,7 @@
     <MicrosoftILVerificationVersion>7.0.0-alpha.1.22060.1</MicrosoftILVerificationVersion>
     <MicrosoftServiceHubVersion>4.0.117</MicrosoftServiceHubVersion>
     <MicrosoftVisualStudioThreadingPackagesVersion>17.3.1-alpha</MicrosoftVisualStudioThreadingPackagesVersion>
+    <MicrosoftTestPlatformVersion>17.4.0-preview-20220707-01</MicrosoftTestPlatformVersion>
   </PropertyGroup>
   <!--
     Dependency versions
@@ -125,6 +126,9 @@
     <MicrosoftServiceHubClientVersion>$(MicrosoftServiceHubVersion)</MicrosoftServiceHubClientVersion>
     <MicrosoftServiceHubFrameworkVersion>$(MicrosoftServiceHubVersion)</MicrosoftServiceHubFrameworkVersion>
     <MicrosoftSourceLinkToolsVersion>1.1.1-beta-21566-01</MicrosoftSourceLinkToolsVersion>
+    <MicrosoftTeamFoundationServerClientVersion>16.170.0</MicrosoftTeamFoundationServerClientVersion>
+    <MicrosoftTestPlatformTranslationLayerVersion>$(MicrosoftTestPlatformVersion)</MicrosoftTestPlatformTranslationLayerVersion>
+    <MicrosoftTestPlatformObjectModelVersion>$(MicrosoftTestPlatformVersion)</MicrosoftTestPlatformObjectModelVersion>
     <MicrosoftVisualBasicVersion>10.1.0</MicrosoftVisualBasicVersion>
     <MicrosoftVisualStudioCacheVersion>17.2.41-alpha</MicrosoftVisualStudioCacheVersion>
     <MicrosoftVisualStudioCallHierarchyPackageDefinitionsVersion>15.8.27812-alpha</MicrosoftVisualStudioCallHierarchyPackageDefinitionsVersion>

eng/build.sh

@@ -327,6 +327,6 @@ if [[ "$test_core_clr" == true ]]; then
   if [[ "$ci" != true ]]; then
     runtests_args="$runtests_args --html"
   fi
-  dotnet exec "$scriptroot/../artifacts/bin/RunTests/${configuration}/net6.0/RunTests.dll" --tfm net6.0 --configuration ${configuration} --dotnet ${_InitializeDotNetCli}/dotnet $runtests_args
+  dotnet exec "$scriptroot/../artifacts/bin/RunTests/${configuration}/net6.0/RunTests.dll" --tfm net6.0 --configuration ${configuration} --logs ${log_dir} --dotnet ${_InitializeDotNetCli}/dotnet $runtests_args
 fi
 ExitWithExitCode 0

eng/prepare-tests.ps1

@@ -11,7 +11,7 @@ try {
   $dotnet = Ensure-DotnetSdk
   # permissions issues make this a pain to do in PrepareTests itself.
   Remove-Item -Recurse -Force "$RepoRoot\artifacts\testPayload" -ErrorAction SilentlyContinue
-  Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload"
+  Exec-Console $dotnet "$RepoRoot\artifacts\bin\PrepareTests\$configuration\net6.0\PrepareTests.dll --source $RepoRoot --destination $RepoRoot\artifacts\testPayload --dotnetPath `"$dotnet`""
   exit 0
 }
 catch {

eng/prepare-tests.sh

@@ -28,4 +28,4 @@ InitializeDotNetCli true
 # permissions issues make this a pain to do in PrepareTests itself.
 rm -rf "$repo_root/artifacts/testPayload"
 
-dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix
+dotnet "$repo_root/artifacts/bin/PrepareTests/Debug/net6.0/PrepareTests.dll" --source "$repo_root" --destination "$repo_root/artifacts/testPayload" --unix --dotnetPath ${_InitializeDotNetCli}/dotnet

src/Compilers/CSharp/Test/EndToEnd/Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests.csproj

@@ -7,7 +7,9 @@
     <TargetFrameworks>net6.0;net472</TargetFrameworks>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
-  <ItemGroup Label="Project References">
+
+  <ItemGroup>
+    <AssemblyAttribute Include="Microsoft.CodeAnalysis.Test.Utilities.RunTestsInSinglePartitionAttribute" />
     <ProjectReference Include="..\..\..\..\Test\PdbUtilities\Roslyn.Test.PdbUtilities.csproj" />
     <ProjectReference Include="..\..\..\Test\Core\Microsoft.CodeAnalysis.Test.Utilities.csproj" />
     <ProjectReference Include="..\..\..\Core\Portable\Microsoft.CodeAnalysis.csproj" />

src/Compilers/Test/Core/RunInSinglePartitionAssemblyAttribute.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+
+namespace Microsoft.CodeAnalysis.Test.Utilities;
+
+/// <summary>
+/// This is a marker attribute used to define that all tests in this assembly
+/// should run in a single partition without other assemblies due to state sharing concerns.
+/// For example, Microsoft.CodeAnalysis.CSharp.EndToEnd.UnitTests
+/// 
+/// RunTests uses this attribute when partitioning tests.
+/// </summary>
+[AttributeUsage(AttributeTargets.Assembly)]
+public sealed class RunTestsInSinglePartitionAttribute : Attribute
+{
+}

src/Tools/PrepareTests/PrepareTests.csproj

@@ -10,5 +10,8 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Mono.Options" Version="$(MonoOptionsVersion)" />
+    <PackageReference Include="Microsoft.TestPlatform.TranslationLayer" Version="$(MicrosoftTestPlatformTranslationLayerVersion)" />
+    <PackageReference Include="Microsoft.TestPlatform.ObjectModel" Version="$(MicrosoftTestPlatformObjectModelVersion)" />
+    <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
   </ItemGroup>
 </Project>

src/Tools/PrepareTests/Program.cs

@@ -5,6 +5,8 @@
 using System;
 using Mono.Options;
 
+namespace PrepareTests;
+
 internal static class Program
 {
     internal const int ExitFailure = 1;
@@ -15,12 +17,14 @@ public static int Main(string[] args)
         string? source = null;
         string? destination = null;
         bool isUnix = false;
+        string? dotnetPath = null;
 
         var options = new OptionSet()
         {
             { "source=", "Path to binaries", (string s) => source = s },
             { "destination=", "Output path", (string s) => destination = s },
-            { "unix", "If true, prepares tests for unix environment instead of Windows", o => isUnix = o is object }
+            { "unix", "If true, prepares tests for unix environment instead of Windows", o => isUnix = o is object },
+            { "dotnetPath=", "Path to the dotnet CLI", (string s) => dotnetPath = s },
         };
         options.Parse(args);
 
@@ -36,6 +40,14 @@ public static int Main(string[] args)
             return ExitFailure;
         }
 
+        if (dotnetPath is null)
+        {
+            Console.Error.WriteLine("--dotnetPath argument must be provided");
+            return ExitFailure;
+        }
+
+        TestDiscovery.RunDiscovery(source, dotnetPath, isUnix);
+
         MinimizeUtil.Run(source, destination, isUnix);
         return ExitSuccess;
     }

src/Tools/PrepareTests/TestDiscovery.cs

@@ -0,0 +1,127 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using System.Diagnostics.Contracts;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using Microsoft.TestPlatform.VsTestConsole.TranslationLayer;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
+
+namespace PrepareTests;
+internal class TestDiscovery
+{
+    public static void RunDiscovery(string repoRootDirectory, string dotnetPath, bool isUnix)
+    {
+        var binDirectory = Path.Combine(repoRootDirectory, "artifacts", "bin");
+        var assemblies = GetAssemblies(binDirectory, isUnix);
+
+        Console.WriteLine($"Found {assemblies.Count} test assemblies");
+
+        var vsTestConsole = Directory.EnumerateFiles(Path.Combine(Path.GetDirectoryName(dotnetPath)!, "sdk"), "vstest.console.dll", SearchOption.AllDirectories).OrderBy(s => s).Last();
+
+        var vstestConsoleWrapper = new VsTestConsoleWrapper(vsTestConsole, new ConsoleParameters
+        {
+            LogFilePath = Path.Combine(repoRootDirectory, "logs", "test_discovery_logs.txt"),
+            TraceLevel = TraceLevel.Error,
+        });
+
+        var discoveryHandler = new DiscoveryHandler();
+
+        var stopwatch = new Stopwatch();
+        stopwatch.Start();
+        vstestConsoleWrapper.DiscoverTests(assemblies, @"<RunSettings><RunConfiguration><MaxCpuCount>0</MaxCpuCount></RunConfiguration></RunSettings>", discoveryHandler);
+        stopwatch.Stop();
+
+        var tests = discoveryHandler.GetTests();
+
+        Console.WriteLine($"Discovered {tests.Length} tests in {stopwatch.Elapsed}");
+
+        stopwatch.Restart();
+        var testGroupedByAssembly = tests.GroupBy(test => test.Source);
+        foreach (var assemblyGroup in testGroupedByAssembly)
+        {
+            var directory = Path.GetDirectoryName(assemblyGroup.Key);
+
+            // Tests with combinatorial data are output multiple times with the same fully qualified test name.
+            // We only need to include it once as run all combinations under the same filter.
+            var testToWrite = assemblyGroup.Select(test => test.FullyQualifiedName).Distinct().ToList();
+
+            using var fileStream = File.Create(Path.Combine(directory!, "testlist.json"));
+            JsonSerializer.Serialize(fileStream, testToWrite);
+        }
+        stopwatch.Stop();
+        Console.WriteLine($"Serialized tests in {stopwatch.Elapsed}");
+    }
+
+    private class DiscoveryHandler : ITestDiscoveryEventsHandler
+    {
+        private readonly ConcurrentBag<TestCase> _tests = new();
+        private bool _isComplete = false;
+
+        public void HandleDiscoveredTests(IEnumerable<TestCase>? discoveredTestCases)
+        {
+            if (discoveredTestCases != null)
+            {
+                foreach (var test in discoveredTestCases)
+                {
+                    _tests.Add(test);
+                }
+            }
+        }
+
+        public void HandleDiscoveryComplete(long totalTests, IEnumerable<TestCase>? lastChunk, bool isAborted)
+        {
+            if (lastChunk != null)
+            {
+                foreach (var test in lastChunk)
+                {
+                    _tests.Add(test);
+                }
+            }
+
+            _isComplete = true;
+        }
+
+        public void HandleLogMessage(TestMessageLevel level, string? message)
+        {
+            Console.WriteLine(message);
+        }
+
+        public void HandleRawMessage(string rawMessage)
+        {
+        }
+
+        public ImmutableArray<TestCase> GetTests()
+        {
+            Contract.Assert(_isComplete);
+            return _tests.ToImmutableArray();
+        }
+    }
+
+    private static List<string> GetAssemblies(string binDirectory, bool isUnix)
+    {
+        var unitTestAssemblies = Directory.GetFiles(binDirectory, "*.UnitTests.dll", SearchOption.AllDirectories).Where(ShouldInclude);
+        return unitTestAssemblies.ToList();
+
+        bool ShouldInclude(string path)
+        {
+            if (isUnix)
+            {
+                // Our unix build will build net framework dlls for multi-targeted projects.
+                // These are not valid testing on unix and discovery will throw if we try.
+                return Path.GetFileName(Path.GetDirectoryName(path)) != "net472";
+            }
+
+            return true;
+        }
+    }
+}

src/Tools/Source/RunTests/AssemblyInfo.cs

@@ -0,0 +1,30 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Immutable;
+using System.IO;
+
+namespace RunTests;
+
+public readonly record struct AssemblyInfo(string AssemblyPath) : IComparable<AssemblyInfo>
+{
+    public string AssemblyName => Path.GetFileName(AssemblyPath);
+
+    public int CompareTo(AssemblyInfo other)
+    {
+        // Ensure we have a consistent ordering by ordering by assembly path.
+        return string.Compare(this.AssemblyPath, other.AssemblyPath, StringComparison.Ordinal);
+    }
+}
+
+public readonly record struct TypeInfo(string Name, string FullyQualifiedName, ImmutableArray<TestMethodInfo> Tests)
+{
+    public override string ToString() => $"[Type]{FullyQualifiedName}";
+}
+
+public readonly record struct TestMethodInfo(string Name, string FullyQualifiedName, TimeSpan ExecutionTime)
+{
+    public override string ToString() => $"[Method]{FullyQualifiedName}";
+}