Benchmark tests

.github/workflows/benchmark.yml

@@ -0,0 +1,31 @@
+name: Benchmark tests
+on: [pull_request]
+
+permissions:
+    contents: write
+    deployments: write
+
+jobs:
+  benchmark:
+    name: Performance regression check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.x'
+      - name: Run benchmark
+        run: cd src/FluentAssertions.Analyzers.BenchmarkTests && dotnet run -c Release --exporters json --filter '*'
+
+      - name: Store benchmark result
+        uses: rhysd/github-action-benchmark@v1
+        with:
+          name: FluentAssertions.Analyzers Benchmark
+          tool: 'benchmarkdotnet'
+          output-file-path: src/FluentAssertions.Analyzers.BenchmarkTests/BenchmarkDotNet.Artifacts/results/FluentAssertions.Analyzers.BenchmarkTests.FluentAssertionsBenchmarks-report.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true

.github/workflows/ci.yml

@@ -21,4 +21,6 @@ jobs:
         uses: actions/setup-dotnet@v3
         with:
           dotnet-version: 6.0.x
-      - run: dotnet test --configuration ${{ matrix.config }} --filter 'TestCategory=Completed' /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura
+      - run: dotnet build
+      - run: dotnet test --configuration Release --filter 'TestCategory=Completed' /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura
+      - run: dotnet pack src/FluentAssertions.Analyzers/FluentAssertions.Analyzers.csproj

FluentAssertions.Analyzers.sln

@@ -1,4 +1,4 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
 VisualStudioVersion = 15.0.27004.2002
 MinimumVisualStudioVersion = 10.0.40219.1
@@ -15,6 +15,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionItems", "SolutionIt
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FluentAssertions.Analyzers", "src\FluentAssertions.Analyzers\FluentAssertions.Analyzers.csproj", "{3BA672F7-00D8-4E77-89A0-D46DD99D35AA}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FluentAssertions.Analyzers.TestUtils", "src\FluentAssertions.Analyzers.TestUtils\FluentAssertions.Analyzers.TestUtils.csproj", "{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FluentAssertions.Analyzers.BenchmarkTests", "src\FluentAssertions.Analyzers.BenchmarkTests\FluentAssertions.Analyzers.BenchmarkTests.csproj", "{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -29,6 +33,14 @@ Global
 		{3BA672F7-00D8-4E77-89A0-D46DD99D35AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3BA672F7-00D8-4E77-89A0-D46DD99D35AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3BA672F7-00D8-4E77-89A0-D46DD99D35AA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BD9FC8CC-C23D-4ECC-A926-4BE35C78D338}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FE6D8A05-1383-4BCD-AD65-2EF741E48F44}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

README.md

@@ -42,6 +42,10 @@ dotnet build
 dotnet test --configuration Release  --filter 'TestCategory=Completed'
 ```
 
+### Benchmarks
+
+https://fluentassertions.github.io/fluentassertions.analyzers/dev/bench/
+
 ## Example Usages
 - https://github.com/SonarSource/sonar-dotnet/pull/2072
 - https://github.com/microsoft/component-detection/pull/634

src/FluentAssertions.Analyzers.BenchmarkTests/FluentAssertions.Analyzers.BenchmarkTests.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <OutputType>Exe</OutputType>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.5" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FluentAssertions.Analyzers.TestUtils\FluentAssertions.Analyzers.TestUtils.csproj" />
+  </ItemGroup>
+
+</Project>

src/FluentAssertions.Analyzers.BenchmarkTests/FluentAssertionsBenchmarks.cs

@@ -0,0 +1,51 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Jobs;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Diagnostics;
+using System.Collections.Immutable;
+using FluentAssertions.Analyzers.TestUtils;
+using System.Threading.Tasks;
+using System;
+using System.Collections.Generic;
+
+namespace FluentAssertions.Analyzers.BenchmarkTests
+{
+    [SimpleJob(RuntimeMoniker.Net60, baseline: true)]
+    [SimpleJob(RuntimeMoniker.Net70)]
+    [JsonExporter]
+    public class FluentAssertionsBenchmarks
+    {
+        private CompilationWithAnalyzers MinimalCompiliation;
+
+        [GlobalSetup]
+        public async Task GlobalSetup()
+        {
+            MinimalCompiliation = await CreateCompilationFromAsync(GenerateCode.ObjectStatement("actual.Should().Equals(expected);"));
+        }
+
+        [Benchmark]
+        public Task MinimalCompilation()
+        {
+            return MinimalCompiliation.GetAnalyzerDiagnosticsAsync();
+        }
+
+        private async Task<CompilationWithAnalyzers> CreateCompilationFromAsync(params string[] sources)
+        {
+            var project = CsProjectGenerator.CreateProject(sources);
+            var compilation = await project.GetCompilationAsync();
+
+            if (compilation is null)
+            {
+                throw new InvalidOperationException("Compilation is null");
+            }
+
+            return compilation.WithOptions(compilation.Options.WithSpecificDiagnosticOptions(new Dictionary<string, ReportDiagnostic>
+            {
+                ["CS1701"] = ReportDiagnostic.Suppress, // Binding redirects
+                ["CS1702"] = ReportDiagnostic.Suppress,
+                ["CS1705"] = ReportDiagnostic.Suppress,
+                ["CS8019"] = ReportDiagnostic.Suppress // TODO: Unnecessary using directive
+            })).WithAnalyzers(CodeAnalyzersUtils.GetAllAnalyzers().ToImmutableArray());
+        }
+    }
+}

src/FluentAssertions.Analyzers.BenchmarkTests/Program.cs

@@ -0,0 +1,10 @@
+using BenchmarkDotNet.Running;
+
+namespace FluentAssertions.Analyzers.BenchmarkTests
+{
+    public class Program
+    {
+        public static void Main() 
+            => BenchmarkDotNet.Running.BenchmarkRunner.Run<FluentAssertionsBenchmarks>();
+    }
+}

src/FluentAssertions.Analyzers.TestUtils/CodeAnalyzersUtils.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Linq;
+using Microsoft.CodeAnalysis.Diagnostics;
+
+namespace FluentAssertions.Analyzers.TestUtils
+{
+    public class CodeAnalyzersUtils
+    {
+        private static readonly DiagnosticAnalyzer[] AllAnalyzers = CreateAllAnalyzers();
+
+        public static DiagnosticAnalyzer[] GetAllAnalyzers() => AllAnalyzers;
+
+        private static DiagnosticAnalyzer[] CreateAllAnalyzers()
+        {
+            var assembly = typeof(Constants).Assembly;
+            var analyzersTypes = assembly.GetTypes()
+                .Where(type => !type.IsAbstract && typeof(DiagnosticAnalyzer).IsAssignableFrom(type));
+            var analyzers = analyzersTypes.Select(type => (DiagnosticAnalyzer)Activator.CreateInstance(type));
+
+            return analyzers.ToArray();
+        }
+    }
+}

src/FluentAssertions.Analyzers.TestUtils/CsProjectGenerator.cs

@@ -0,0 +1,141 @@
+using System;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reflection;
+using FluentAssertions.Execution;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Text;
+
+using XunitAssert = Xunit.Assert;
+using System.Net.Http;
+using System.Collections.Concurrent;
+using System.Collections.ObjectModel;
+
+namespace FluentAssertions.Analyzers.TestUtils
+{
+    public class CsProjectGenerator
+    {
+        static CsProjectGenerator()
+        {
+            References = new[]
+            {
+                typeof(object), // System.Private.CoreLib
+                typeof(Console), // System
+                typeof(Uri), // System.Private.Uri
+                typeof(Enumerable), // System.Linq
+                typeof(CSharpCompilation), // Microsoft.CodeAnalysis.CSharp
+                typeof(Compilation), // Microsoft.CodeAnalysis
+                typeof(AssertionScope), // FluentAssertions.Core
+                typeof(AssertionExtensions), // FluentAssertions
+                typeof(HttpRequestMessage), // System.Net.Http
+                typeof(ImmutableArray), // System.Collections.Immutable
+                typeof(ConcurrentBag<>), // System.Collections.Concurrent
+                typeof(ReadOnlyDictionary<,>), // System.ObjectModel
+                typeof(Microsoft.VisualStudio.TestTools.UnitTesting.Assert), // MsTest
+                typeof(XunitAssert), // Xunit
+            }.Select(type => type.GetTypeInfo().Assembly.Location)
+            .Append(GetSystemAssemblyPathByName("System.Globalization.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Text.RegularExpressions.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Runtime.Extensions.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Data.Common.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Threading.Tasks.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Runtime.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Reflection.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Xml.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Xml.XDocument.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Private.Xml.Linq.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Linq.Expressions.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Collections.dll"))
+            .Append(GetSystemAssemblyPathByName("netstandard.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Xml.ReaderWriter.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Private.Xml.dll"))
+            .Select(location => (MetadataReference)MetadataReference.CreateFromFile(location))
+            .ToImmutableArray();
+
+            string GetSystemAssemblyPathByName(string assemblyName)
+            {
+                var root = System.IO.Path.GetDirectoryName(typeof(object).Assembly.Location);
+                return System.IO.Path.Combine(root, assemblyName);
+            }
+        }
+        // based on http://code.fitness/post/2017/02/using-csharpscript-with-netstandard.html
+        public static string GetSystemAssemblyPathByName(string assemblyName)
+        {
+            var root = System.IO.Path.GetDirectoryName(typeof(object).Assembly.Location);
+            return System.IO.Path.Combine(root, assemblyName);
+        }
+
+        private static readonly ImmutableArray<MetadataReference> References;
+
+        private static readonly string DefaultFilePathPrefix = "Test";
+        private static readonly string CSharpDefaultFileExt = "cs";
+        private static readonly string VisualBasicDefaultExt = "vb";
+        private static readonly string TestProjectName = "TestProject";
+
+        /// <summary>
+        /// Given an array of strings as sources and a language, turn them into a project and return the documents and spans of it.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Tuple containing the Documents produced from the sources and their TextSpans if relevant</returns>
+        public static Document[] GetDocuments(string[] sources, string language)
+        {
+            if (language != LanguageNames.CSharp && language != LanguageNames.VisualBasic)
+            {
+                throw new ArgumentException("Unsupported Language");
+            }
+
+            var project = CreateProject(sources, language);
+            var documents = project.Documents.ToArray();
+
+            if (sources.Length != documents.Length)
+            {
+                throw new SystemException("Amount of sources did not match amount of Documents created");
+            }
+
+            return documents;
+        }
+
+        /// <summary>
+        /// Create a Document from a string through creating a project that contains it.
+        /// </summary>
+        /// <param name="source">Classes in the form of a string</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Document created from the source string</returns>
+        public static Document CreateDocument(string source, string language = LanguageNames.CSharp)
+        {
+            return CreateProject(new[] { source }, language).Documents.First();
+        }
+
+        /// <summary>
+        /// Create a project using the inputted strings as sources.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Project created out of the Documents created from the source strings</returns>
+        public static Project CreateProject(string[] sources, string language = LanguageNames.CSharp)
+        {
+            string fileNamePrefix = DefaultFilePathPrefix;
+            string fileExt = language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt;
+
+            var projectId = ProjectId.CreateNewId(debugName: TestProjectName);
+
+            var solution = new AdhocWorkspace()
+                .CurrentSolution
+                .AddProject(projectId, TestProjectName, TestProjectName, language)
+                .AddMetadataReferences(projectId, References);
+
+            int count = 0;
+            foreach (var source in sources)
+            {
+                var newFileName = fileNamePrefix + count + "." + fileExt;
+                var documentId = DocumentId.CreateNewId(projectId, debugName: newFileName);
+                solution = solution.AddDocument(documentId, newFileName, SourceText.From(source));
+                count++;
+            }
+            return solution.GetProject(projectId);
+        }
+    }
+}
+

src/FluentAssertions.Analyzers.TestUtils/FluentAssertions.Analyzers.TestUtils.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="FluentAssertions" Version="6.1.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.0.0" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.0.0" />
+    <PackageReference Include="xunit.assert" Version="2.4.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FluentAssertions.Analyzers\FluentAssertions.Analyzers.csproj" />
+  </ItemGroup>
+
+</Project>

src/FluentAssertions.Analyzers.Tests/GenerateCode.cs → src/FluentAssertions.Analyzers.TestUtils/GenerateCode.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Text;
 
-namespace FluentAssertions.Analyzers.Tests
+namespace FluentAssertions.Analyzers.TestUtils
 {
     public static class GenerateCode
     {