Benchmark tests

.github/workflows/benchmark.yml

@@ -0,0 +1,37 @@
+name: Benchmark tests
+on: [pull_request]
+
+permissions:
+    contents: write
+    deployments: write
+
+jobs:
+  benchmark:
+    name: Performance regression check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.x'
+      - name: Run benchmark
+        run: cd src/FluentAssertions.Analyzers.BenchmarkTests && dotnet run -c Release --exporters json --filter '*'
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'gobenchmarkdotnet'
+          # Where the output from the benchmark tool is stored
+          output-file-path: output.txt
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          # Workflow will fail when an alert happens
+          fail-on-alert: true

.github/workflows/ci.yml

@@ -14,5 +14,6 @@ jobs:
         uses: actions/setup-dotnet@v3
         with:
           dotnet-version: 6.0.x
+      - run: dotnet build src
       - run: dotnet test src --configuration Release --filter 'TestCategory=Completed'
       - run: dotnet pack src/FluentAssertions.Analyzers/FluentAssertions.Analyzers.csproj

src/FluentAssertions.Analyzers.BenchmarkTests/FluentAssertions.Analyzers.BenchmarkTests.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <OutputType>Exe</OutputType>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.5" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FluentAssertions.Analyzers.TestUtils\FluentAssertions.Analyzers.TestUtils.csproj" />
+  </ItemGroup>
+
+</Project>

src/FluentAssertions.Analyzers.BenchmarkTests/FluentAssertionsBenchmarks.cs

@@ -0,0 +1,53 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Jobs;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Diagnostics;
+using System.Collections.Immutable;
+using FluentAssertions.Analyzers.TestUtils;
+using System.Threading.Tasks;
+using System;
+using System.Collections.Generic;
+
+namespace FluentAssertions.Analyzers.BenchmarkTests
+{
+    [SimpleJob(RuntimeMoniker.Net472, baseline: true)]
+    [SimpleJob(RuntimeMoniker.NetCoreApp31)]
+    [SimpleJob(RuntimeMoniker.Net60)]
+    [SimpleJob(RuntimeMoniker.Net70)]
+    [RPlotExporter]
+    public class FluentAssertionsBenchmarks
+    {
+        private CompilationWithAnalyzers MinimalCompiliation;
+
+        [GlobalSetup]
+        public async Task GlobalSetup()
+        {
+            MinimalCompiliation = await CreateCompilationFromAsync(GenerateCode.ObjectStatement("actual.Should().Equals(expected);"));
+        }
+
+        [Benchmark]
+        public Task MinimalCompilation()
+        {
+            return MinimalCompiliation.GetAnalyzerDiagnosticsAsync();
+        }
+
+        private async Task<CompilationWithAnalyzers> CreateCompilationFromAsync(params string[] sources)
+        {
+            var project = CsProjectGenerator.CreateProject(sources);
+            var compilation = await project.GetCompilationAsync();
+
+            if (compilation is null)
+            {
+                throw new InvalidOperationException("Compilation is null");
+            }
+
+            return compilation.WithOptions(compilation.Options.WithSpecificDiagnosticOptions(new Dictionary<string, ReportDiagnostic>
+            {
+                ["CS1701"] = ReportDiagnostic.Suppress, // Binding redirects
+                ["CS1702"] = ReportDiagnostic.Suppress,
+                ["CS1705"] = ReportDiagnostic.Suppress,
+                ["CS8019"] = ReportDiagnostic.Suppress // TODO: Unnecessary using directive
+            })).WithAnalyzers(CodeAnalyzersUtils.GetAllAnalyzers().ToImmutableArray());
+        }
+    }
+}

src/FluentAssertions.Analyzers.BenchmarkTests/Program.cs

@@ -0,0 +1,10 @@
+using BenchmarkDotNet.Running;
+
+namespace FluentAssertions.Analyzers.BenchmarkTests
+{
+    public class Program
+    {
+        public static void Main() 
+            => BenchmarkDotNet.Running.BenchmarkRunner.Run<FluentAssertionsBenchmarks>();
+    }
+}

src/FluentAssertions.Analyzers.TestUtils/CodeAnalyzersUtils.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Linq;
+using Microsoft.CodeAnalysis.Diagnostics;
+
+namespace FluentAssertions.Analyzers.TestUtils
+{
+    public class CodeAnalyzersUtils
+    {
+        private static readonly DiagnosticAnalyzer[] AllAnalyzers = CreateAllAnalyzers();
+
+        public static DiagnosticAnalyzer[] GetAllAnalyzers() => AllAnalyzers;
+
+        private static DiagnosticAnalyzer[] CreateAllAnalyzers()
+        {
+            var assembly = typeof(Constants).Assembly;
+            var analyzersTypes = assembly.GetTypes()
+                .Where(type => !type.IsAbstract && typeof(DiagnosticAnalyzer).IsAssignableFrom(type));
+            var analyzers = analyzersTypes.Select(type => (DiagnosticAnalyzer)Activator.CreateInstance(type));
+
+            return analyzers.ToArray();
+        }
+    }
+}

src/FluentAssertions.Analyzers.TestUtils/CsProjectGenerator.cs

@@ -0,0 +1,133 @@
+using System;
+using System.Collections.Immutable;
+using System.Linq;
+using System.Reflection;
+using FluentAssertions.Execution;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Text;
+
+using XunitAssert = Xunit.Assert;
+
+namespace FluentAssertions.Analyzers.TestUtils
+{
+    public class CsProjectGenerator
+    {
+        static CsProjectGenerator()
+        {
+            References = new[]
+            {
+                typeof(object), // System.Private.CoreLib
+                typeof(Console), // System
+                typeof(Enumerable), // System.Linq
+                typeof(CSharpCompilation), // Microsoft.CodeAnalysis.CSharp
+                typeof(Compilation), // Microsoft.CodeAnalysis
+                typeof(AssertionScope), // FluentAssertions.Core
+                typeof(AssertionExtensions), // FluentAssertions
+                typeof(Microsoft.VisualStudio.TestTools.UnitTesting.Assert), // MsTest
+                typeof(XunitAssert), // Xunit
+            }.Select(type => type.GetTypeInfo().Assembly.Location)
+            .Append(GetSystemAssemblyPathByName("System.Globalization.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Text.RegularExpressions.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Runtime.Extensions.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Data.Common.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Threading.Tasks.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Runtime.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Reflection.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Xml.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Xml.XDocument.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Private.Xml.Linq.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Linq.Expressions.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Collections.dll"))
+            .Append(GetSystemAssemblyPathByName("netstandard.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Xml.ReaderWriter.dll"))
+            .Append(GetSystemAssemblyPathByName("System.Private.Xml.dll"))
+            .Select(location => (MetadataReference)MetadataReference.CreateFromFile(location))
+            .ToImmutableArray();
+
+            string GetSystemAssemblyPathByName(string assemblyName)
+            {
+                var root = System.IO.Path.GetDirectoryName(typeof(object).Assembly.Location);
+                return System.IO.Path.Combine(root, assemblyName);
+            }
+        }
+        // based on http://code.fitness/post/2017/02/using-csharpscript-with-netstandard.html
+        public static string GetSystemAssemblyPathByName(string assemblyName)
+        {
+            var root = System.IO.Path.GetDirectoryName(typeof(object).Assembly.Location);
+            return System.IO.Path.Combine(root, assemblyName);
+        }
+
+        private static readonly ImmutableArray<MetadataReference> References;
+
+        private static readonly string DefaultFilePathPrefix = "Test";
+        private static readonly string CSharpDefaultFileExt = "cs";
+        private static readonly string VisualBasicDefaultExt = "vb";
+        private static readonly string TestProjectName = "TestProject";
+
+        /// <summary>
+        /// Given an array of strings as sources and a language, turn them into a project and return the documents and spans of it.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Tuple containing the Documents produced from the sources and their TextSpans if relevant</returns>
+        public static Document[] GetDocuments(string[] sources, string language)
+        {
+            if (language != LanguageNames.CSharp && language != LanguageNames.VisualBasic)
+            {
+                throw new ArgumentException("Unsupported Language");
+            }
+
+            var project = CreateProject(sources, language);
+            var documents = project.Documents.ToArray();
+
+            if (sources.Length != documents.Length)
+            {
+                throw new SystemException("Amount of sources did not match amount of Documents created");
+            }
+
+            return documents;
+        }
+
+        /// <summary>
+        /// Create a Document from a string through creating a project that contains it.
+        /// </summary>
+        /// <param name="source">Classes in the form of a string</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Document created from the source string</returns>
+        public static Document CreateDocument(string source, string language = LanguageNames.CSharp)
+        {
+            return CreateProject(new[] { source }, language).Documents.First();
+        }
+
+        /// <summary>
+        /// Create a project using the inputted strings as sources.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Project created out of the Documents created from the source strings</returns>
+        public static Project CreateProject(string[] sources, string language = LanguageNames.CSharp)
+        {
+            string fileNamePrefix = DefaultFilePathPrefix;
+            string fileExt = language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt;
+
+            var projectId = ProjectId.CreateNewId(debugName: TestProjectName);
+
+            var solution = new AdhocWorkspace()
+                .CurrentSolution
+                .AddProject(projectId, TestProjectName, TestProjectName, language)
+                .AddMetadataReferences(projectId, References);
+
+            int count = 0;
+            foreach (var source in sources)
+            {
+                var newFileName = fileNamePrefix + count + "." + fileExt;
+                var documentId = DocumentId.CreateNewId(projectId, debugName: newFileName);
+                solution = solution.AddDocument(documentId, newFileName, SourceText.From(source));
+                count++;
+            }
+            return solution.GetProject(projectId);
+        }
+    }
+}
+

src/FluentAssertions.Analyzers.TestUtils/FluentAssertions.Analyzers.TestUtils.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="FluentAssertions" Version="6.1.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.0.0" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.0.0" />
+    <PackageReference Include="xunit.assert" Version="2.4.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FluentAssertions.Analyzers\FluentAssertions.Analyzers.csproj" />
+  </ItemGroup>
+
+</Project>

src/FluentAssertions.Analyzers.Tests/GenerateCode.cs → src/FluentAssertions.Analyzers.TestUtils/GenerateCode.cs

@@ -1,7 +1,7 @@
 using System;
 using System.Text;
 
-namespace FluentAssertions.Analyzers.Tests
+namespace FluentAssertions.Analyzers.TestUtils
 {
     public static class GenerateCode
     {