diff --git a/tools/code/aspire/Program.cs b/tools/code/aspire/Program.cs new file mode 100644 index 00000000..e4b493f6 --- /dev/null +++ b/tools/code/aspire/Program.cs @@ -0,0 +1,15 @@ +using Aspire.Hosting; +using Projects; + +internal static class Program +{ + private static void Main(string[] args) + { + var builder = DistributedApplication.CreateBuilder(args); + + builder.AddProject("integration-tests"); + //.WithEnvironment("CSCHECK_SEED", "0000KOIPe036"); + + builder.Build().Run(); + } +} \ No newline at end of file diff --git a/tools/code/aspire/Properties/launchSettings.json b/tools/code/aspire/Properties/launchSettings.json new file mode 100644 index 00000000..88bd7039 --- /dev/null +++ b/tools/code/aspire/Properties/launchSettings.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17016;http://localhost:15029", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21043", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22139", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:15029", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19296", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20230", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + } + } +} diff --git a/tools/code/aspire/appsettings.Development.json b/tools/code/aspire/appsettings.Development.json new file mode 100644 index 00000000..0c208ae9 --- /dev/null +++ b/tools/code/aspire/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/tools/code/aspire/aspire.csproj b/tools/code/aspire/aspire.csproj new file mode 100644 index 00000000..71e7c345 --- /dev/null +++ b/tools/code/aspire/aspire.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + enable + true + 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 + + + + + + + + + + + diff --git a/tools/code/code.sln b/tools/code/code.sln index 989c524b..8dfdd6e5 100644 --- a/tools/code/code.sln +++ b/tools/code/code.sln @@ -11,7 +11,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "publisher", "publisher\publ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "common.tests", "common.tests\common.tests.csproj", "{D1ED4FC5-8C52-4380-8520-BC46F475A4F8}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "integration.tests", "integration.tests\integration.tests.csproj", "{76339A55-86A2-4CBE-991B-515DED712804}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "integration.tests", "integration.tests\integration.tests.csproj", "{05923D65-3595-445A-8EB3-5ECC1F99A727}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "aspire", "aspire\aspire.csproj", "{C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -35,10 +37,14 @@ Global {D1ED4FC5-8C52-4380-8520-BC46F475A4F8}.Debug|Any CPU.Build.0 = Debug|Any CPU {D1ED4FC5-8C52-4380-8520-BC46F475A4F8}.Release|Any CPU.ActiveCfg = Release|Any CPU {D1ED4FC5-8C52-4380-8520-BC46F475A4F8}.Release|Any CPU.Build.0 = Release|Any CPU - {76339A55-86A2-4CBE-991B-515DED712804}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {76339A55-86A2-4CBE-991B-515DED712804}.Debug|Any CPU.Build.0 = Debug|Any CPU - {76339A55-86A2-4CBE-991B-515DED712804}.Release|Any CPU.ActiveCfg = Release|Any CPU - {76339A55-86A2-4CBE-991B-515DED712804}.Release|Any CPU.Build.0 = Release|Any CPU + {05923D65-3595-445A-8EB3-5ECC1F99A727}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05923D65-3595-445A-8EB3-5ECC1F99A727}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05923D65-3595-445A-8EB3-5ECC1F99A727}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05923D65-3595-445A-8EB3-5ECC1F99A727}.Release|Any CPU.Build.0 = Release|Any CPU + {C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/tools/code/common.tests/common.tests.csproj b/tools/code/common.tests/common.tests.csproj index a33f307b..d5beb1ff 100644 --- a/tools/code/common.tests/common.tests.csproj +++ b/tools/code/common.tests/common.tests.csproj @@ -2,15 +2,16 @@ net8.0 + false true - latest-all + 8-all CA1034,CA1062,CA1724,CA2007,CA1848,CA1716 enable - + diff --git a/tools/code/common/OpenTelemetry.cs b/tools/code/common/OpenTelemetry.cs new file mode 100644 index 00000000..6a874bcb --- /dev/null +++ b/tools/code/common/OpenTelemetry.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Instrumentation.Http; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using System.Diagnostics; + +namespace common; + +public static class OpenTelemetryServices +{ + public static void Configure(IServiceCollection services) + { + var sourceName = services.BuildServiceProvider().GetService()?.Name ?? "ApiOps.*"; + + services.AddOpenTelemetry() + .WithMetrics(metrics => metrics.AddHttpClientInstrumentation() + .AddRuntimeInstrumentation() + .AddMeter("Azure.*")) + .WithTracing(tracing => tracing.AddHttpClientInstrumentation(ConfigureHttpClientTraceInstrumentationOptions) + .AddSource("Azure.*") + .AddSource(sourceName) + .SetSampler()); + + var configuration = services.BuildServiceProvider().GetRequiredService(); + configuration.TryGetValue("OTEL_EXPORTER_OTLP_ENDPOINT") + .Iter(_ => + { + services.AddLogging(builder => builder.AddOpenTelemetry()); + services.Configure(logging => logging.AddOtlpExporter()) + .ConfigureOpenTelemetryMeterProvider(metrics => metrics.AddOtlpExporter()) + .ConfigureOpenTelemetryTracerProvider(tracing => tracing.AddOtlpExporter()); + }); + } + + private static void ConfigureHttpClientTraceInstrumentationOptions(HttpClientTraceInstrumentationOptions options) + { + options.FilterHttpRequestMessage = (_) => Activity.Current?.Parent?.Source?.Name != "Azure.Core.Http"; + } +} diff --git a/tools/code/common/common.csproj b/tools/code/common/common.csproj index 9222e806..eb08ee52 100644 --- a/tools/code/common/common.csproj +++ b/tools/code/common/common.csproj @@ -3,7 +3,8 @@ net8.0 true - latest-all + 8-all + false CA1034,CA1062,CA1724,CA2007,CA1848 enable @@ -16,12 +17,16 @@ - + - + - + + + + + diff --git a/tools/code/extractor/Common.cs b/tools/code/extractor/Common.cs index d0a19133..9fbc3219 100644 --- a/tools/code/extractor/Common.cs +++ b/tools/code/extractor/Common.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.JsonWebTokens; using System; +using System.Diagnostics; using System.IO; using System.Reflection; @@ -20,6 +21,7 @@ internal static class CommonServices { public static void Configure(IServiceCollection services) { + services.AddSingleton(GetActivitySource); services.AddSingleton(GetAzureEnvironment); services.AddSingleton(GetTokenCredential); services.AddSingleton(GetHttpPipeline); @@ -28,8 +30,12 @@ public static void Configure(IServiceCollection services) services.AddSingleton(GetManagementServiceDirectory); services.AddSingleton(GetConfigurationJson); services.AddSingleton(); + OpenTelemetryServices.Configure(services); } + private static ActivitySource GetActivitySource(IServiceProvider provider) => + new("ApiOps.Extractor"); + private static AzureEnvironment GetAzureEnvironment(IServiceProvider provider) { var configuration = provider.GetRequiredService(); diff --git a/tools/code/extractor/extractor.csproj b/tools/code/extractor/extractor.csproj index 659093a1..ed8018c3 100644 --- a/tools/code/extractor/extractor.csproj +++ b/tools/code/extractor/extractor.csproj @@ -2,8 +2,9 @@ net8.0 + false true - latest-all + 8-all CA1708,CA1724,CA1812,CA1848,CA2007,CA1034,CA1062 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 Exe diff --git a/tools/code/integration.tests/Api.cs b/tools/code/integration.tests/Api.cs index 56c110d9..244bc013 100644 --- a/tools/code/integration.tests/Api.cs +++ b/tools/code/integration.tests/Api.cs @@ -6,121 +6,60 @@ using Flurl; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Api -{ - public static Gen GenerateUpdate(ApiModel original) => - from revisions in GenerateRevisionUpdates(original.Revisions, original.Type, original.Name) - select original with - { - Revisions = revisions - }; +internal delegate ValueTask DeleteAllApis(ManagementServiceName serviceName, CancellationToken cancellationToken); - private static Gen> GenerateRevisionUpdates(FrozenSet revisions, ApiType type, ApiName name) - { - var newGen = ApiRevision.GenerateSet(type, name); - var updateGen = (ApiRevision revision) => GenerateRevisionUpdate(revision, type); +internal delegate ValueTask PutApiModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); - return Fixture.GenerateNewSet(revisions, newGen, updateGen); - } +internal delegate ValueTask ValidateExtractedApis(Option> apiNamesOption, Option defaultApiSpecification, Option> versionSetNamesOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); - private static Gen GenerateRevisionUpdate(ApiRevision revision, ApiType type) => - from serviceUri in type is ApiType.Soap or ApiType.WebSocket - ? Gen.Const(revision.ServiceUri) - : ApiRevision.GenerateServiceUri(type) - from description in ApiRevision.GenerateDescription().OptionOf() - select revision with - { - ServiceUri = serviceUri.ValueUnsafe(), - Description = description.ValueUnsafe() - }; +file delegate ValueTask> GetApimApis(ManagementServiceName serviceName, CancellationToken cancellationToken); - public static Gen GenerateOverride(ApiDto original) => - from serviceUrl in (original.Properties.Type ?? original.Properties.ApiType) switch - { - "websocket" or "soap" => Gen.Const(original.Properties.ServiceUrl), - _ => Generator.AbsoluteUri.Select(uri => (string?)uri.ToString()) - } - from revisionDescription in ApiRevision.GenerateDescription().OptionOf() - select new ApiDto() - { - Properties = new ApiDto.ApiCreateOrUpdateProperties - { - ServiceUrl = serviceUrl, - ApiRevisionDescription = revisionDescription.ValueUnsafe() - } - }; +file delegate ValueTask> TryGetApimGraphQlSchema(ApiName name, ManagementServiceName serviceName, CancellationToken cancellationToken); - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.SelectMany(model => model.Revisions.Select((revision, index) => - { - var apiName = GetApiName(model.Name, revision, index); - var dto = GetDto(model.Name, model.Type, model.Path, model.Version, revision); - return (apiName, dto); - })) - .Where(api => ApiName.IsNotRevisioned(api.apiName)) - .ToFrozenDictionary(); +file delegate ValueTask> GetFileApis(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); - private static ApiName GetApiName(ApiName name, ApiRevision revision, int index) +internal delegate ValueTask WriteApiModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedApis(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllApisHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var rootApiName = ApiName.GetRootName(name); + using var _ = activitySource.StartActivity(nameof(DeleteAllApis)); - return index == 0 ? rootApiName : ApiName.GetRevisionedName(rootApiName, revision.Number); + logger.LogInformation("Deleting all APIs in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await ApisUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); } +} - private static ApiDto GetDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => - new ApiDto() - { - Properties = new ApiDto.ApiCreateOrUpdateProperties - { - // APIM sets the description to null when it imports for SOAP APIs. - DisplayName = name.ToString(), - Path = path, - ApiType = type switch - { - ApiType.Http => null, - ApiType.Soap => "soap", - ApiType.GraphQl => null, - ApiType.WebSocket => null, - _ => throw new NotSupportedException() - }, - Type = type switch - { - ApiType.Http => "http", - ApiType.Soap => "soap", - ApiType.GraphQl => "graphql", - ApiType.WebSocket => "websocket", - _ => throw new NotSupportedException() - }, - Protocols = type switch - { - ApiType.Http => ["http", "https"], - ApiType.Soap => ["http", "https"], - ApiType.GraphQl => ["http", "https"], - ApiType.WebSocket => ["ws", "wss"], - _ => throw new NotSupportedException() - }, - ServiceUrl = revision.ServiceUri.ValueUnsafe()?.ToString(), - ApiRevisionDescription = revision.Description.ValueUnsafe(), - ApiRevision = $"{revision.Number.ToInt()}", - ApiVersion = version.Map(version => version.Version).ValueUnsafe(), - ApiVersionSetId = version.Map(version => $"/apiVersionSets/{version.VersionSetName}").ValueUnsafe() - } - }; +file sealed class PutApiModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutApiModels)); - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => + logger.LogInformation("Putting API models in {ServiceName}...", serviceName); await models.IterParallel(async model => { - async ValueTask putRevision(ApiRevision revision) => await Put(model.Name, model.Type, model.Path, model.Version, revision, serviceUri, pipeline, cancellationToken); + var serviceUri = getServiceUri(serviceName); + async ValueTask putRevision(ApiRevision revision) => await Put(model.Name, model.Type, model.Path, model.Version, revision, serviceUri, cancellationToken); // Put first revision to make sure it's the current revision. await model.Revisions.HeadOrNone().IterTask(putRevision); @@ -128,8 +67,9 @@ await models.IterParallel(async model => // Put other revisions await model.Revisions.Skip(1).IterParallel(putRevision, cancellationToken); }, cancellationToken); + } - private static async ValueTask Put(ApiName name, ApiType type, string path, Option version, ApiRevision revision, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private async ValueTask Put(ApiName name, ApiType type, string path, Option version, ApiRevision revision, ManagementServiceUri serviceUri, CancellationToken cancellationToken) { var rootName = ApiName.GetRootName(name); var dto = GetDto(rootName, type, path, version, revision); @@ -147,7 +87,7 @@ private static async ValueTask Put(ApiName name, ApiType type, string path, Opti await ApiTag.Put(revision.Tags, revisionedName, serviceUri, pipeline, cancellationToken); } - private static ApiDto GetPutDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => + private static ApiDto GetDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => new ApiDto() { Properties = new ApiDto.ApiCreateOrUpdateProperties @@ -171,17 +111,6 @@ private static ApiDto GetPutDto(ApiName name, ApiType type, string path, Option< ApiType.WebSocket => "websocket", _ => throw new NotSupportedException() }, - Format = revision.Specification.IsSome - ? type switch - { - ApiType.Http => "openapi+json", - ApiType.Soap => "wsdl", - _ => null - } - : null, - Value = type is ApiType.Http or ApiType.Soap - ? revision.Specification.ValueUnsafe() - : null, Protocols = type switch { ApiType.Http => ["http", "https"], @@ -197,9 +126,190 @@ private static ApiDto GetPutDto(ApiName name, ApiType type, string path, Option< ApiVersionSetId = version.Map(version => $"/apiVersionSets/{version.VersionSetName}").ValueUnsafe() } }; +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await ApisUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class ValidateExtractedApisHandler(ILogger logger, GetApimApis getApimResources, TryGetApimGraphQlSchema tryGetApimGraphQlSchema, GetFileApis getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> apiNamesOption, Option defaultApiSpecification, Option> versionSetNamesOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedApis)); + + logger.LogInformation("Validating extracted APIs in {ServiceName}...", serviceName); + + var expected = await GetExpectedResources(apiNamesOption, versionSetNamesOption, serviceName, cancellationToken); + + await ValidateExtractedInformationFiles(expected, serviceDirectory, cancellationToken); + await ValidateExtractedSpecificationFiles(expected, defaultApiSpecification, serviceName, serviceDirectory, cancellationToken); + } + + private async ValueTask> GetExpectedResources(Option> apiNamesOption, Option> versionSetNamesOption, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var apimResources = await getApimResources(serviceName, cancellationToken); + + return apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, apiNamesOption)) + .WhereValue(dto => ApiModule.TryGetVersionSetName(dto) + .Map(name => ExtractorOptions.ShouldExtract(name, versionSetNamesOption)) + .IfNone(true)); + } + + private async ValueTask ValidateExtractedInformationFiles(IDictionary expectedResources, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = expectedResources.MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + private static string NormalizeDto(ApiDto dto) => + new + { + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Path = dto.Properties.Path ?? string.Empty, + RevisionDescription = dto.Properties.ApiRevisionDescription ?? string.Empty, + Revision = dto.Properties.ApiRevision ?? string.Empty, + ServiceUrl = Uri.TryCreate(dto.Properties.ServiceUrl, UriKind.Absolute, out var uri) + ? uri.RemovePath().ToString() + : string.Empty + }.ToString()!; + + private async ValueTask ValidateExtractedSpecificationFiles(IDictionary expectedResources, Option defaultApiSpecification, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var expected = await expectedResources.ToAsyncEnumerable() + .Choose(async kvp => + { + var name = kvp.Key; + return from specification in await GetExpectedApiSpecification(name, kvp.Value, defaultApiSpecification, serviceName, cancellationToken) + // Skip XML specification files. Sometimes they get extracted, other times they fail. + where specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl) + select (name, specification); + }) + .ToFrozenDictionary(cancellationToken); + + var actual = await ApiModule.ListSpecificationFiles(serviceDirectory, cancellationToken) + .Select(file => (file.Parent.Name, file.Specification)) + // Skip XML specification files. Sometimes they get extracted, other times they fail. + .Where(file => file.Specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl)) + .ToFrozenDictionary(cancellationToken); + + actual.Should().BeEquivalentTo(expected); + } + + private async ValueTask> GetExpectedApiSpecification(ApiName name, ApiDto dto, Option defaultApiSpecification, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + switch (dto.Properties.ApiType ?? dto.Properties.Type) + { + case "graphql": + var specificationContents = await tryGetApimGraphQlSchema(name, serviceName, cancellationToken); + return specificationContents.Map(contents => new ApiSpecification.GraphQl() as ApiSpecification); + case "soap": + return new ApiSpecification.Wsdl(); + case "websocket": + return Option.None; + default: +#pragma warning disable CA1849 // Call async methods when in an async method + return defaultApiSpecification.IfNone(() => new ApiSpecification.OpenApi + { + Format = new OpenApiFormat.Yaml(), + Version = new OpenApiVersion.V3() + }); +#pragma warning restore CA1849 // Call async methods when in an async method + } + } +} + +file sealed class GetApimApisHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetApimApis)); + + logger.LogInformation("Getting APIs from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = ApisUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class TryGetApimGraphQlSchemaHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ApiName name, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(TryGetApimGraphQlSchema)); + + logger.LogInformation("Getting GraphQL schema for {ApiName} from {ServiceName}...", name, serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = ApiUri.From(name, serviceUri); + + return await uri.TryGetGraphQlSchema(pipeline, cancellationToken); + } +} + +file sealed class GetFileApisHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileApis)); + + logger.LogInformation("Getting apis from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => ApiInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ApiInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileApis)); + + logger.LogInformation("Getting apis from {ServiceDirectory}...", serviceDirectory); + + return await ApiModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteApiModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteApiModels)); + + logger.LogInformation("Writing api models to {ServiceDirectory}...", serviceDirectory); + await models.IterParallel(async model => + { + await WriteRevisionArtifacts(model, serviceDirectory, cancellationToken); + }, cancellationToken); + } public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => await models.IterParallel(async model => @@ -228,6 +338,13 @@ private static async ValueTask WriteInformationFile(ApiName name, ApiType type, await informationFile.WriteDto(dto, cancellationToken); } + private static ApiName GetApiName(ApiName name, ApiRevision revision, int index) + { + var rootApiName = ApiName.GetRootName(name); + + return index == 0 ? rootApiName : ApiName.GetRevisionedName(rootApiName, revision.Number); + } + private static async ValueTask WriteSpecificationFile(ApiName name, ApiType type, ApiRevision revision, int index, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { var specificationOption = from contents in revision.Specification @@ -253,72 +370,65 @@ await specificationOption.IterTask(async x => }); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, Option defaultApiSpecification, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var expectedResources = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)); - - await ValidateExtractedInformationFiles(expectedResources, serviceDirectory, cancellationToken); - await ValidateExtractedSpecificationFiles(expectedResources, defaultApiSpecification, serviceDirectory, serviceUri, pipeline, cancellationToken); - - await expectedResources.Keys.IterParallel(async name => + private static ApiDto GetDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => + new ApiDto() { - await ApiPolicy.ValidateExtractedArtifacts(serviceDirectory, name, serviceUri, pipeline, cancellationToken); - await ApiTag.ValidateExtractedArtifacts(serviceDirectory, name, serviceUri, pipeline, cancellationToken); - }, cancellationToken); - } - - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = ApisUri.From(serviceUri); - - return await uri.List(pipeline, cancellationToken) - .Select(api => - { - var normalizedName = NormalizeName(api.Name, api.Dto); - return (normalizedName, api.Dto); - }) - .ToFrozenDictionary(cancellationToken); - } + Properties = new ApiDto.ApiCreateOrUpdateProperties + { + // APIM sets the description to null when it imports for SOAP APIs. + DisplayName = name.ToString(), + Path = path, + ApiType = type switch + { + ApiType.Http => null, + ApiType.Soap => "soap", + ApiType.GraphQl => null, + ApiType.WebSocket => null, + _ => throw new NotSupportedException() + }, + Type = type switch + { + ApiType.Http => "http", + ApiType.Soap => "soap", + ApiType.GraphQl => "graphql", + ApiType.WebSocket => "websocket", + _ => throw new NotSupportedException() + }, + Protocols = type switch + { + ApiType.Http => ["http", "https"], + ApiType.Soap => ["http", "https"], + ApiType.GraphQl => ["http", "https"], + ApiType.WebSocket => ["ws", "wss"], + _ => throw new NotSupportedException() + }, + ServiceUrl = revision.ServiceUri.ValueUnsafe()?.ToString(), + ApiRevisionDescription = revision.Description.ValueUnsafe(), + ApiRevision = $"{revision.Number.ToInt()}", + ApiVersion = version.Map(version => version.Version).ValueUnsafe(), + ApiVersionSetId = version.Map(version => $"/apiVersionSets/{version.VersionSetName}").ValueUnsafe() + } + }; +} - // APIM has an issue where it sometimes returns duplicate API names. - private static ApiName NormalizeName(ApiName name, ApiDto dto) +file sealed class ValidatePublishedApisHandler(ILogger logger, GetFileApis getFileResources, GetApimApis getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - if (dto.Properties.IsCurrent is true) - { - return name; - } - - if (ApiName.IsRevisioned(name)) - { - return name; - } - - var revisionNumber = ApiRevisionNumber.TryFrom(dto.Properties.ApiRevision) - .IfNone(() => throw new InvalidOperationException("Could not get revision number.")); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedApis)); - var rootName = ApiName.GetRootName(name); + logger.LogInformation("Validating published apis in {ServiceDirectory}...", serviceDirectory); - return ApiName.GetRevisionedName(rootName, revisionNumber); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask ValidateExtractedInformationFiles(IDictionary expectedResources, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) - { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - - var expected = expectedResources.MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await ApiModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(ApiDto dto) => new { @@ -331,106 +441,172 @@ private static string NormalizeDto(ApiDto dto) => // ? uri.RemovePath().ToString() // : string.Empty }.ToString()!; +} - private static async ValueTask ValidateExtractedSpecificationFiles(IDictionary expectedResources, Option defaultApiSpecification, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class ApiServices +{ + public static void ConfigureDeleteAllApis(IServiceCollection services) { - var expected = await expectedResources.ToAsyncEnumerable() - .Choose(async kvp => - { - var name = kvp.Key; - return from specification in await GetExpectedApiSpecification(name, kvp.Value, defaultApiSpecification, serviceUri, pipeline, cancellationToken) - // Skip XML specification files. Sometimes they get extracted, other times they fail. - where specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl) - select (name, specification); - }) - .ToFrozenDictionary(cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var actual = await ApiModule.ListSpecificationFiles(serviceDirectory, cancellationToken) - .Select(file => (file.Parent.Name, file.Specification)) - // Skip XML specification files. Sometimes they get extracted, other times they fail. - .Where(file => file.Specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl)) - .ToFrozenDictionary(cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - actual.Should().BeEquivalentTo(expected); + public static void ConfigurePutApiModels(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetExpectedApiSpecification(ApiName name, ApiDto dto, Option defaultApiSpecification, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedApis(IServiceCollection services) { - switch (dto.Properties.ApiType ?? dto.Properties.Type) - { - case "graphql": - var specificationContents = await ApiUri.From(name, serviceUri) - .TryGetGraphQlSchema(pipeline, cancellationToken); - return specificationContents.Map(contents => new ApiSpecification.GraphQl() as ApiSpecification); - case "soap": - return new ApiSpecification.Wsdl(); - case "websocket": - return Option.None; - default: -#pragma warning disable CA1849 // Call async methods when in an async method - return defaultApiSpecification.IfNone(() => new ApiSpecification.OpenApi - { - Format = new OpenApiFormat.Yaml(), - Version = new OpenApiVersion.V3() - }); -#pragma warning restore CA1849 // Call async methods when in an async method - } + ConfigureGetApimApis(services); + ConfigureTryGetApimGraphQlSchema(services); + ConfigureGetFileApis(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetApimApis(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - await fileResources.Keys.IterParallel(async name => - { - await ApiPolicy.ValidatePublisherChanges(name, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ApiTag.ValidatePublisherChanges(name, serviceDirectory, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureTryGetApimGraphQlSchema(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetFileApis(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - await fileResources.Keys.IterParallel(async name => - { - await ApiPolicy.ValidatePublisherCommitChanges(name, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ApiTag.ValidatePublisherCommitChanges(name, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + public static void ConfigureWriteApiModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => ApiInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + public static void ConfigureValidatePublishedApis(IServiceCollection services) + { + ConfigureGetFileApis(services); + ConfigureGetApimApis(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ApiInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Api +{ + public static Gen GenerateUpdate(ApiModel original) => + from revisions in GenerateRevisionUpdates(original.Revisions, original.Type, original.Name) + select original with + { + Revisions = revisions + }; + + private static Gen> GenerateRevisionUpdates(FrozenSet revisions, ApiType type, ApiName name) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + var newGen = ApiRevision.GenerateSet(type, name); + var updateGen = (ApiRevision revision) => GenerateRevisionUpdate(revision, type); - return await contentsOption.MapTask(async contents => + return Fixture.GenerateNewSet(revisions, newGen, updateGen); + } + + private static Gen GenerateRevisionUpdate(ApiRevision revision, ApiType type) => + from serviceUri in type is ApiType.Soap or ApiType.WebSocket + ? Gen.Const(revision.ServiceUri) + : ApiRevision.GenerateServiceUri(type) + from description in ApiRevision.GenerateDescription().OptionOf() + select revision with { - using (contents) + ServiceUri = serviceUri.ValueUnsafe(), + Description = description.ValueUnsafe() + }; + + public static Gen GenerateOverride(ApiDto original) => + from serviceUrl in (original.Properties.Type ?? original.Properties.ApiType) switch + { + "websocket" or "soap" => Gen.Const(original.Properties.ServiceUrl), + _ => Generator.AbsoluteUri.Select(uri => (string?)uri.ToString()) + } + from revisionDescription in ApiRevision.GenerateDescription().OptionOf() + select new ApiDto() + { + Properties = new ApiDto.ApiCreateOrUpdateProperties { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + ServiceUrl = serviceUrl, + ApiRevisionDescription = revisionDescription.ValueUnsafe() } - }); + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.SelectMany(model => model.Revisions.Select((revision, index) => + { + var apiName = GetApiName(model.Name, revision, index); + var dto = GetDto(model.Name, model.Type, model.Path, model.Version, revision); + return (apiName, dto); + })) + .Where(api => ApiName.IsNotRevisioned(api.apiName)) + .ToFrozenDictionary(); + + private static ApiName GetApiName(ApiName name, ApiRevision revision, int index) + { + var rootApiName = ApiName.GetRootName(name); + + return index == 0 ? rootApiName : ApiName.GetRevisionedName(rootApiName, revision.Number); } + + private static ApiDto GetDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => + new ApiDto() + { + Properties = new ApiDto.ApiCreateOrUpdateProperties + { + // APIM sets the description to null when it imports for SOAP APIs. + DisplayName = name.ToString(), + Path = path, + ApiType = type switch + { + ApiType.Http => null, + ApiType.Soap => "soap", + ApiType.GraphQl => null, + ApiType.WebSocket => null, + _ => throw new NotSupportedException() + }, + Type = type switch + { + ApiType.Http => "http", + ApiType.Soap => "soap", + ApiType.GraphQl => "graphql", + ApiType.WebSocket => "websocket", + _ => throw new NotSupportedException() + }, + Protocols = type switch + { + ApiType.Http => ["http", "https"], + ApiType.Soap => ["http", "https"], + ApiType.GraphQl => ["http", "https"], + ApiType.WebSocket => ["ws", "wss"], + _ => throw new NotSupportedException() + }, + ServiceUrl = revision.ServiceUri.ValueUnsafe()?.ToString(), + ApiRevisionDescription = revision.Description.ValueUnsafe(), + ApiRevision = $"{revision.Number.ToInt()}", + ApiVersion = version.Map(version => version.Version).ValueUnsafe(), + ApiVersionSetId = version.Map(version => $"/apiVersionSets/{version.VersionSetName}").ValueUnsafe() + } + }; } diff --git a/tools/code/integration.tests/App.cs b/tools/code/integration.tests/App.cs new file mode 100644 index 00000000..06c382b5 --- /dev/null +++ b/tools/code/integration.tests/App.cs @@ -0,0 +1,30 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace integration.tests; + +internal delegate ValueTask RunApplication(CancellationToken cancellationToken); + +file sealed class RunApplicationHandler(ActivitySource activitySource, RunTests runTests) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(RunApplication)); + + await runTests(cancellationToken); + } +} + +internal static class AppServices +{ + public static void ConfigureRunApplication(IServiceCollection services) + { + TestServices.ConfigureRunTests(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/Backend.cs b/tools/code/integration.tests/Backend.cs index 92373b8c..3c71eddc 100644 --- a/tools/code/integration.tests/Backend.cs +++ b/tools/code/integration.tests/Backend.cs @@ -5,41 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Backend +internal delegate ValueTask DeleteAllBackends(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutBackendModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedBackends(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimBackends(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileBackends(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteBackendModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedBackends(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllBackendsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(BackendModel original) => - from url in Generator.AbsoluteUri - from description in BackendModel.GenerateDescription().OptionOf() - select original with - { - Url = url, - Description = description - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllBackends)); - public static Gen GenerateOverride(BackendDto original) => - from url in Generator.AbsoluteUri - from description in BackendModel.GenerateDescription().OptionOf() - select new BackendDto + logger.LogInformation("Deleting all backends in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await BackendsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutBackendModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutBackendModels)); + + logger.LogInformation("Putting backend models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new BackendDto.BackendContract - { - Url = url.ToString(), - Description = description.ValueUnsafe() - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(BackendModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = BackendUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static BackendDto GetDto(BackendModel model) => new() @@ -51,29 +77,111 @@ private static BackendDto GetDto(BackendModel model) => Protocol = model.Protocol } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedBackendsHandler(ILogger logger, GetApimBackends getApimResources, GetFileBackends getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedBackends)); + + logger.LogInformation("Validating extracted backends in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(BackendDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + Url = dto.Properties.Url ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty, + Protocol = dto.Properties.Protocol ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(BackendModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimBackendsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = BackendUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimBackends)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting backends from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = BackendsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await BackendsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileBackendsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileBackends)); + + logger.LogInformation("Getting backends from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => BackendInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, BackendInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileBackends)); + + logger.LogInformation("Getting backends from {ServiceDirectory}...", serviceDirectory); + + return await BackendModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => +file sealed class WriteBackendModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteBackendModels)); + + logger.LogInformation("Writing backend models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(BackendModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -83,33 +191,36 @@ private static async ValueTask WriteInformationFile(BackendModel model, Manageme await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static BackendDto GetDto(BackendModel model) => + new() + { + Properties = new BackendDto.BackendContract + { + Url = model.Url.ToString(), + Description = model.Description.ValueUnsafe(), + Protocol = model.Protocol + } + }; +} + +file sealed class ValidatePublishedBackendsHandler(ILogger logger, GetFileBackends getFileResources, GetApimBackends getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedBackends)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published backends in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = BackendsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await BackendModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(BackendDto dto) => new { @@ -117,49 +228,99 @@ private static string NormalizeDto(BackendDto dto) => Description = dto.Properties.Description ?? string.Empty, Protocol = dto.Properties.Protocol ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class BackendServices +{ + public static void ConfigureDeleteAllBackends(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutBackendModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedBackends(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimBackends(services); + ConfigureGetFileBackends(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => BackendInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimBackends(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, BackendInformationFile file, CancellationToken cancellationToken) + private static void ConfigureGetFileBackends(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteBackendModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedBackends(IServiceCollection services) + { + ConfigureGetFileBackends(services); + ConfigureGetApimBackends(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Backend +{ + public static Gen GenerateUpdate(BackendModel original) => + from url in Generator.AbsoluteUri + from description in BackendModel.GenerateDescription().OptionOf() + select original with { - using (contents) + Url = url, + Description = description + }; + + public static Gen GenerateOverride(BackendDto original) => + from url in Generator.AbsoluteUri + from description in BackendModel.GenerateDescription().OptionOf() + select new BackendDto + { + Properties = new BackendDto.BackendContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + Url = url.ToString(), + Description = description.ValueUnsafe() } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static BackendDto GetDto(BackendModel model) => + new() + { + Properties = new BackendDto.BackendContract + { + Url = model.Url.ToString(), + Description = model.Description.ValueUnsafe(), + Protocol = model.Protocol + } + }; } diff --git a/tools/code/integration.tests/Common.cs b/tools/code/integration.tests/Common.cs new file mode 100644 index 00000000..3c9ecc50 --- /dev/null +++ b/tools/code/integration.tests/Common.cs @@ -0,0 +1,114 @@ +using Azure.Core; +using Azure.Core.Pipeline; +using Azure.Identity; +using Azure.ResourceManager; +using common; +using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.JsonWebTokens; +using System; +using System.Diagnostics; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; + +internal delegate string GetSubscriptionId(); +internal delegate string GetResourceGroupName(); +internal delegate ValueTask GetBearerToken(CancellationToken cancellationToken); + +internal static class CommonServices +{ + public static void Configure(IServiceCollection services) + { + services.AddSingleton(GetActivitySource); + services.AddSingleton(GetAzureEnvironment); + services.AddSingleton(GetTokenCredential); + services.AddSingleton(GetHttpPipeline); + services.AddSingleton(GetSubscriptionId); + services.AddSingleton(GetResourceGroupName); + services.AddSingleton(GetBearerToken); + OpenTelemetryServices.Configure(services); + } + + private static ActivitySource GetActivitySource(IServiceProvider provider) => + new("ApiOps.Integration.Tests"); + + private static AzureEnvironment GetAzureEnvironment(IServiceProvider provider) + { + var configuration = provider.GetRequiredService(); + + return configuration.TryGetValue("AZURE_CLOUD_ENVIRONMENT").ValueUnsafe() switch + { + null => AzureEnvironment.Public, + "AzureGlobalCloud" or nameof(ArmEnvironment.AzurePublicCloud) => AzureEnvironment.Public, + "AzureChinaCloud" or nameof(ArmEnvironment.AzureChina) => AzureEnvironment.China, + "AzureUSGovernment" or nameof(ArmEnvironment.AzureGovernment) => AzureEnvironment.USGovernment, + "AzureGermanCloud" or nameof(ArmEnvironment.AzureGermany) => AzureEnvironment.Germany, + _ => throw new InvalidOperationException($"AZURE_CLOUD_ENVIRONMENT is invalid. Valid values are {nameof(ArmEnvironment.AzurePublicCloud)}, {nameof(ArmEnvironment.AzureChina)}, {nameof(ArmEnvironment.AzureGovernment)}, {nameof(ArmEnvironment.AzureGermany)}") + }; + } + + private static TokenCredential GetTokenCredential(IServiceProvider provider) + { + var configuration = provider.GetRequiredService(); + var azureAuthorityHost = provider.GetRequiredService().AuthorityHost; + + return configuration.TryGetValue("AZURE_BEARER_TOKEN") + .Map(GetCredentialFromToken) + .IfNone(() => GetDefaultAzureCredential(azureAuthorityHost)); + } + + private static TokenCredential GetCredentialFromToken(string token) + { + var jsonWebToken = new JsonWebToken(token); + var expirationDate = new DateTimeOffset(jsonWebToken.ValidTo); + var accessToken = new AccessToken(token, expirationDate); + + return DelegatedTokenCredential.Create((context, cancellationToken) => accessToken); + } + + private static DefaultAzureCredential GetDefaultAzureCredential(Uri azureAuthorityHost) => + new(new DefaultAzureCredentialOptions + { + AuthorityHost = azureAuthorityHost + }); + + private static HttpPipeline GetHttpPipeline(IServiceProvider provider) + { + var clientOptions = ClientOptions.Default; + clientOptions.RetryPolicy = new CommonRetryPolicy(); + + var tokenCredential = provider.GetRequiredService(); + var azureEnvironment = provider.GetRequiredService(); + var bearerAuthenticationPolicy = new BearerTokenAuthenticationPolicy(tokenCredential, azureEnvironment.DefaultScope); + + var logger = provider.GetRequiredService().CreateLogger(nameof(HttpPipeline)); + var loggingPolicy = new ILoggerHttpPipelinePolicy(logger); + + var version = Assembly.GetExecutingAssembly()?.GetName().Version ?? new Version("-1"); + var telemetryPolicy = new TelemetryPolicy(version); + + return HttpPipelineBuilder.Build(clientOptions, bearerAuthenticationPolicy, loggingPolicy, telemetryPolicy); + } + + private static GetSubscriptionId GetSubscriptionId(IServiceProvider provider) => + () => provider.GetRequiredService().GetValue("AZURE_SUBSCRIPTION_ID"); + + private static GetResourceGroupName GetResourceGroupName(IServiceProvider provider) => + () => provider.GetRequiredService().GetValue("AZURE_RESOURCE_GROUP_NAME"); + + private static GetBearerToken GetBearerToken(IServiceProvider provider) + { + var tokenCredential = provider.GetRequiredService(); + var azureEnvironment = provider.GetRequiredService(); + var context = new TokenRequestContext([azureEnvironment.DefaultScope]); + + return async cancellationToken => + { + var token = await tokenCredential.GetTokenAsync(context, cancellationToken); + return token.Token; + }; + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/Configuration.cs b/tools/code/integration.tests/Configuration.cs index d508ad1e..dbf3a1ea 100644 --- a/tools/code/integration.tests/Configuration.cs +++ b/tools/code/integration.tests/Configuration.cs @@ -1,169 +1,169 @@ -using Azure.Core; -using Azure.Core.Pipeline; -using Azure.Identity; -using Azure.ResourceManager; -using common; -using Flurl; -using LanguageExt; -using LanguageExt.UnsafeValueAccess; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.IdentityModel.JsonWebTokens; -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace integration.tests; - -internal static class Configuration -{ - private static readonly Lazy @default = new(() => CreateDefault()); - public static IConfiguration Default => @default.Value; - - private static readonly Lazy azureEnvironment = new(() => GetAzureEnvironment(Default)); - private static AzureEnvironment AzureEnvironment => azureEnvironment.Value; - - private static readonly Lazy location = new(() => GetLocation(Default)); - public static string Location => location.Value; - - private static readonly Lazy apimProviderUri = new(() => GetApimProviderUri(Default, AzureEnvironment)); - public static Uri ApimProviderUri => apimProviderUri.Value; - - private static readonly Lazy managementServiceProviderUri = new(() => GetManagementServiceProviderUri(ApimProviderUri)); - public static Uri ManagementServiceProviderUri => managementServiceProviderUri.Value; - - private static readonly Lazy subscriptionId = new(() => GetSubscriptionId(Default)); - public static string SubscriptionId => subscriptionId.Value; - - private static readonly Lazy resourceGroupName = new(() => GetResourceGroupName(Default)); - public static string ResourceGroupName => resourceGroupName.Value; - - private static readonly Lazy tokenCredential = new(() => GetTokenCredential(AzureEnvironment.AuthorityHost, Default)); - private static TokenCredential TokenCredential => tokenCredential.Value; - - private static readonly Lazy httpPipeline = new(() => GetHttpPipeline(TokenCredential, AzureEnvironment)); - public static HttpPipeline HttpPipeline => httpPipeline.Value; - - private static readonly Lazy firstManagementServiceName = new(() => GetFirstManagementServiceName(Default)); - public static ManagementServiceName FirstServiceName => firstManagementServiceName.Value; - - private static readonly Lazy firstServiceUri = new(() => GetManagementServiceUri(FirstServiceName)); - public static ManagementServiceUri FirstServiceUri => firstServiceUri.Value; - - private static readonly Lazy secondManagementServiceName = new(() => GetSecondManagementServiceName(Default)); - public static ManagementServiceName SecondServiceName => secondManagementServiceName.Value; - - private static readonly Lazy secondServiceUri = new(() => GetManagementServiceUri(SecondServiceName)); - public static ManagementServiceUri SecondServiceUri => secondServiceUri.Value; - - private static IConfiguration CreateDefault() => - new ConfigurationBuilder().AddEnvironmentVariables() - .AddUserSecrets(typeof(Extractor).Assembly) - .Build(); - - private static AzureEnvironment GetAzureEnvironment(IConfiguration configuration) => - configuration.TryGetValue("AZURE_CLOUD_ENVIRONMENT").ValueUnsafe() switch - { - null => AzureEnvironment.Public, - "AzureGlobalCloud" or nameof(ArmEnvironment.AzurePublicCloud) => AzureEnvironment.Public, - "AzureChinaCloud" or nameof(ArmEnvironment.AzureChina) => AzureEnvironment.China, - "AzureUSGovernment" or nameof(ArmEnvironment.AzureGovernment) => AzureEnvironment.USGovernment, - "AzureGermanCloud" or nameof(ArmEnvironment.AzureGermany) => AzureEnvironment.Germany, - _ => throw new InvalidOperationException($"AZURE_CLOUD_ENVIRONMENT is invalid. Valid values are {nameof(ArmEnvironment.AzurePublicCloud)}, {nameof(ArmEnvironment.AzureChina)}, {nameof(ArmEnvironment.AzureGovernment)}, {nameof(ArmEnvironment.AzureGermany)}") - }; - - private static Uri GetApimProviderUri(IConfiguration configuration, AzureEnvironment azureEnvironment) - { - var apiVersion = configuration.TryGetValue("ARM_API_VERSION") - .IfNone(() => "2022-08-01"); - - return azureEnvironment.ManagementEndpoint - .AppendPathSegment("subscriptions") - .AppendPathSegment(GetSubscriptionId(configuration)) - .AppendPathSegment("resourceGroups") - .AppendPathSegment(GetResourceGroupName(configuration)) - .AppendPathSegment("providers/Microsoft.ApiManagement") - .SetQueryParam("api-version", apiVersion) - .ToUri(); - } - - private static Uri GetManagementServiceProviderUri(Uri apimProviderUri) => - apimProviderUri.AppendPathSegment("service").ToUri(); - - public static ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName) => - GetManagementServiceUri(serviceName, ManagementServiceProviderUri); - - private static ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName, Uri managementServiceProviderUri) - { - var uri = managementServiceProviderUri.AppendPathSegment(serviceName.ToString()) - .ToUri(); - - return ManagementServiceUri.From(uri); - } - - private static string GetLocation(IConfiguration configuration) => - configuration.TryGetValue("AZURE_LOCATION").IfNone("westus"); - - private static string GetSubscriptionId(IConfiguration configuration) => - configuration.GetValue("AZURE_SUBSCRIPTION_ID"); - - private static string GetResourceGroupName(IConfiguration configuration) => - configuration.GetValue("AZURE_RESOURCE_GROUP_NAME"); - - private static TokenCredential GetTokenCredential(Uri azureAuthorityHost, IConfiguration configuration) => - configuration.TryGetValue("AZURE_BEARER_TOKEN") - .Map(GetCredentialFromToken) - .IfNone(() => GetDefaultAzureCredential(azureAuthorityHost)); - - private static DefaultAzureCredential GetDefaultAzureCredential(Uri azureAuthorityHost) => - new(new DefaultAzureCredentialOptions - { - AuthorityHost = azureAuthorityHost, - ExcludeVisualStudioCredential = true - }); - - private static HttpPipeline GetHttpPipeline(TokenCredential tokenCredential, AzureEnvironment azureEnvironment) - { - var clientOptions = ClientOptions.Default; - clientOptions.RetryPolicy = new RetryPolicy(); - var bearerAuthenticationPolicy = new BearerTokenAuthenticationPolicy(tokenCredential, azureEnvironment.DefaultScope); - -#pragma warning disable CA2000 // Dispose objects before losing scope - var logger = LoggerFactory.Create(builder => - { - builder.AddDebug().AddConsole(); - builder.SetMinimumLevel(LogLevel.Trace); - }).CreateLogger(); -#pragma warning restore CA2000 // Dispose objects before losing scope - var loggingPolicy = new ILoggerHttpPipelinePolicy(logger); - - return HttpPipelineBuilder.Build(clientOptions, bearerAuthenticationPolicy, loggingPolicy); - } - - private static TokenCredential GetCredentialFromToken(string token) - { - var jsonWebToken = new JsonWebToken(token); - var expirationDate = new DateTimeOffset(jsonWebToken.ValidTo); - var accessToken = new AccessToken(token, expirationDate); - - return DelegatedTokenCredential.Create((context, cancellationToken) => accessToken); - } - - public static async ValueTask GetBearerToken(CancellationToken cancellationToken) => - await GetBearerToken(TokenCredential, AzureEnvironment, cancellationToken); - - private static async ValueTask GetBearerToken(TokenCredential tokenCredential, AzureEnvironment azureEnvironment, CancellationToken cancellationToken) - { - var context = new TokenRequestContext([azureEnvironment.DefaultScope]); - - var token = await tokenCredential.GetTokenAsync(context, cancellationToken); - - return token.Token; - } - - private static ManagementServiceName GetFirstManagementServiceName(IConfiguration configuration) => - ManagementServiceName.From(configuration.GetValue("FIRST_API_MANAGEMENT_SERVICE_NAME")); - - private static ManagementServiceName GetSecondManagementServiceName(IConfiguration configuration) => - ManagementServiceName.From(configuration.GetValue("SECOND_API_MANAGEMENT_SERVICE_NAME")); -} \ No newline at end of file +//using Azure.Core; +//using Azure.Core.Pipeline; +//using Azure.Identity; +//using Azure.ResourceManager; +//using common; +//using Flurl; +//using LanguageExt; +//using LanguageExt.UnsafeValueAccess; +//using Microsoft.Extensions.Configuration; +//using Microsoft.Extensions.Logging; +//using Microsoft.IdentityModel.JsonWebTokens; +//using System; +//using System.Threading; +//using System.Threading.Tasks; + +//namespace integration.tests; + +//internal static class Configuration +//{ +// private static readonly Lazy @default = new(() => CreateDefault()); +// public static IConfiguration Default => @default.Value; + +// private static readonly Lazy azureEnvironment = new(() => GetAzureEnvironment(Default)); +// private static AzureEnvironment AzureEnvironment => azureEnvironment.Value; + +// private static readonly Lazy location = new(() => GetLocation(Default)); +// public static string Location => location.Value; + +// private static readonly Lazy apimProviderUri = new(() => GetApimProviderUri(Default, AzureEnvironment)); +// public static Uri ApimProviderUri => apimProviderUri.Value; + +// private static readonly Lazy managementServiceProviderUri = new(() => GetManagementServiceProviderUri(ApimProviderUri)); +// public static Uri ManagementServiceProviderUri => managementServiceProviderUri.Value; + +// private static readonly Lazy subscriptionId = new(() => GetSubscriptionId(Default)); +// public static string SubscriptionId => subscriptionId.Value; + +// private static readonly Lazy resourceGroupName = new(() => GetResourceGroupName(Default)); +// public static string ResourceGroupName => resourceGroupName.Value; + +// private static readonly Lazy tokenCredential = new(() => GetTokenCredential(AzureEnvironment.AuthorityHost, Default)); +// private static TokenCredential TokenCredential => tokenCredential.Value; + +// private static readonly Lazy httpPipeline = new(() => GetHttpPipeline(TokenCredential, AzureEnvironment)); +// public static HttpPipeline HttpPipeline => httpPipeline.Value; + +// private static readonly Lazy firstManagementServiceName = new(() => GetFirstManagementServiceName(Default)); +// public static ManagementServiceName FirstServiceName => firstManagementServiceName.Value; + +// private static readonly Lazy firstServiceUri = new(() => GetManagementServiceUri(FirstServiceName)); +// public static ManagementServiceUri FirstServiceUri => firstServiceUri.Value; + +// private static readonly Lazy secondManagementServiceName = new(() => GetSecondManagementServiceName(Default)); +// public static ManagementServiceName SecondServiceName => secondManagementServiceName.Value; + +// private static readonly Lazy secondServiceUri = new(() => GetManagementServiceUri(SecondServiceName)); +// public static ManagementServiceUri SecondServiceUri => secondServiceUri.Value; + +// private static IConfiguration CreateDefault() => +// new ConfigurationBuilder().AddEnvironmentVariables() +// .AddUserSecrets(typeof(Extractor).Assembly) +// .Build(); + +// private static AzureEnvironment GetAzureEnvironment(IConfiguration configuration) => +// configuration.TryGetValue("AZURE_CLOUD_ENVIRONMENT").ValueUnsafe() switch +// { +// null => AzureEnvironment.Public, +// "AzureGlobalCloud" or nameof(ArmEnvironment.AzurePublicCloud) => AzureEnvironment.Public, +// "AzureChinaCloud" or nameof(ArmEnvironment.AzureChina) => AzureEnvironment.China, +// "AzureUSGovernment" or nameof(ArmEnvironment.AzureGovernment) => AzureEnvironment.USGovernment, +// "AzureGermanCloud" or nameof(ArmEnvironment.AzureGermany) => AzureEnvironment.Germany, +// _ => throw new InvalidOperationException($"AZURE_CLOUD_ENVIRONMENT is invalid. Valid values are {nameof(ArmEnvironment.AzurePublicCloud)}, {nameof(ArmEnvironment.AzureChina)}, {nameof(ArmEnvironment.AzureGovernment)}, {nameof(ArmEnvironment.AzureGermany)}") +// }; + +// private static Uri GetApimProviderUri(IConfiguration configuration, AzureEnvironment azureEnvironment) +// { +// var apiVersion = configuration.TryGetValue("ARM_API_VERSION") +// .IfNone(() => "2022-08-01"); + +// return azureEnvironment.ManagementEndpoint +// .AppendPathSegment("subscriptions") +// .AppendPathSegment(GetSubscriptionId(configuration)) +// .AppendPathSegment("resourceGroups") +// .AppendPathSegment(GetResourceGroupName(configuration)) +// .AppendPathSegment("providers/Microsoft.ApiManagement") +// .SetQueryParam("api-version", apiVersion) +// .ToUri(); +// } + +// private static Uri GetManagementServiceProviderUri(Uri apimProviderUri) => +// apimProviderUri.AppendPathSegment("service").ToUri(); + +// public static ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName) => +// GetManagementServiceUri(serviceName, ManagementServiceProviderUri); + +// private static ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName, Uri managementServiceProviderUri) +// { +// var uri = managementServiceProviderUri.AppendPathSegment(serviceName.ToString()) +// .ToUri(); + +// return ManagementServiceUri.From(uri); +// } + +// private static string GetLocation(IConfiguration configuration) => +// configuration.TryGetValue("AZURE_LOCATION").IfNone("westus"); + +// private static string GetSubscriptionId(IConfiguration configuration) => +// configuration.GetValue("AZURE_SUBSCRIPTION_ID"); + +// private static string GetResourceGroupName(IConfiguration configuration) => +// configuration.GetValue("AZURE_RESOURCE_GROUP_NAME"); + +// private static TokenCredential GetTokenCredential(Uri azureAuthorityHost, IConfiguration configuration) => +// configuration.TryGetValue("AZURE_BEARER_TOKEN") +// .Map(GetCredentialFromToken) +// .IfNone(() => GetDefaultAzureCredential(azureAuthorityHost)); + +// private static DefaultAzureCredential GetDefaultAzureCredential(Uri azureAuthorityHost) => +// new(new DefaultAzureCredentialOptions +// { +// AuthorityHost = azureAuthorityHost, +// ExcludeVisualStudioCredential = true +// }); + +// private static HttpPipeline GetHttpPipeline(TokenCredential tokenCredential, AzureEnvironment azureEnvironment) +// { +// var clientOptions = ClientOptions.Default; +// clientOptions.RetryPolicy = new RetryPolicy(); +// var bearerAuthenticationPolicy = new BearerTokenAuthenticationPolicy(tokenCredential, azureEnvironment.DefaultScope); + +//#pragma warning disable CA2000 // Dispose objects before losing scope +// var logger = LoggerFactory.Create(builder => +// { +// builder.AddDebug().AddConsole(); +// builder.SetMinimumLevel(LogLevel.Trace); +// }).CreateLogger(); +//#pragma warning restore CA2000 // Dispose objects before losing scope +// var loggingPolicy = new ILoggerHttpPipelinePolicy(logger); + +// return HttpPipelineBuilder.Build(clientOptions, bearerAuthenticationPolicy, loggingPolicy); +// } + +// private static TokenCredential GetCredentialFromToken(string token) +// { +// var jsonWebToken = new JsonWebToken(token); +// var expirationDate = new DateTimeOffset(jsonWebToken.ValidTo); +// var accessToken = new AccessToken(token, expirationDate); + +// return DelegatedTokenCredential.Create((context, cancellationToken) => accessToken); +// } + +// public static async ValueTask GetBearerToken(CancellationToken cancellationToken) => +// await GetBearerToken(TokenCredential, AzureEnvironment, cancellationToken); + +// private static async ValueTask GetBearerToken(TokenCredential tokenCredential, AzureEnvironment azureEnvironment, CancellationToken cancellationToken) +// { +// var context = new TokenRequestContext([azureEnvironment.DefaultScope]); + +// var token = await tokenCredential.GetTokenAsync(context, cancellationToken); + +// return token.Token; +// } + +// private static ManagementServiceName GetFirstManagementServiceName(IConfiguration configuration) => +// ManagementServiceName.From(configuration.GetValue("FIRST_API_MANAGEMENT_SERVICE_NAME")); + +// private static ManagementServiceName GetSecondManagementServiceName(IConfiguration configuration) => +// ManagementServiceName.From(configuration.GetValue("SECOND_API_MANAGEMENT_SERVICE_NAME")); +//} \ No newline at end of file diff --git a/tools/code/integration.tests/Diagnostic.cs b/tools/code/integration.tests/Diagnostic.cs index 7a157e80..fa563fc9 100644 --- a/tools/code/integration.tests/Diagnostic.cs +++ b/tools/code/integration.tests/Diagnostic.cs @@ -5,45 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Diagnostic +internal delegate ValueTask DeleteAllDiagnostics(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutDiagnosticModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedDiagnostics(Option> diagnosticNamesOption, Option> loggerNamesOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimDiagnostics(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileDiagnostics(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteDiagnosticModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedDiagnostics(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllDiagnosticsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(DiagnosticModel original) => - from alwaysLog in Gen.Const("allErrors").OptionOf() - from sampling in DiagnosticSampling.Generate().OptionOf() - select original with - { - AlwaysLog = alwaysLog, - Sampling = sampling - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllDiagnostics)); - public static Gen GenerateOverride(DiagnosticDto original) => - from alwaysLog in Gen.Const("allErrors").OptionOf() - from sampling in DiagnosticSampling.Generate().OptionOf() - select new DiagnosticDto + logger.LogInformation("Deleting all diagnostics in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await DiagnosticsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutDiagnosticModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutDiagnosticModels)); + + logger.LogInformation("Putting diagnostic models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new DiagnosticDto.DiagnosticContract - { - AlwaysLog = alwaysLog.ValueUnsafe(), - Sampling = sampling.Map(sampling => new DiagnosticDto.SamplingSettings - { - SamplingType = sampling.Type, - Percentage = sampling.Percentage - }).ValueUnsafe() - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(DiagnosticModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = DiagnosticUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static DiagnosticDto GetDto(DiagnosticModel model) => new() @@ -59,29 +81,118 @@ private static DiagnosticDto GetDto(DiagnosticModel model) => }).ValueUnsafe() } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedDiagnosticsHandler(ILogger logger, GetApimDiagnostics getApimResources, GetFileDiagnostics getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> diagnosticNamesOption, Option> loggerNamesOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedDiagnostics)); + + logger.LogInformation("Validating extracted diagnostics in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, diagnosticNamesOption)) + .WhereValue(dto => DiagnosticModule.TryGetLoggerName(dto) + .Map(name => ExtractorOptions.ShouldExtract(name, loggerNamesOption)) + .IfNone(true)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(DiagnosticDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + LoggerId = string.Join('/', dto.Properties.LoggerId?.Split('/')?.TakeLast(2)?.ToArray() ?? []), + AlwaysLog = dto.Properties.AlwaysLog ?? string.Empty, + Sampling = new + { + Type = dto.Properties.Sampling?.SamplingType ?? string.Empty, + Percentage = dto.Properties.Sampling?.Percentage ?? 0 + } + }.ToString()!; +} - private static async ValueTask Put(DiagnosticModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimDiagnosticsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = DiagnosticUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimDiagnostics)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting diagnostics from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = DiagnosticsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await DiagnosticsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileDiagnosticsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileDiagnostics)); + + logger.LogInformation("Getting diagnostics from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => DiagnosticInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, DiagnosticInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileDiagnostics)); + + logger.LogInformation("Getting diagnostics from {ServiceDirectory}...", serviceDirectory); + + return await DiagnosticModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => +file sealed class WriteDiagnosticModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteDiagnosticModels)); + + logger.LogInformation("Writing diagnostic models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(DiagnosticModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -91,33 +202,40 @@ private static async ValueTask WriteInformationFile(DiagnosticModel model, Manag await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static DiagnosticDto GetDto(DiagnosticModel model) => + new() + { + Properties = new DiagnosticDto.DiagnosticContract + { + LoggerId = $"/loggers/{model.LoggerName}", + AlwaysLog = model.AlwaysLog.ValueUnsafe(), + Sampling = model.Sampling.Map(sampling => new DiagnosticDto.SamplingSettings + { + SamplingType = sampling.Type, + Percentage = sampling.Percentage + }).ValueUnsafe() + } + }; +} + +file sealed class ValidatePublishedDiagnosticsHandler(ILogger logger, GetFileDiagnostics getFileResources, GetApimDiagnostics getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedDiagnostics)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published diagnostics in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = DiagnosticsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await DiagnosticModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(DiagnosticDto dto) => new { @@ -129,49 +247,107 @@ private static string NormalizeDto(DiagnosticDto dto) => Percentage = dto.Properties.Sampling?.Percentage ?? 0 } }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class DiagnosticServices +{ + public static void ConfigureDeleteAllDiagnostics(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutDiagnosticModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedDiagnostics(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimDiagnostics(services); + ConfigureGetFileDiagnostics(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => DiagnosticInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimDiagnostics(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, DiagnosticInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileDiagnostics(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteDiagnosticModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedDiagnostics(IServiceCollection services) + { + ConfigureGetFileDiagnostics(services); + ConfigureGetApimDiagnostics(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Diagnostic +{ + public static Gen GenerateUpdate(DiagnosticModel original) => + from alwaysLog in Gen.Const("allErrors").OptionOf() + from sampling in DiagnosticSampling.Generate().OptionOf() + select original with { - using (contents) + AlwaysLog = alwaysLog, + Sampling = sampling + }; + + public static Gen GenerateOverride(DiagnosticDto original) => + from alwaysLog in Gen.Const("allErrors").OptionOf() + from sampling in DiagnosticSampling.Generate().OptionOf() + select new DiagnosticDto + { + Properties = new DiagnosticDto.DiagnosticContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + AlwaysLog = alwaysLog.ValueUnsafe(), + Sampling = sampling.Map(sampling => new DiagnosticDto.SamplingSettings + { + SamplingType = sampling.Type, + Percentage = sampling.Percentage + }).ValueUnsafe() } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static DiagnosticDto GetDto(DiagnosticModel model) => + new() + { + Properties = new DiagnosticDto.DiagnosticContract + { + LoggerId = $"/loggers/{model.LoggerName}", + AlwaysLog = model.AlwaysLog.ValueUnsafe(), + Sampling = model.Sampling.Map(sampling => new DiagnosticDto.SamplingSettings + { + SamplingType = sampling.Type, + Percentage = sampling.Percentage + }).ValueUnsafe() + } + }; } diff --git a/tools/code/integration.tests/Extractor.cs b/tools/code/integration.tests/Extractor.cs index 78a3b8de..123426ad 100644 --- a/tools/code/integration.tests/Extractor.cs +++ b/tools/code/integration.tests/Extractor.cs @@ -3,9 +3,13 @@ using CsCheck; using extractor; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Linq.Expressions; @@ -16,43 +20,9 @@ namespace integration.tests; -internal static class Extractor -{ - public static async ValueTask Run(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, string subscriptionId, string resourceGroupName, string bearerToken, CancellationToken cancellationToken) - { - var argumentDictionary = new Dictionary - { - [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), - ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, - ["AZURE_SUBSCRIPTION_ID"] = subscriptionId, - ["AZURE_RESOURCE_GROUP_NAME"] = resourceGroupName, - ["AZURE_BEARER_TOKEN"] = bearerToken, - ["Logging:LogLevel:Default"] = "Trace" - }; - - var optionsJson = options.ToJsonObject(); - if (optionsJson.Count > 0) - { - var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.extractor.yaml"); - var yamlFile = new FileInfo(yamlFilePath); - await WriteYamlToFile(optionsJson, yamlFile, cancellationToken); - argumentDictionary.Add("CONFIGURATION_YAML_PATH", yamlFile.FullName); - } +internal delegate ValueTask RunExtractor(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); - var arguments = argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); - await extractor.Program.Main(arguments); - } - - private static string GetApiManagementServiceNameParameter() => - Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); - - private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) - { - var yaml = YamlConverter.Serialize(json); - var content = BinaryData.FromString(yaml); - await file.OverwriteWithBinaryData(content, cancellationToken); - } -} +internal delegate ValueTask ValidateExtractorArtifacts(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); internal sealed record ExtractorOptions { @@ -70,6 +40,23 @@ internal sealed record ExtractorOptions public required Option DefaultApiSpecification { get; init; } public required Option> SubscriptionNamesToExport { get; init; } + public static ExtractorOptions NoFilter { get; } = new() + { + ApiNamesToExport = Option>.None, + BackendNamesToExport = Option>.None, + DefaultApiSpecification = Option.None, + DiagnosticNamesToExport = Option>.None, + GatewayNamesToExport = Option>.None, + GroupNamesToExport = Option>.None, + LoggerNamesToExport = Option>.None, + NamedValueNamesToExport = Option>.None, + PolicyFragmentNamesToExport = Option>.None, + ProductNamesToExport = Option>.None, + SubscriptionNamesToExport = Option>.None, + TagNamesToExport = Option>.None, + VersionSetNamesToExport = Option>.None + }; + public static Gen Generate(ServiceModel service) => from namedValues in GenerateOptionalNamesToExport(service.NamedValues) from tags in GenerateOptionalNamesToExport(service.Tags) @@ -192,7 +179,129 @@ public static bool ShouldExtract(T name, Option> namesToExport) // Run T.From(nameToFindString) var nameToFind = Expression.Lambda>(Expression.Call(typeof(T), "From", [], Expression.Constant(nameToFindString))).Compile()(); - + return names.Contains(nameToFind); }, () => true); } + +file sealed class RunExtractorHandler(ILogger logger, + ActivitySource activitySource, + GetSubscriptionId getSubscriptionId, + GetResourceGroupName getResourceGroupName, + GetBearerToken getBearerToken) +{ + public async ValueTask Handle(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(RunExtractor)); + + logger.LogInformation("Running extractor..."); + + var configurationFileOption = await TryGetConfigurationYamlFile(options, serviceDirectory, cancellationToken); + var arguments = await GetArguments(serviceName, serviceDirectory, configurationFileOption, cancellationToken); + await extractor.Program.Main(arguments); + } + + private static async ValueTask> TryGetConfigurationYamlFile(ExtractorOptions extractorOptions, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var optionsJson = extractorOptions.ToJsonObject(); + if (optionsJson.Count == 0) + { + return Option.None; + } + + var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.extractor.yaml"); + var yamlFile = new FileInfo(yamlFilePath); + await WriteYamlToFile(optionsJson, yamlFile, cancellationToken); + + return yamlFile; + } + + private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) + { + var yaml = YamlConverter.Serialize(json); + var content = BinaryData.FromString(yaml); + await file.OverwriteWithBinaryData(content, cancellationToken); + } + + private async ValueTask GetArguments(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, Option configurationFileOption, CancellationToken cancellationToken) + { + var argumentDictionary = new Dictionary + { + [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), + ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, + ["AZURE_SUBSCRIPTION_ID"] = getSubscriptionId(), + ["AZURE_RESOURCE_GROUP_NAME"] = getResourceGroupName(), + ["AZURE_BEARER_TOKEN"] = await getBearerToken(cancellationToken) + }; + +#pragma warning disable CA1849 // Call async methods when in an async method + configurationFileOption.Iter(file => argumentDictionary.Add("CONFIGURATION_YAML_PATH", file.FullName)); +#pragma warning restore CA1849 // Call async methods when in an async method + + return argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); + } + + private static string GetApiManagementServiceNameParameter() => + Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); +} + +file sealed class ValidateExtractorArtifactsHandler(ILogger logger, + ActivitySource activitySource, + ValidateExtractedNamedValues validateNamedValues, + ValidateExtractedTags validateTags, + ValidateExtractedVersionSets validateVersionSets, + ValidateExtractedBackends validateBackends, + ValidateExtractedLoggers validateLoggers, + ValidateExtractedDiagnostics validateDiagnostics, + ValidateExtractedPolicyFragments validatePolicyFragments, + ValidateExtractedServicePolicies validateServicePolicies, + ValidateExtractedGroups validateGroups, + ValidateExtractedProducts validateProducts, + ValidateExtractedApis validateApis) +{ + public async ValueTask Handle(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractorArtifacts)); + + logger.LogInformation("Validating extractor artifacts..."); + + await validateNamedValues(options.NamedValueNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateTags(options.TagNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateVersionSets(options.VersionSetNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateBackends(options.BackendNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateLoggers(options.LoggerNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateDiagnostics(options.DiagnosticNamesToExport, options.LoggerNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validatePolicyFragments(options.PolicyFragmentNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateServicePolicies(serviceName, serviceDirectory, cancellationToken); + await validateGroups(options.GroupNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateProducts(options.ProductNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateApis(options.ApiNamesToExport, options.DefaultApiSpecification, options.VersionSetNamesToExport, serviceName, serviceDirectory, cancellationToken); + } +} + +internal static class ExtractorServices +{ + public static void ConfigureRunExtractor(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidateExtractorArtifacts(IServiceCollection services) + { + NamedValueServices.ConfigureValidateExtractedNamedValues(services); + TagServices.ConfigureValidateExtractedTags(services); + VersionSetServices.ConfigureValidateExtractedVersionSets(services); + BackendServices.ConfigureValidateExtractedBackends(services); + LoggerServices.ConfigureValidateExtractedLoggers(services); + DiagnosticServices.ConfigureValidateExtractedDiagnostics(services); + PolicyFragmentServices.ConfigureValidateExtractedPolicyFragments(services); + ServicePolicyServices.ConfigureValidateExtractedServicePolicies(services); + GroupServices.ConfigureValidateExtractedGroups(services); + ProductServices.ConfigureValidateExtractedProducts(services); + ApiServices.ConfigureValidateExtractedApis(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/Fixture.cs b/tools/code/integration.tests/Fixture.cs index f7adc76c..1da9bfe0 100644 --- a/tools/code/integration.tests/Fixture.cs +++ b/tools/code/integration.tests/Fixture.cs @@ -1,4 +1,4 @@ -using common; +using common; using common.tests; using CsCheck; using LanguageExt; @@ -177,4 +177,4 @@ private sealed record ChangeParameters public Option MaxSize { get; init; } = Option.None; } -} +} \ No newline at end of file diff --git a/tools/code/integration.tests/Gateway.cs b/tools/code/integration.tests/Gateway.cs index d9796d05..0641b886 100644 --- a/tools/code/integration.tests/Gateway.cs +++ b/tools/code/integration.tests/Gateway.cs @@ -5,16 +5,45 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; +internal delegate ValueTask DeleteAllGateways(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file sealed class DeleteAllGatewaysHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllGateways)); + + logger.LogInformation("Deleting all gateways in {ServiceName}.", serviceName); + var serviceUri = getServiceUri(serviceName); + await GatewaysUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +internal static class GatewayServices +{ + public static void ConfigureDeleteAllGateways(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + internal static class Gateway { public static Gen GenerateUpdate(GatewayModel original) => diff --git a/tools/code/integration.tests/Group.cs b/tools/code/integration.tests/Group.cs index 5d5d1a91..e158119a 100644 --- a/tools/code/integration.tests/Group.cs +++ b/tools/code/integration.tests/Group.cs @@ -5,41 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Group +internal delegate ValueTask DeleteAllGroups(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutGroupModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedGroups(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimGroups(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileGroups(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteGroupModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedGroups(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllGroupsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(GroupModel original) => - from displayName in GroupModel.GenerateDisplayName() - from description in GroupModel.GenerateDescription().OptionOf() - select original with - { - DisplayName = displayName, - Description = description - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllGroups)); - public static Gen GenerateOverride(GroupDto original) => - from displayName in GroupModel.GenerateDisplayName() - from description in GroupModel.GenerateDescription().OptionOf() - select new GroupDto + logger.LogInformation("Deleting all groups in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await GroupsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutGroupModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutGroupModels)); + + logger.LogInformation("Putting group models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new GroupDto.GroupContract - { - DisplayName = displayName, - Description = description.ValueUnsafe() - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(GroupModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = GroupUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static GroupDto GetDto(GroupModel model) => new() @@ -50,29 +76,110 @@ private static GroupDto GetDto(GroupModel model) => Description = model.Description.ValueUnsafe() } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedGroupsHandler(ILogger logger, GetApimGroups getApimResources, GetFileGroups getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedGroups)); + + logger.LogInformation("Validating extracted groups in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(GroupDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(GroupModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimGroupsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = GroupUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimGroups)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting groups from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = GroupsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await GroupsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileGroupsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileGroups)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Getting groups from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => GroupInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, GroupInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileGroups)); + + logger.LogInformation("Getting groups from {ServiceDirectory}...", serviceDirectory); + + return await GroupModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteGroupModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteGroupModels)); + + logger.LogInformation("Writing group models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(GroupModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -82,86 +189,135 @@ private static async ValueTask WriteInformationFile(GroupModel model, Management await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static GroupDto GetDto(GroupModel model) => + new() + { + Properties = new GroupDto.GroupContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe() + } + }; +} + +file sealed class ValidatePublishedGroupsHandler(ILogger logger, GetFileGroups getFileResources, GetApimGroups getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedGroups)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published groups in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = GroupsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .Where(kvp => (kvp.Value.Properties.Type?.Contains("system", StringComparison.OrdinalIgnoreCase) ?? false) is false) + .MapValue(NormalizeDto); + var actual = apimResources.Where(kvp => (kvp.Value.Properties.Type?.Contains("system", StringComparison.OrdinalIgnoreCase) ?? false) is false) + .MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await GroupModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(GroupDto dto) => new { DisplayName = dto.Properties.DisplayName ?? string.Empty, Description = dto.Properties.Description ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class GroupServices +{ + public static void ConfigureDeleteAllGroups(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutGroupModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .Where(kvp => (kvp.Value.Properties.Type?.Contains("system", StringComparison.OrdinalIgnoreCase) ?? false) is false) - .MapValue(NormalizeDto); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - var actual = apimResources.Where(kvp => (kvp.Value.Properties.Type?.Contains("system", StringComparison.OrdinalIgnoreCase) ?? false) is false) - .MapValue(NormalizeDto); + public static void ConfigureValidateExtractedGroups(IServiceCollection services) + { + ConfigureGetApimGroups(services); + ConfigureGetFileGroups(services); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetApimGroups(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => GroupInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetFileGroups(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, GroupInformationFile file, CancellationToken cancellationToken) + public static void ConfigureWriteGroupModels(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureValidatePublishedGroups(IServiceCollection services) + { + ConfigureGetFileGroups(services); + ConfigureGetApimGroups(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Group +{ + public static Gen GenerateUpdate(GroupModel original) => + from displayName in GroupModel.GenerateDisplayName() + from description in GroupModel.GenerateDescription().OptionOf() + select original with { - using (contents) + DisplayName = displayName, + Description = description + }; + + public static Gen GenerateOverride(GroupDto original) => + from displayName in GroupModel.GenerateDisplayName() + from description in GroupModel.GenerateDescription().OptionOf() + select new GroupDto + { + Properties = new GroupDto.GroupContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + DisplayName = displayName, + Description = description.ValueUnsafe() } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static GroupDto GetDto(GroupModel model) => + new() + { + Properties = new GroupDto.GroupContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe() + } + }; } diff --git a/tools/code/integration.tests/Logger.cs b/tools/code/integration.tests/Logger.cs index 302d57aa..cea70b0d 100644 --- a/tools/code/integration.tests/Logger.cs +++ b/tools/code/integration.tests/Logger.cs @@ -5,10 +5,14 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text.Json.Nodes; using System.Threading; @@ -16,33 +20,53 @@ namespace integration.tests; -internal static class Logger +internal delegate ValueTask DeleteAllLoggers(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutLoggerModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedLoggers(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimLoggers(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileLoggers(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteLoggerModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedLoggers(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllLoggersHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(LoggerModel original) => - from type in LoggerType.Generate() - from description in LoggerModel.GenerateDescription().OptionOf() - from isBuffered in Gen.Bool - select original with - { - Type = type, - Description = description, - IsBuffered = isBuffered - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllLoggers)); - public static Gen GenerateOverride(LoggerDto original) => - from description in LoggerModel.GenerateDescription().OptionOf() - from isBuffered in Gen.Bool - select new LoggerDto + logger.LogInformation("Deleting all loggers in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await LoggersUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutLoggerModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutLoggerModels)); + + logger.LogInformation("Putting logger models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new LoggerDto.LoggerContract - { - Description = description.ValueUnsafe(), - IsBuffered = isBuffered - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(LoggerModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = LoggerUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static LoggerDto GetDto(LoggerModel model) => new() @@ -79,29 +103,118 @@ private static LoggerDto GetDto(LoggerModel model) => } } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedLoggersHandler(ILogger logger, GetApimLoggers getApimResources, GetFileLoggers getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedLoggers)); + + logger.LogInformation("Validating extracted loggers in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(LoggerDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + LoggerType = dto.Properties.LoggerType ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty, + IsBuffered = dto.Properties.IsBuffered ?? false, + ResourceId = dto.Properties.ResourceId ?? string.Empty, + Credentials = new + { + Name = dto.Properties.Credentials?.TryGetStringProperty("name").ValueUnsafe() ?? string.Empty, + ConnectionString = dto.Properties.Credentials?.TryGetStringProperty("connectionString").ValueUnsafe() ?? string.Empty, + InstrumentationKey = dto.Properties.Credentials?.TryGetStringProperty("instrumentationKey").ValueUnsafe() ?? string.Empty + } + }.ToString()!; +} - private static async ValueTask Put(LoggerModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimLoggersHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = LoggerUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimLoggers)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting loggers from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = LoggersUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class GetFileLoggersHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileLoggers)); + + logger.LogInformation("Getting loggers from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => LoggerInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); } - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await LoggersUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, LoggerInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileLoggers)); + + logger.LogInformation("Getting loggers from {ServiceDirectory}...", serviceDirectory); + + return await LoggerModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteLoggerModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteLoggerModels)); + + logger.LogInformation("Writing logger models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(LoggerModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -111,33 +224,61 @@ private static async ValueTask WriteInformationFile(LoggerModel model, Managemen await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static LoggerDto GetDto(LoggerModel model) => + new() + { + Properties = new LoggerDto.LoggerContract + { + LoggerType = model.Type switch + { + LoggerType.ApplicationInsights => "applicationInsights", + LoggerType.AzureMonitor => "azureMonitor", + LoggerType.EventHub => "azureEventHub", + _ => throw new ArgumentException($"Model type '{model.Type}' is not supported.", nameof(model)) + }, + Description = model.Description.ValueUnsafe(), + IsBuffered = model.IsBuffered, + ResourceId = model.Type switch + { + LoggerType.ApplicationInsights applicationInsights => applicationInsights.ResourceId, + LoggerType.EventHub eventHub => eventHub.ResourceId, + _ => null + }, + Credentials = model.Type switch + { + LoggerType.ApplicationInsights applicationInsights => new JsonObject + { + ["instrumentationKey"] = $"{{{{{applicationInsights.InstrumentationKeyNamedValue}}}}}" + }, + LoggerType.EventHub eventHub => new JsonObject + { + ["name"] = eventHub.Name, + ["connectionString"] = $"{{{{{eventHub.ConnectionStringNamedValue}}}}}" + }, + _ => null + } + } + }; +} + +file sealed class ValidatePublishedLoggersHandler(ILogger logger, GetFileLoggers getFileResources, GetApimLoggers getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedLoggers)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published loggers in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = LoggersUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await LoggerModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(LoggerDto dto) => new { @@ -152,49 +293,126 @@ private static string NormalizeDto(LoggerDto dto) => InstrumentationKey = dto.Properties.Credentials?.TryGetStringProperty("instrumentationKey").ValueUnsafe() ?? string.Empty } }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class LoggerServices +{ + public static void ConfigureDeleteAllLoggers(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutLoggerModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedLoggers(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimLoggers(services); + ConfigureGetFileLoggers(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => LoggerInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimLoggers(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, LoggerInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileLoggers(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteLoggerModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedLoggers(IServiceCollection services) + { + ConfigureGetFileLoggers(services); + ConfigureGetApimLoggers(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Logger +{ + public static Gen GenerateUpdate(LoggerModel original) => + from type in LoggerType.Generate() + from description in LoggerModel.GenerateDescription().OptionOf() + from isBuffered in Gen.Bool + select original with { - using (contents) + Type = type, + Description = description, + IsBuffered = isBuffered + }; + + public static Gen GenerateOverride(LoggerDto original) => + from description in LoggerModel.GenerateDescription().OptionOf() + from isBuffered in Gen.Bool + select new LoggerDto + { + Properties = new LoggerDto.LoggerContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + Description = description.ValueUnsafe(), + IsBuffered = isBuffered } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static LoggerDto GetDto(LoggerModel model) => + new() + { + Properties = new LoggerDto.LoggerContract + { + LoggerType = model.Type switch + { + LoggerType.ApplicationInsights => "applicationInsights", + LoggerType.AzureMonitor => "azureMonitor", + LoggerType.EventHub => "azureEventHub", + _ => throw new ArgumentException($"Model type '{model.Type}' is not supported.", nameof(model)) + }, + Description = model.Description.ValueUnsafe(), + IsBuffered = model.IsBuffered, + ResourceId = model.Type switch + { + LoggerType.ApplicationInsights applicationInsights => applicationInsights.ResourceId, + LoggerType.EventHub eventHub => eventHub.ResourceId, + _ => null + }, + Credentials = model.Type switch + { + LoggerType.ApplicationInsights applicationInsights => new JsonObject + { + ["instrumentationKey"] = $"{{{{{applicationInsights.InstrumentationKeyNamedValue}}}}}" + }, + LoggerType.EventHub eventHub => new JsonObject + { + ["name"] = eventHub.Name, + ["connectionString"] = $"{{{{{eventHub.ConnectionStringNamedValue}}}}}" + }, + _ => null + } + } + }; } diff --git a/tools/code/integration.tests/ManagementService.cs b/tools/code/integration.tests/ManagementService.cs new file mode 100644 index 00000000..22926bed --- /dev/null +++ b/tools/code/integration.tests/ManagementService.cs @@ -0,0 +1,462 @@ +using Azure.Core; +using Azure.Core.Pipeline; +using common; +using common.tests; +using Flurl; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Polly; +using publisher; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace integration.tests; + +internal delegate IAsyncEnumerable ListApimServiceNames(CancellationToken cancellationToken); + +internal delegate ValueTask DeleteApimService(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask CreateApimService(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask EmptyApimService(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutServiceModel(ServiceModel serviceModel, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName); + +internal delegate ValueTask WriteServiceModelArtifacts(ServiceModel serviceModel, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask> WriteServiceModelCommits(IEnumerable serviceModels, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed record ManagementServiceProviderUri : ResourceUri +{ + private readonly Uri uri; + + public ManagementServiceProviderUri(Uri uri) + { + this.uri = uri; + } + + protected override Uri Value => uri; +} + +file sealed class ListApimServiceNamesHandler(ILogger logger, + ActivitySource activitySource, + HttpPipeline pipeline, + ManagementServiceProviderUri serviceProviderUri) +{ + public async IAsyncEnumerable Handle([EnumeratorCancellation] CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ListApimServiceNames)); + + logger.LogInformation("Listing APIM service names..."); + + var serviceNames = pipeline.ListJsonObjects(serviceProviderUri.ToUri(), cancellationToken) + .Choose(json => json.TryGetStringProperty("name").ToOption()) + .Select(ManagementServiceName.From); + + await foreach (var serviceName in serviceNames.WithCancellation(cancellationToken)) + { + yield return serviceName; + } + } +} + +file sealed class DeleteApimServiceHandler(ILogger logger, + ActivitySource activitySource, + HttpPipeline pipeline, + ManagementServiceProviderUri serviceProviderUri) +{ + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteApimService)); + + logger.LogInformation("Deleting APIM service {ServiceName}...", serviceName); + + var uri = serviceProviderUri.ToUri().AppendPathSegment(serviceName.Value).ToUri(); + + try + { + await pipeline.DeleteResource(uri, waitForCompletion: false, cancellationToken); + } + catch (HttpRequestException exception) when (exception.StatusCode == HttpStatusCode.Conflict && exception.Message.Contains("ServiceLocked", StringComparison.OrdinalIgnoreCase)) + { + } + } +} + +file sealed class CreateApimServiceHandler(ILogger logger, + ActivitySource activitySource, + HttpPipeline pipeline, + ManagementServiceProviderUri serviceProviderUri, + AzureLocation location) +{ + private static readonly ResiliencePipeline resiliencePipeline = + new ResiliencePipelineBuilder() + .AddRetry(new() + { + ShouldHandle = async arguments => + { + await ValueTask.CompletedTask; + var result = arguments.Outcome.Result; + var succeeded = "Succeeded".Equals(result, StringComparison.OrdinalIgnoreCase); + return succeeded is false; + }, + Delay = TimeSpan.FromSeconds(5), + BackoffType = DelayBackoffType.Linear, + MaxRetryAttempts = 100 + }) + .AddTimeout(TimeSpan.FromMinutes(3)) + .Build(); + + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(CreateApimService)); + + logger.LogInformation("Creating APIM service {ServiceName}...", serviceName); + + var uri = serviceProviderUri.ToUri().AppendPathSegment(serviceName.Value).ToUri(); + var body = BinaryData.FromObjectAsJson(new + { + location = location.Name, + sku = new + { + name = "StandardV2", + capacity = 1 + }, + identity = new + { + type = "SystemAssigned" + }, + properties = new + { + publisherEmail = "admin@contoso.com", + publisherName = "Contoso" + } + }); + + await pipeline.PutContent(uri, body, cancellationToken); + + // Wait until the service is successfully provisioned + await resiliencePipeline.ExecuteAsync(async cancellationToken => + { + var content = await pipeline.GetJsonObject(uri, cancellationToken); + + return content.TryGetJsonObjectProperty("properties") + .Bind(properties => properties.TryGetStringProperty("provisioningState")) + .IfLeft(string.Empty); + }, cancellationToken); + } +} + +file sealed class EmptyApimServiceHandler(ILogger logger, + ActivitySource activitySource, + DeleteAllSubscriptions deleteSubscriptions, + DeleteAllApis deleteApis, + DeleteAllGroups deleteGroups, + DeleteAllProducts deleteProducts, + DeleteAllServicePolicies deleteServicePolicies, + DeleteAllPolicyFragments deletePolicyFragments, + DeleteAllDiagnostics deleteDiagnostics, + DeleteAllLoggers deleteLoggers, + DeleteAllBackends deleteBackends, + DeleteAllVersionSets deleteVersionSets, + DeleteAllGateways deleteGateways, + DeleteAllTags deleteTags, + DeleteAllNamedValues deleteNamedValues) +{ + private static ResiliencePipeline resiliencePipeline = + new ResiliencePipelineBuilder() + .AddRetry(new() + { + BackoffType = DelayBackoffType.Constant, + UseJitter = true, + MaxRetryAttempts = 3, + ShouldHandle = new PredicateBuilder().Handle(exception => exception.StatusCode == HttpStatusCode.PreconditionFailed && exception.Message.Contains("Resource was modified since last retrieval", StringComparison.OrdinalIgnoreCase)) + }) + .Build(); + + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(EmptyApimService)); + + logger.LogInformation("Emptying APIM service {ServiceName}...", serviceName); + + await resiliencePipeline.ExecuteAsync(async cancellationToken => + { + await deleteSubscriptions(serviceName, cancellationToken); + await deleteApis(serviceName, cancellationToken); + await deleteGroups(serviceName, cancellationToken); + await deleteProducts(serviceName, cancellationToken); + await deleteServicePolicies(serviceName, cancellationToken); + await deletePolicyFragments(serviceName, cancellationToken); + await deleteDiagnostics(serviceName, cancellationToken); + await deleteLoggers(serviceName, cancellationToken); + await deleteBackends(serviceName, cancellationToken); + await deleteVersionSets(serviceName, cancellationToken); + await deleteGateways(serviceName, cancellationToken); + await deleteTags(serviceName, cancellationToken); + await deleteNamedValues(serviceName, cancellationToken); + }, cancellationToken); + } +} + +file sealed class PutServiceModelHandler(ILogger logger, + ActivitySource activitySource, + PutNamedValueModels putNamedValues, + PutTagModels putTags, + PutVersionSetModels putVersionSets, + PutBackendModels putBackends, + PutLoggerModels putLoggers, + PutDiagnosticModels putDiagnostics, + PutPolicyFragmentModels putPolicyFragments, + PutServicePolicyModels putServicePolicies, + PutGroupModels putGroups, + PutProductModels putProducts, + PutApiModels putApis) +{ + public async ValueTask Handle(ServiceModel serviceModel, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutServiceModel)); + + logger.LogInformation("Putting service model in APIM service {ServiceName}...", serviceName); + + await putNamedValues(serviceModel.NamedValues, serviceName, cancellationToken); + await putTags(serviceModel.Tags, serviceName, cancellationToken); + await putVersionSets(serviceModel.VersionSets, serviceName, cancellationToken); + await putBackends(serviceModel.Backends, serviceName, cancellationToken); + await putLoggers(serviceModel.Loggers, serviceName, cancellationToken); + await putDiagnostics(serviceModel.Diagnostics, serviceName, cancellationToken); + await putPolicyFragments(serviceModel.PolicyFragments, serviceName, cancellationToken); + await putServicePolicies(serviceModel.ServicePolicies, serviceName, cancellationToken); + await putGroups(serviceModel.Groups, serviceName, cancellationToken); + await putProducts(serviceModel.Products, serviceName, cancellationToken); + await putApis(serviceModel.Apis, serviceName, cancellationToken); + } +} + +file sealed class GetManagementServiceUriHandler(ManagementServiceProviderUri serviceProviderUri) +{ + public ManagementServiceUri Handle(ManagementServiceName serviceName) => + ManagementServiceUri.From(serviceProviderUri.ToUri() + .AppendPathSegment(serviceName.Value) + .ToUri()); +} + +file sealed class WriteServiceModelArtifactsHandler(ILogger logger, + ActivitySource activitySource, + WriteNamedValueModels writeNamedValues, + WriteTagModels writeTags, + WriteVersionSetModels writeVersionSets, + WriteBackendModels writeBackends, + WriteLoggerModels writeLoggers, + WriteDiagnosticModels writeDiagnostics, + WritePolicyFragmentModels writePolicyFragments, + WriteServicePolicyModels writeServicePolicies, + WriteGroupModels writeGroups, + WriteProductModels writeProducts, + WriteApiModels writeApis) +{ + public async ValueTask Handle(ServiceModel serviceModel, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteServiceModelArtifacts)); + + logger.LogInformation("Writing service model artifacts to {ServiceDirectory}...", serviceDirectory); + + await writeNamedValues(serviceModel.NamedValues, serviceDirectory, cancellationToken); + await writeTags(serviceModel.Tags, serviceDirectory, cancellationToken); + await writeVersionSets(serviceModel.VersionSets, serviceDirectory, cancellationToken); + await writeBackends(serviceModel.Backends, serviceDirectory, cancellationToken); + await writeLoggers(serviceModel.Loggers, serviceDirectory, cancellationToken); + await writeDiagnostics(serviceModel.Diagnostics, serviceDirectory, cancellationToken); + await writePolicyFragments(serviceModel.PolicyFragments, serviceDirectory, cancellationToken); + await writeServicePolicies(serviceModel.ServicePolicies, serviceDirectory, cancellationToken); + await writeGroups(serviceModel.Groups, serviceDirectory, cancellationToken); + await writeProducts(serviceModel.Products, serviceDirectory, cancellationToken); + await writeApis(serviceModel.Apis, serviceDirectory, cancellationToken); + } +} + +file sealed class WriteServiceModelCommitsHandler(ILogger logger, + ActivitySource activitySource, + WriteServiceModelArtifacts writeServiceModelArtifacts) +{ + public async ValueTask> Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteServiceModelCommits)); + + logger.LogInformation("Writing service model commits to {ServiceDirectory}...", serviceDirectory); + + var authorName = "apiops"; + var authorEmail = "apiops@apiops.com"; + var serviceDirectoryInfo = serviceDirectory.ToDirectoryInfo(); + Git.InitializeRepository(serviceDirectoryInfo, commitMessage: "Initial commit", authorName, authorEmail, DateTimeOffset.UtcNow); + + var commitIds = ImmutableArray.Empty; + await models.Map((index, model) => (index, model)) + .Iter(async x => + { + var (index, model) = x; + DeleteNonGitDirectories(serviceDirectory); + await writeServiceModelArtifacts(model, serviceDirectory, cancellationToken); + var commit = Git.CommitChanges(serviceDirectoryInfo, commitMessage: $"Commit {index}", authorName, authorEmail, DateTimeOffset.UtcNow); + var commitId = new CommitId(commit.Sha); + ImmutableInterlocked.Update(ref commitIds, commitIds => commitIds.Add(commitId)); + }, cancellationToken); + + return commitIds; + } + + private static void DeleteNonGitDirectories(ManagementServiceDirectory serviceDirectory) => + serviceDirectory.ToDirectoryInfo() + .ListDirectories("*") + .Where(directory => directory.Name.Equals(".git", StringComparison.OrdinalIgnoreCase) is false) + .Iter(directory => directory.ForceDelete()); +} + +internal static class ManagementServices +{ + public static void ConfigureListApimServiceNames(IServiceCollection services) + { + ConfigureManagementServiceProviderUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureManagementServiceProviderUri(IServiceCollection services) + { + services.TryAddSingleton(provider => + { + var configuration = provider.GetRequiredService(); + var apiVersion = configuration.TryGetValue("ARM_API_VERSION") + .IfNone(() => "2022-08-01"); + + var azureEnvironment = provider.GetRequiredService(); + var subscriptionId = provider.GetRequiredService().Invoke(); + var resourceGroupName = provider.GetRequiredService().Invoke(); + var uri = azureEnvironment.ManagementEndpoint + .AppendPathSegment("subscriptions") + .AppendPathSegment(subscriptionId) + .AppendPathSegment("resourceGroups") + .AppendPathSegment(resourceGroupName) + .AppendPathSegment("providers/Microsoft.ApiManagement/service") + .SetQueryParam("api-version", apiVersion) + .ToUri(); + + return new ManagementServiceProviderUri(uri); + }); + } + + public static void ConfigureDeleteApimService(IServiceCollection services) + { + ConfigureManagementServiceProviderUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureCreateApimService(IServiceCollection services) + { + ConfigureManagementServiceProviderUri(services); + ConfigureAzureLocation(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureAzureLocation(IServiceCollection services) + { + services.TryAddSingleton(typeof(AzureLocation), provider => + { + var configuration = provider.GetRequiredService(); + var locationName = configuration.TryGetValue("AZURE_LOCATION") + .IfNone("westus"); + + return new AzureLocation(locationName); + }); + } + + public static void ConfigureEmptyApimService(IServiceCollection services) + { + SubscriptionServices.ConfigureDeleteAllSubscriptions(services); + ApiServices.ConfigureDeleteAllApis(services); + GroupServices.ConfigureDeleteAllGroups(services); + ProductServices.ConfigureDeleteAllProducts(services); + ServicePolicyServices.ConfigureDeleteAllServicePolicies(services); + PolicyFragmentServices.ConfigureDeleteAllPolicyFragments(services); + DiagnosticServices.ConfigureDeleteAllDiagnostics(services); + LoggerServices.ConfigureDeleteAllLoggers(services); + BackendServices.ConfigureDeleteAllBackends(services); + VersionSetServices.ConfigureDeleteAllVersionSets(services); + GatewayServices.ConfigureDeleteAllGateways(services); + TagServices.ConfigureDeleteAllTags(services); + NamedValueServices.ConfigureDeleteAllNamedValues(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigurePutServiceModel(IServiceCollection services) + { + NamedValueServices.ConfigurePutNamedValueModels(services); + TagServices.ConfigurePutTagModels(services); + VersionSetServices.ConfigurePutVersionSetModels(services); + BackendServices.ConfigurePutBackendModels(services); + ApiServices.ConfigurePutApiModels(services); + LoggerServices.ConfigurePutLoggerModels(services); + DiagnosticServices.ConfigurePutDiagnosticModels(services); + PolicyFragmentServices.ConfigurePutPolicyFragmentModels(services); + ServicePolicyServices.ConfigurePutServicePolicyModels(services); + GroupServices.ConfigurePutGroupModels(services); + ProductServices.ConfigurePutProductModels(services); + ApiServices.ConfigurePutApiModels(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureGetManagementServiceUri(IServiceCollection services) + { + ConfigureManagementServiceProviderUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureWriteServiceModelArtifacts(IServiceCollection services) + { + NamedValueServices.ConfigureWriteNamedValueModels(services); + TagServices.ConfigureWriteTagModels(services); + VersionSetServices.ConfigureWriteVersionSetModels(services); + BackendServices.ConfigureWriteBackendModels(services); + LoggerServices.ConfigureWriteLoggerModels(services); + DiagnosticServices.ConfigureWriteDiagnosticModels(services); + PolicyFragmentServices.ConfigureWritePolicyFragmentModels(services); + ServicePolicyServices.ConfigureWriteServicePolicyModels(services); + GroupServices.ConfigureWriteGroupModels(services); + ProductServices.ConfigureWriteProductModels(services); + ApiServices.ConfigureWriteApiModels(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureWriteServiceModelCommits(IServiceCollection services) + { + ConfigureWriteServiceModelArtifacts(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/NamedValue.cs b/tools/code/integration.tests/NamedValue.cs index 69b14f62..b327e0f8 100644 --- a/tools/code/integration.tests/NamedValue.cs +++ b/tools/code/integration.tests/NamedValue.cs @@ -5,41 +5,69 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class NamedValue +internal delegate ValueTask DeleteAllNamedValues(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutNamedValueModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedNamedValues(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimNamedValues(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileNamedValues(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteNamedValueModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedNamedValues(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllNamedValuesHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(NamedValueModel original) => - from tags in NamedValueModel.GenerateTags() - select original with - { - Tags = tags - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllNamedValues)); - public static Gen GenerateOverride(NamedValueDto original) => - from value in Generator.AlphaNumericStringBetween(1, 100) - from tags in NamedValueModel.GenerateTags() - select new NamedValueDto + logger.LogInformation("Deleting all named values in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await NamedValuesUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutNamedValueModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutNamedValueModels)); + + logger.LogInformation("Putting named value models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new NamedValueDto.NamedValueContract - { - Value = value, - Tags = tags - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetPublisherDto); + private async ValueTask Put(NamedValueModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = NamedValueUri.From(model.Name, serviceUri); + var dto = GetDto(model); - private static NamedValueDto GetPublisherDto(NamedValueModel model) => + await uri.PutDto(dto, pipeline, cancellationToken); + } + + private static NamedValueDto GetDto(NamedValueModel model) => new() { Properties = new NamedValueDto.NamedValueContract @@ -64,39 +92,122 @@ private static NamedValueDto GetPublisherDto(NamedValueModel model) => } } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedNamedValuesHandler(ILogger logger, GetApimNamedValues getApimResources, GetFileNamedValues getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedNamedValues)); + + logger.LogInformation("Validating extracted named values in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(NamedValueDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Tags = string.Join(',', (dto.Properties.Tags ?? []).Order()), + Value = dto.Properties.Secret is true ? string.Empty : dto.Properties.Value ?? string.Empty, + Secret = dto.Properties.Secret + }.ToString()!; +} - private static async ValueTask Put(NamedValueModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimNamedValuesHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = NamedValueUri.From(model.Name, serviceUri); - var dto = GetPublisherDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimNamedValues)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting named values from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = NamedValuesUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await NamedValuesUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileNamedValuesHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileNamedValues)); + + logger.LogInformation("Getting named values from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => NamedValueInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, NamedValueInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileNamedValues)); + + logger.LogInformation("Getting named values from {ServiceDirectory}...", serviceDirectory); + + return await NamedValueModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteNamedValueModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteNamedValueModels)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Writing named value models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(NamedValueModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { var informationFile = NamedValueInformationFile.From(model.Name, serviceDirectory); - var dto = GetExtractorDto(model); + var dto = GetDto(model); await informationFile.WriteDto(dto, cancellationToken); } - private static NamedValueDto GetExtractorDto(NamedValueModel model) => + private static NamedValueDto GetDto(NamedValueModel model) => new() { Properties = new NamedValueDto.NamedValueContract @@ -116,34 +227,29 @@ private static NamedValueDto GetExtractorDto(NamedValueModel model) => Value = model.Type is NamedValueType.Default @default ? @default.Value : null } }; +} - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class ValidatePublishedNamedValuesHandler(ILogger logger, GetFileNamedValues getFileResources, GetApimNamedValues getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedNamedValues)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published named values in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = NamedValuesUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .WhereValue(dto => (dto.Properties.Secret is true + && dto.Properties.Value is null + && dto.Properties.KeyVault?.SecretIdentifier is null) is false) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await NamedValueModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(NamedValueDto dto) => new { @@ -152,52 +258,112 @@ private static string NormalizeDto(NamedValueDto dto) => Value = dto.Properties.Secret is true ? string.Empty : dto.Properties.Value ?? string.Empty, Secret = dto.Properties.Secret }.ToString()!; +} + +internal static class NamedValueServices +{ + public static void ConfigureDeleteAllNamedValues(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutNamedValueModels(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedNamedValues(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ConfigureGetApimNamedValues(services); + ConfigureGetFileNamedValues(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .WhereValue(dto => (dto.Properties.Secret is true - && dto.Properties.Value is null - && dto.Properties.KeyVault?.SecretIdentifier is null) is false) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetApimNamedValues(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => NamedValueInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetFileNamedValues(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, NamedValueInformationFile file, CancellationToken cancellationToken) + public static void ConfigureWriteNamedValueModels(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureValidatePublishedNamedValues(IServiceCollection services) + { + ConfigureGetFileNamedValues(services); + ConfigureGetApimNamedValues(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class NamedValue +{ + public static Gen GenerateUpdate(NamedValueModel original) => + from tags in NamedValueModel.GenerateTags() + select original with { - using (contents) + Tags = tags + }; + + public static Gen GenerateOverride(NamedValueDto original) => + from value in Generator.AlphaNumericStringBetween(1, 100) + from tags in NamedValueModel.GenerateTags() + select new NamedValueDto + { + Properties = new NamedValueDto.NamedValueContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + Value = value, + Tags = tags } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static NamedValueDto GetDto(NamedValueModel model) => + new() + { + Properties = new NamedValueDto.NamedValueContract + { + DisplayName = model.Name.ToString(), + Tags = model.Tags, + KeyVault = model.Type switch + { + NamedValueType.KeyVault keyVault => new NamedValueDto.KeyVaultContract + { + SecretIdentifier = keyVault.SecretIdentifier, + IdentityClientId = keyVault.IdentityClientId.ValueUnsafe() + }, + _ => null + }, + Secret = model.Type is NamedValueType.Secret or NamedValueType.KeyVault, + Value = model.Type switch + { + NamedValueType.Secret secret => secret.Value, + NamedValueType.Default @default => @default.Value, + _ => null + } + } + }; } diff --git a/tools/code/integration.tests/PolicyFragment.cs b/tools/code/integration.tests/PolicyFragment.cs index 5b511a95..12a0213c 100644 --- a/tools/code/integration.tests/PolicyFragment.cs +++ b/tools/code/integration.tests/PolicyFragment.cs @@ -5,42 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class PolicyFragment +internal delegate ValueTask DeleteAllPolicyFragments(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutPolicyFragmentModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedPolicyFragments(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimPolicyFragments(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFilePolicyFragments(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WritePolicyFragmentModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedPolicyFragments(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllPolicyFragmentsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(PolicyFragmentModel original) => - from description in PolicyFragmentModel.GenerateDescription().OptionOf() - from content in PolicyFragmentModel.GenerateContent() - select original with - { - Description = description, - Content = content - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllPolicyFragments)); - public static Gen GenerateOverride(PolicyFragmentDto original) => - from description in PolicyFragmentModel.GenerateDescription().OptionOf() - from content in PolicyFragmentModel.GenerateContent() - select new PolicyFragmentDto + logger.LogInformation("Deleting all policy fragments in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await PolicyFragmentsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutPolicyFragmentModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutPolicyFragmentModels)); + + logger.LogInformation("Putting policy fragment models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new PolicyFragmentDto.PolicyFragmentContract - { - Description = description.ValueUnsafe(), - Format = "rawxml", - Value = content - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(PolicyFragmentModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = PolicyFragmentUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static PolicyFragmentDto GetDto(PolicyFragmentModel model) => new() @@ -52,30 +77,110 @@ private static PolicyFragmentDto GetDto(PolicyFragmentModel model) => Value = model.Content } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedPolicyFragmentsHandler(ILogger logger, GetApimPolicyFragments getApimResources, GetFilePolicyFragments getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedPolicyFragments)); + + logger.LogInformation("Validating extracted policy fragments in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(PolicyFragmentDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + Description = dto.Properties.Description ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(PolicyFragmentModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimPolicyFragmentsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = PolicyFragmentUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimPolicyFragments)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting policy fragments from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = PolicyFragmentsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await PolicyFragmentsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFilePolicyFragmentsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFilePolicyFragments)); + + logger.LogInformation("Getting policy fragments from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => PolicyFragmentInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, PolicyFragmentInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFilePolicyFragments)); + + logger.LogInformation("Getting policy fragments from {ServiceDirectory}...", serviceDirectory); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + return await PolicyFragmentModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WritePolicyFragmentModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WritePolicyFragmentModels)); + + logger.LogInformation("Writing policy fragment models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); await WritePolicyFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(PolicyFragmentModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -85,87 +190,141 @@ private static async ValueTask WriteInformationFile(PolicyFragmentModel model, M await informationFile.WriteDto(dto, cancellationToken); } + private static PolicyFragmentDto GetDto(PolicyFragmentModel model) => + new() + { + Properties = new PolicyFragmentDto.PolicyFragmentContract + { + Description = model.Description.ValueUnsafe(), + Format = "rawxml", + Value = model.Content + } + }; + private static async ValueTask WritePolicyFile(PolicyFragmentModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { var policyFile = PolicyFragmentPolicyFile.From(model.Name, serviceDirectory); await policyFile.WritePolicy(model.Content, cancellationToken); } +} - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class ValidatePublishedPolicyFragmentsHandler(ILogger logger, GetFilePolicyFragments getFileResources, GetApimPolicyFragments getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedPolicyFragments)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published policy fragments in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = PolicyFragmentsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await PolicyFragmentModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(PolicyFragmentDto dto) => new { Description = dto.Properties.Description ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class PolicyFragmentServices +{ + public static void ConfigureDeleteAllPolicyFragments(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutPolicyFragmentModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedPolicyFragments(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimPolicyFragments(services); + ConfigureGetFilePolicyFragments(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => PolicyFragmentInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimPolicyFragments(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, PolicyFragmentInformationFile file, CancellationToken cancellationToken) + private static void ConfigureGetFilePolicyFragments(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWritePolicyFragmentModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedPolicyFragments(IServiceCollection services) + { + ConfigureGetFilePolicyFragments(services); + ConfigureGetApimPolicyFragments(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class PolicyFragment +{ + public static Gen GenerateUpdate(PolicyFragmentModel original) => + from description in PolicyFragmentModel.GenerateDescription().OptionOf() + from content in PolicyFragmentModel.GenerateContent() + select original with { - using (contents) + Description = description, + Content = content + }; + + public static Gen GenerateOverride(PolicyFragmentDto original) => + from description in PolicyFragmentModel.GenerateDescription().OptionOf() + from content in PolicyFragmentModel.GenerateContent() + select new PolicyFragmentDto + { + Properties = new PolicyFragmentDto.PolicyFragmentContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + Description = description.ValueUnsafe(), + Format = "rawxml", + Value = content } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static PolicyFragmentDto GetDto(PolicyFragmentModel model) => + new() + { + Properties = new PolicyFragmentDto.PolicyFragmentContract + { + Description = model.Description.ValueUnsafe(), + Format = "rawxml", + Value = model.Content + } + }; } diff --git a/tools/code/integration.tests/Product.cs b/tools/code/integration.tests/Product.cs index 79dc03a7..a32557ad 100644 --- a/tools/code/integration.tests/Product.cs +++ b/tools/code/integration.tests/Product.cs @@ -5,49 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Product +internal delegate ValueTask DeleteAllProducts(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutProductModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedProducts(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimProducts(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileProducts(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteProductModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedProducts(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllProductsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(ProductModel original) => - from displayName in ProductModel.GenerateDisplayName() - from description in ProductModel.GenerateDescription().OptionOf() - from terms in ProductModel.GenerateTerms().OptionOf() - from state in ProductModel.GenerateState() - select original with - { - DisplayName = displayName, - Description = description, - Terms = terms, - State = state - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllProducts)); - public static Gen GenerateOverride(ProductDto original) => - from displayName in ProductModel.GenerateDisplayName() - from description in ProductModel.GenerateDescription().OptionOf() - from terms in ProductModel.GenerateTerms().OptionOf() - from state in ProductModel.GenerateState() - select new ProductDto + logger.LogInformation("Deleting all products in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await ProductsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutProductModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutProductModels)); + + logger.LogInformation("Putting product models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new ProductDto.ProductContract - { - DisplayName = displayName, - Description = description.ValueUnsafe(), - Terms = terms.ValueUnsafe(), - State = state - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(ProductModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = ProductUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static ProductDto GetDto(ProductModel model) => new() @@ -60,39 +78,112 @@ private static ProductDto GetDto(ProductModel model) => State = model.State } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedProductsHandler(ILogger logger, GetApimProducts getApimResources, GetFileProducts getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedProducts)); + + logger.LogInformation("Validating extracted products in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(ProductDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty, + Terms = dto.Properties.Terms ?? string.Empty, + State = dto.Properties.State ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(ProductModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimProductsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = ProductUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimProducts)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting products from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = ProductsUri.From(serviceUri); - await ProductPolicy.Put(model.Policies, model.Name, serviceUri, pipeline, cancellationToken); - await ProductGroup.Put(model.Groups, model.Name, serviceUri, pipeline, cancellationToken); - await ProductTag.Put(model.Tags, model.Name, serviceUri, pipeline, cancellationToken); - await ProductApi.Put(model.Apis, model.Name, serviceUri, pipeline, cancellationToken); + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await ProductsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileProductsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileProducts)); + + logger.LogInformation("Getting products from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => ProductInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ProductInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileProducts)); + + logger.LogInformation("Getting products from {ServiceDirectory}...", serviceDirectory); + + return await ProductModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteProductModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteProductModels)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Writing product models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); - - await ProductPolicy.WriteArtifacts(model.Policies, model.Name, serviceDirectory, cancellationToken); - await ProductGroup.WriteArtifacts(model.Groups, model.Name, serviceDirectory, cancellationToken); - await ProductTag.WriteArtifacts(model.Tags, model.Name, serviceDirectory, cancellationToken); - await ProductApi.WriteArtifacts(model.Apis, model.Name, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(ProductModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -102,41 +193,37 @@ private static async ValueTask WriteInformationFile(ProductModel model, Manageme await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + private static ProductDto GetDto(ProductModel model) => + new() + { + Properties = new ProductDto.ProductContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe(), + Terms = model.Terms.ValueUnsafe(), + State = model.State + } + }; +} - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); +file sealed class ValidatePublishedProductsHandler(ILogger logger, GetFileProducts getFileResources, GetApimProducts getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidatePublishedProducts)); - actual.Should().BeEquivalentTo(expected); + logger.LogInformation("Validating published products in {ServiceDirectory}...", serviceDirectory); - await expected.Keys.IterParallel(async productName => - { - await ProductPolicy.ValidateExtractedArtifacts(serviceDirectory, productName, serviceUri, pipeline, cancellationToken); - await ProductGroup.ValidateExtractedArtifacts(serviceDirectory, productName, serviceUri, pipeline, cancellationToken); - await ProductTag.ValidateExtractedArtifacts(serviceDirectory, productName, serviceUri, pipeline, cancellationToken); - await ProductApi.ValidateExtractedArtifacts(serviceDirectory, productName, serviceUri, pipeline, cancellationToken); - }, cancellationToken); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = ProductsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await ProductModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(ProductDto dto) => new { @@ -145,65 +232,108 @@ private static string NormalizeDto(ProductDto dto) => Terms = dto.Properties.Terms ?? string.Empty, State = dto.Properties.State ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class ProductServices +{ + public static void ConfigureDeleteAllProducts(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - await fileResources.Keys.IterParallel(async productName => - { - await ProductPolicy.ValidatePublisherChanges(productName, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductGroup.ValidatePublisherChanges(productName, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductTag.ValidatePublisherChanges(productName, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductApi.ValidatePublisherChanges(productName, serviceDirectory, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutProductModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedProducts(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimProducts(services); + ConfigureGetFileProducts(services); - await fileResources.Keys.IterParallel(async productName => - { - await ProductPolicy.ValidatePublisherCommitChanges(productName, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductGroup.ValidatePublisherCommitChanges(productName, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductTag.ValidatePublisherCommitChanges(productName, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductApi.ValidatePublisherCommitChanges(productName, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => ProductInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimProducts(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ProductInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileProducts(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteProductModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedProducts(IServiceCollection services) + { + ConfigureGetFileProducts(services); + ConfigureGetApimProducts(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Product +{ + public static Gen GenerateUpdate(ProductModel original) => + from displayName in ProductModel.GenerateDisplayName() + from description in ProductModel.GenerateDescription().OptionOf() + from terms in ProductModel.GenerateTerms().OptionOf() + from state in ProductModel.GenerateState() + select original with { - using (contents) + DisplayName = displayName, + Description = description, + Terms = terms, + State = state + }; + + public static Gen GenerateOverride(ProductDto original) => + from displayName in ProductModel.GenerateDisplayName() + from description in ProductModel.GenerateDescription().OptionOf() + from terms in ProductModel.GenerateTerms().OptionOf() + from state in ProductModel.GenerateState() + select new ProductDto + { + Properties = new ProductDto.ProductContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + DisplayName = displayName, + Description = description.ValueUnsafe(), + Terms = terms.ValueUnsafe(), + State = state } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static ProductDto GetDto(ProductModel model) => + new() + { + Properties = new ProductDto.ProductContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe(), + Terms = model.Terms.ValueUnsafe(), + State = model.State + } + }; } diff --git a/tools/code/integration.tests/Program.cs b/tools/code/integration.tests/Program.cs new file mode 100644 index 00000000..d2a4f0fc --- /dev/null +++ b/tools/code/integration.tests/Program.cs @@ -0,0 +1,65 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Threading.Tasks; + +namespace integration.tests; + +internal static class Program +{ + public static async Task Main(string[] args) + { + var builder = GetHostBuilder(args); + using var host = builder.Build(); + await Run(host); + } + + private static HostApplicationBuilder GetHostBuilder(string[] arguments) + { + var builder = Host.CreateApplicationBuilder(arguments); + Configure(builder); + return builder; + } + + private static void Configure(HostApplicationBuilder builder) + { + Configure(builder.Configuration); + Configure(builder.Services); + } + + private static void Configure(IConfigurationBuilder builder) + { + builder.AddUserSecrets(typeof(Program).Assembly); ; + } + + private static void Configure(IServiceCollection services) + { + CommonServices.Configure(services); + AppServices.ConfigureRunApplication(services); + } + + private static async Task Run(IHost host) + { + var applicationLifetime = host.Services.GetRequiredService(); + var cancellationToken = applicationLifetime.ApplicationStopping; + + try + { + await host.StartAsync(cancellationToken); + var runApplication = host.Services.GetRequiredService(); + await runApplication(cancellationToken); + } + catch (Exception exception) + { + var logger = host.Services.GetRequiredService().CreateLogger(nameof(Program)); + logger.LogCritical(exception, "An unhandled exception occurred."); + throw; + } + finally + { + applicationLifetime.StopApplication(); + } + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/Publisher.cs b/tools/code/integration.tests/Publisher.cs index 55b013f3..372b217f 100644 --- a/tools/code/integration.tests/Publisher.cs +++ b/tools/code/integration.tests/Publisher.cs @@ -2,10 +2,14 @@ using common.tests; using CsCheck; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Linq.Expressions; @@ -16,51 +20,9 @@ namespace integration.tests; -internal static class Publisher -{ - public static async ValueTask Run(PublisherOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, string subscriptionId, string resourceGroupName, string bearerToken, Option commitId, CancellationToken cancellationToken) - { - var argumentDictionary = new Dictionary - { - [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), - ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, - ["AZURE_SUBSCRIPTION_ID"] = subscriptionId, - ["AZURE_RESOURCE_GROUP_NAME"] = resourceGroupName, - ["AZURE_BEARER_TOKEN"] = bearerToken, - ["Logging:LogLevel:Default"] = "Trace" - }; +internal delegate ValueTask RunPublisher(PublisherOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); -#pragma warning disable CA1849 // Call async methods when in an async method - commitId.Iter(id => argumentDictionary.Add("COMMIT_ID", id.Value)); -#pragma warning restore CA1849 // Call async methods when in an async method - - var yamlFile = await WriteConfigurationYaml(options, serviceDirectory, cancellationToken); - argumentDictionary.Add("CONFIGURATION_YAML_PATH", yamlFile.FullName); - - var arguments = argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); - await Program.Main(arguments); - } - - private static string GetApiManagementServiceNameParameter() => - Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); - - private static async ValueTask WriteConfigurationYaml(PublisherOptions options, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) - { - var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.publisher.yaml"); - var yamlFile = new FileInfo(yamlFilePath); - var json = options.ToJsonObject(); - await WriteYamlToFile(json, yamlFile, cancellationToken); - - return yamlFile; - } - - private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) - { - var yaml = YamlConverter.Serialize(json); - var content = BinaryData.FromString(yaml); - await file.OverwriteWithBinaryData(content, cancellationToken); - } -} +internal delegate ValueTask ValidatePublishedArtifacts(PublisherOptions options, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); internal sealed record PublisherOptions { @@ -178,3 +140,172 @@ public static FrozenDictionary Override(IDictionary logger, + ActivitySource activitySource, + GetSubscriptionId getSubscriptionId, + GetResourceGroupName getResourceGroupName, + GetBearerToken getBearerToken) +{ + public async ValueTask Handle(PublisherOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(RunPublisher)); + + logger.LogInformation("Running publisher..."); + + var configurationFileOption = await TryGetConfigurationYamlFile(options, serviceDirectory, cancellationToken); + var arguments = await GetArguments(serviceName, serviceDirectory, configurationFileOption, commitIdOption, cancellationToken); + await publisher.Program.Main(arguments); + } + + private static async ValueTask> TryGetConfigurationYamlFile(PublisherOptions options, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var optionsJson = options.ToJsonObject(); + if (optionsJson.Count == 0) + { + return Option.None; + } + + var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.publisher.yaml"); + var yamlFile = new FileInfo(yamlFilePath); + await WriteYamlToFile(optionsJson, yamlFile, cancellationToken); + + return yamlFile; + } + + private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) + { + var yaml = YamlConverter.Serialize(json); + var content = BinaryData.FromString(yaml); + await file.OverwriteWithBinaryData(content, cancellationToken); + } + + private async ValueTask GetArguments(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, Option configurationFileOption, Option commitIdOption, CancellationToken cancellationToken) + { + var argumentDictionary = new Dictionary + { + [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), + ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, + ["AZURE_SUBSCRIPTION_ID"] = getSubscriptionId(), + ["AZURE_RESOURCE_GROUP_NAME"] = getResourceGroupName(), + ["AZURE_BEARER_TOKEN"] = await getBearerToken(cancellationToken) + }; + +#pragma warning disable CA1849 // Call async methods when in an async method + configurationFileOption.Iter(file => argumentDictionary.Add("CONFIGURATION_YAML_PATH", file.FullName)); + commitIdOption.Iter(id => argumentDictionary.Add("COMMIT_ID", id.Value)); +#pragma warning restore CA1849 // Call async methods when in an async method + + return argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); + } + + private static string GetApiManagementServiceNameParameter() => + Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); +} + +file sealed class ValidatePublishedArtifactsHandler(ILogger logger, + ActivitySource activitySource, + ValidatePublishedNamedValues validateNamedValues, + ValidatePublishedTags validateTags, + ValidatePublishedVersionSets validateVersionSets, + ValidatePublishedBackends validateBackends, + ValidatePublishedLoggers validateLoggers, + ValidatePublishedDiagnostics validateDiagnostics, + ValidatePublishedPolicyFragments validatePolicyFragments, + ValidatePublishedServicePolicies validateServicePolicies, + ValidatePublishedGroups validateGroups, + ValidatePublishedProducts validateProducts, + ValidatePublishedApis validateApis) +{ + public async ValueTask Handle(PublisherOptions options, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidatePublishedArtifacts)); + + logger.LogInformation("Validating published artifacts..."); + + await validateNamedValues(options.NamedValueOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateTags(options.TagOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateVersionSets(options.VersionSetOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateBackends(options.BackendOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateLoggers(options.LoggerOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateDiagnostics(options.DiagnosticOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validatePolicyFragments(options.PolicyFragmentOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateServicePolicies(options.ServicePolicyOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateGroups(options.GroupOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateProducts(options.ProductOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateApis(options.ApiOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + } +} + +internal static class PublisherServices +{ + public static void ConfigureRunPublisher(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedArtifacts(IServiceCollection services) + { + NamedValueServices.ConfigureValidatePublishedNamedValues(services); + TagServices.ConfigureValidatePublishedTags(services); + VersionSetServices.ConfigureValidatePublishedVersionSets(services); + BackendServices.ConfigureValidatePublishedBackends(services); + LoggerServices.ConfigureValidatePublishedLoggers(services); + DiagnosticServices.ConfigureValidatePublishedDiagnostics(services); + PolicyFragmentServices.ConfigureValidatePublishedPolicyFragments(services); + ServicePolicyServices.ConfigureValidatePublishedServicePolicies(services); + GroupServices.ConfigureValidatePublishedGroups(services); + ProductServices.ConfigureValidatePublishedProducts(services); + ApiServices.ConfigureValidatePublishedApis(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Publisher +{ + public static async ValueTask Run(PublisherOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, string subscriptionId, string resourceGroupName, string bearerToken, Option commitId, CancellationToken cancellationToken) + { + var argumentDictionary = new Dictionary + { + [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), + ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, + ["AZURE_SUBSCRIPTION_ID"] = subscriptionId, + ["AZURE_RESOURCE_GROUP_NAME"] = resourceGroupName, + ["AZURE_BEARER_TOKEN"] = bearerToken, + ["Logging:LogLevel:Default"] = "Trace" + }; + +#pragma warning disable CA1849 // Call async methods when in an async method + commitId.Iter(id => argumentDictionary.Add("COMMIT_ID", id.Value)); +#pragma warning restore CA1849 // Call async methods when in an async method + + var yamlFile = await WriteConfigurationYaml(options, serviceDirectory, cancellationToken); + argumentDictionary.Add("CONFIGURATION_YAML_PATH", yamlFile.FullName); + + var arguments = argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); + await Program.Main(arguments); + } + + private static string GetApiManagementServiceNameParameter() => + Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); + + private static async ValueTask WriteConfigurationYaml(PublisherOptions options, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.publisher.yaml"); + var yamlFile = new FileInfo(yamlFilePath); + var json = options.ToJsonObject(); + await WriteYamlToFile(json, yamlFile, cancellationToken); + + return yamlFile; + } + + private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) + { + var yaml = YamlConverter.Serialize(json); + var content = BinaryData.FromString(yaml); + await file.OverwriteWithBinaryData(content, cancellationToken); + } +} diff --git a/tools/code/integration.tests/Service.cs b/tools/code/integration.tests/Service.cs deleted file mode 100644 index 72a036e1..00000000 --- a/tools/code/integration.tests/Service.cs +++ /dev/null @@ -1,266 +0,0 @@ -using Azure.Core.Pipeline; -using common; -using common.tests; -using Flurl; -using LanguageExt; -using Polly; -using publisher; -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace integration.tests; - -internal static class ServiceModule -{ - public static async ValueTask DeleteManagementServices(Uri serviceProviderUri, string serviceNamesToDeletePrefix, HttpPipeline pipeline, CancellationToken cancellationToken) - { - try - { - await pipeline.ListJsonObjects(serviceProviderUri, cancellationToken) - .Choose(json => json.TryGetStringProperty("name").ToOption()) - .Where(name => name.StartsWith(serviceNamesToDeletePrefix, StringComparison.OrdinalIgnoreCase)) - .Select(name => ManagementServiceUri.From(serviceProviderUri.AppendPathSegment(name).ToUri())) - .IterParallel(async uri => await DeleteManagementService(uri, pipeline, cancellationToken), cancellationToken); - } - catch (HttpRequestException) - { - return; - } - } - - public static async ValueTask DeleteManagementService(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await pipeline.DeleteResource(serviceUri.ToUri(), waitForCompletion: false, cancellationToken); - - public static async ValueTask CreateManagementService(ManagementServiceUri serviceUri, string location, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var body = BinaryData.FromObjectAsJson(new - { - location = location, - sku = new - { - name = "StandardV2", - capacity = 1 - }, - identity = new - { - type = "SystemAssigned" - }, - properties = new - { - publisherEmail = "admin@contoso.com", - publisherName = "Contoso" - } - }); - - await pipeline.PutContent(serviceUri.ToUri(), body, cancellationToken); - - // Wait until the service is successfully provisioned - var resiliencePipeline = GetCreationStatusResiliencePipeline(); - await resiliencePipeline.ExecuteAsync(async cancellationToken => - { - var content = await pipeline.GetJsonObject(serviceUri.ToUri(), cancellationToken); - - return content.TryGetJsonObjectProperty("properties") - .Bind(properties => properties.TryGetStringProperty("provisioningState")) - .IfLeft(string.Empty); - }, cancellationToken); - } - - private static ResiliencePipeline GetCreationStatusResiliencePipeline() => - new ResiliencePipelineBuilder() - .AddRetry(new() - { - ShouldHandle = async arguments => - { - await ValueTask.CompletedTask; - var result = arguments.Outcome.Result; - var succeeded = "Succeeded".Equals(result, StringComparison.OrdinalIgnoreCase); - return succeeded is false; - }, - Delay = TimeSpan.FromSeconds(5), - BackoffType = DelayBackoffType.Linear, - MaxRetryAttempts = 100 - }) - .AddTimeout(TimeSpan.FromMinutes(3)) - .Build(); - - public static async ValueTask Put(ServiceModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, bool putSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.Put(model.NamedValues, serviceUri, pipeline, cancellationToken); - await Tag.Put(model.Tags, serviceUri, pipeline, cancellationToken); - await VersionSet.Put(model.VersionSets, serviceUri, pipeline, cancellationToken); - await Backend.Put(model.Backends, serviceUri, pipeline, cancellationToken); - await Logger.Put(model.Loggers, serviceUri, pipeline, cancellationToken); - await Diagnostic.Put(model.Diagnostics, serviceUri, pipeline, cancellationToken); - await PolicyFragment.Put(model.PolicyFragments, serviceUri, pipeline, cancellationToken); - await Group.Put(model.Groups, serviceUri, pipeline, cancellationToken); - await Api.Put(model.Apis, serviceUri, pipeline, cancellationToken); - await ServicePolicy.Put(model.ServicePolicies, serviceUri, pipeline, cancellationToken); - await Product.Put(model.Products, serviceUri, pipeline, cancellationToken); - - if (putSpecialSkuResources) - { - await Gateway.Put(model.Gateways, serviceUri, pipeline, cancellationToken); - } - - await Subscription.Put(model.Subscriptions, serviceUri, pipeline, cancellationToken); - } - - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await ResiliencePipelines.DeletePolicy.ExecuteAsync(async cancellationToken => - { - await Subscription.DeleteAll(serviceUri, pipeline, cancellationToken); - await Api.DeleteAll(serviceUri, pipeline, cancellationToken); - await Group.DeleteAll(serviceUri, pipeline, cancellationToken); - await Product.DeleteAll(serviceUri, pipeline, cancellationToken); - await ServicePolicy.DeleteAll(serviceUri, pipeline, cancellationToken); - await PolicyFragment.DeleteAll(serviceUri, pipeline, cancellationToken); - await Diagnostic.DeleteAll(serviceUri, pipeline, cancellationToken); - await Logger.DeleteAll(serviceUri, pipeline, cancellationToken); - await Backend.DeleteAll(serviceUri, pipeline, cancellationToken); - await VersionSet.DeleteAll(serviceUri, pipeline, cancellationToken); - await Gateway.DeleteAll(serviceUri, pipeline, cancellationToken); - await Tag.DeleteAll(serviceUri, pipeline, cancellationToken); - await NamedValue.DeleteAll(serviceUri, pipeline, cancellationToken); - }, cancellationToken); - - public static void DeleteServiceDirectory(ManagementServiceDirectory serviceDirectory) - { - var directoryInfo = serviceDirectory.ToDirectoryInfo(); - - if (directoryInfo.Exists()) - { - directoryInfo.ForceDelete(); - } - } - - public static async ValueTask ValidateExtractedArtifacts(ExtractorOptions extractorOptions, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, bool validateSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.ValidateExtractedArtifacts(extractorOptions.NamedValueNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Tag.ValidateExtractedArtifacts(extractorOptions.TagNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await VersionSet.ValidateExtractedArtifacts(extractorOptions.VersionSetNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Backend.ValidateExtractedArtifacts(extractorOptions.BackendNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Logger.ValidateExtractedArtifacts(extractorOptions.LoggerNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Diagnostic.ValidateExtractedArtifacts(extractorOptions.DiagnosticNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await PolicyFragment.ValidateExtractedArtifacts(extractorOptions.PolicyFragmentNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ServicePolicy.ValidateExtractedArtifacts(serviceDirectory, serviceUri, pipeline, cancellationToken); - await Product.ValidateExtractedArtifacts(extractorOptions.ProductNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Group.ValidateExtractedArtifacts(extractorOptions.GroupNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Api.ValidateExtractedArtifacts(extractorOptions.ApiNamesToExport, extractorOptions.DefaultApiSpecification, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Subscription.ValidateExtractedArtifacts(extractorOptions.SubscriptionNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - - if (validateSpecialSkuResources) - { - await Gateway.ValidateExtractedArtifacts(extractorOptions.GatewayNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - } - } - - public static async ValueTask WriteArtifacts(ServiceModel model, ManagementServiceDirectory serviceDirectory, bool writeSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.WriteArtifacts(model.NamedValues, serviceDirectory, cancellationToken); - await Tag.WriteArtifacts(model.Tags, serviceDirectory, cancellationToken); - await Gateway.WriteArtifacts(model.Gateways, serviceDirectory, cancellationToken); - await VersionSet.WriteArtifacts(model.VersionSets, serviceDirectory, cancellationToken); - await Backend.WriteArtifacts(model.Backends, serviceDirectory, cancellationToken); - await Logger.WriteArtifacts(model.Loggers, serviceDirectory, cancellationToken); - await Diagnostic.WriteArtifacts(model.Diagnostics, serviceDirectory, cancellationToken); - await PolicyFragment.WriteArtifacts(model.PolicyFragments, serviceDirectory, cancellationToken); - await ServicePolicy.WriteArtifacts(model.ServicePolicies, serviceDirectory, cancellationToken); - await Product.WriteArtifacts(model.Products, serviceDirectory, cancellationToken); - await Group.WriteArtifacts(model.Groups, serviceDirectory, cancellationToken); - await Api.WriteArtifacts(model.Apis, serviceDirectory, cancellationToken); - await Subscription.WriteArtifacts(model.Subscriptions, serviceDirectory, cancellationToken); - } - - public static async ValueTask ValidatePublisherChanges(PublisherOptions publisherOptions, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, bool validateSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.ValidatePublisherChanges(serviceDirectory, publisherOptions.NamedValueOverrides, serviceUri, pipeline, cancellationToken); - await Tag.ValidatePublisherChanges(serviceDirectory, publisherOptions.TagOverrides, serviceUri, pipeline, cancellationToken); - await VersionSet.ValidatePublisherChanges(serviceDirectory, publisherOptions.VersionSetOverrides, serviceUri, pipeline, cancellationToken); - await Backend.ValidatePublisherChanges(serviceDirectory, publisherOptions.BackendOverrides, serviceUri, pipeline, cancellationToken); - await Logger.ValidatePublisherChanges(serviceDirectory, publisherOptions.LoggerOverrides, serviceUri, pipeline, cancellationToken); - await Diagnostic.ValidatePublisherChanges(serviceDirectory, publisherOptions.DiagnosticOverrides, serviceUri, pipeline, cancellationToken); - await PolicyFragment.ValidatePublisherChanges(serviceDirectory, publisherOptions.PolicyFragmentOverrides, serviceUri, pipeline, cancellationToken); - await ServicePolicy.ValidatePublisherChanges(serviceDirectory, publisherOptions.ServicePolicyOverrides, serviceUri, pipeline, cancellationToken); - await Product.ValidatePublisherChanges(serviceDirectory, publisherOptions.ProductOverrides, serviceUri, pipeline, cancellationToken); - await Group.ValidatePublisherChanges(serviceDirectory, publisherOptions.GroupOverrides, serviceUri, pipeline, cancellationToken); - await Api.ValidatePublisherChanges(serviceDirectory, publisherOptions.ApiOverrides, serviceUri, pipeline, cancellationToken); - await Subscription.ValidatePublisherChanges(serviceDirectory, publisherOptions.SubscriptionOverrides, serviceUri, pipeline, cancellationToken); - - if (validateSpecialSkuResources) - { - await Gateway.ValidatePublisherChanges(serviceDirectory, publisherOptions.GatewayOverrides, serviceUri, pipeline, cancellationToken); - } - } - - public static async ValueTask> WriteCommitArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, bool writeSpecialSkuResources, CancellationToken cancellationToken) - { - var authorName = "apiops"; - var authorEmail = "apiops@apiops.com"; - var serviceDirectoryInfo = serviceDirectory.ToDirectoryInfo(); - Git.InitializeRepository(serviceDirectoryInfo, commitMessage: "Initial commit", authorName, authorEmail, DateTimeOffset.UtcNow); - - var commitIds = ImmutableArray.Empty; - await models.Map((index, model) => (index, model)) - .Iter(async x => - { - var (index, model) = x; - DeleteNonGitDirectories(serviceDirectory); - await WriteArtifacts(model, serviceDirectory, writeSpecialSkuResources, cancellationToken); - var commit = Git.CommitChanges(serviceDirectoryInfo, commitMessage: $"Commit {index}", authorName, authorEmail, DateTimeOffset.UtcNow); - var commitId = new CommitId(commit.Sha); - ImmutableInterlocked.Update(ref commitIds, commitIds => commitIds.Add(commitId)); - }, cancellationToken); - - return commitIds; - } - - private static void DeleteNonGitDirectories(ManagementServiceDirectory serviceDirectory) => - serviceDirectory.ToDirectoryInfo() - .ListDirectories("*") - .Where(directory => directory.Name.Equals(".git", StringComparison.OrdinalIgnoreCase) is false) - .Iter(directory => directory.Delete(recursive: true)); - - public static async ValueTask ValidatePublisherCommitChanges(PublisherOptions publisherOptions, CommitId commitId, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, bool validateSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.NamedValueOverrides, serviceUri, pipeline, cancellationToken); - await Tag.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.TagOverrides, serviceUri, pipeline, cancellationToken); - await VersionSet.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.VersionSetOverrides, serviceUri, pipeline, cancellationToken); - await Backend.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.BackendOverrides, serviceUri, pipeline, cancellationToken); - await Logger.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.LoggerOverrides, serviceUri, pipeline, cancellationToken); - await Diagnostic.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.DiagnosticOverrides, serviceUri, pipeline, cancellationToken); - await PolicyFragment.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.PolicyFragmentOverrides, serviceUri, pipeline, cancellationToken); - await ServicePolicy.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.ServicePolicyOverrides, serviceUri, pipeline, cancellationToken); - await Product.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.ProductOverrides, serviceUri, pipeline, cancellationToken); - await Group.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.GroupOverrides, serviceUri, pipeline, cancellationToken); - await Api.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.ApiOverrides, serviceUri, pipeline, cancellationToken); - await Subscription.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.SubscriptionOverrides, serviceUri, pipeline, cancellationToken); - - if (validateSpecialSkuResources) - { - await Gateway.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.GatewayOverrides, serviceUri, pipeline, cancellationToken); - } - } -} - -file static class ResiliencePipelines -{ - private static readonly Lazy deletePolicy = new(() => - new ResiliencePipelineBuilder() - .AddRetry(new() - { - BackoffType = DelayBackoffType.Constant, - UseJitter = true, - MaxRetryAttempts = 3, - ShouldHandle = new PredicateBuilder().Handle(exception => exception.StatusCode == HttpStatusCode.PreconditionFailed && exception.Message.Contains("Resource was modified since last retrieval", StringComparison.OrdinalIgnoreCase)) - }) - .Build()); - - public static ResiliencePipeline DeletePolicy => deletePolicy.Value; -} \ No newline at end of file diff --git a/tools/code/integration.tests/ServicePolicy.cs b/tools/code/integration.tests/ServicePolicy.cs index 6a3c1e4f..978e47c6 100644 --- a/tools/code/integration.tests/ServicePolicy.cs +++ b/tools/code/integration.tests/ServicePolicy.cs @@ -4,37 +4,67 @@ using CsCheck; using FluentAssertions; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class ServicePolicy +internal delegate ValueTask DeleteAllServicePolicies(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutServicePolicyModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedServicePolicies(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimServicePolicies(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileServicePolicies(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteServicePolicyModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedServicePolicies(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllServicePoliciesHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(ServicePolicyModel original) => - from content in ServicePolicyModel.GenerateContent() - select original with - { - Content = content - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllServicePolicies)); - public static Gen GenerateOverride(ServicePolicyDto original) => - from content in ServicePolicyModel.GenerateContent() - select new ServicePolicyDto + logger.LogInformation("Deleting all service policies in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await ServicePoliciesUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutServicePolicyModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutServicePolicyModels)); + + logger.LogInformation("Putting version set models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new ServicePolicyDto.ServicePolicyContract - { - Value = content - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(ServicePolicyModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = ServicePolicyUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static ServicePolicyDto GetDto(ServicePolicyModel model) => new() @@ -45,29 +75,123 @@ private static ServicePolicyDto GetDto(ServicePolicyModel model) => Value = model.Content } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedServicePoliciesHandler(ILogger logger, GetApimServicePolicies getApimResources, GetFileServicePolicies getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedServicePolicies)); + + logger.LogInformation("Validating extracted service policies in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(ServicePolicyDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + Value = new string((dto.Properties.Value ?? string.Empty) + .ReplaceLineEndings(string.Empty) + .Where(c => char.IsWhiteSpace(c) is false) + .ToArray()) + }.ToString()!; +} - private static async ValueTask Put(ServicePolicyModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimServicePoliciesHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = ServicePolicyUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimServicePolicies)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting service policies from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = ServicePoliciesUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await ServicePoliciesUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileServicePoliciesHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileServicePolicies)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Getting service policies from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => ServicePolicyFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ServicePolicyFile file, CancellationToken cancellationToken) + { + var name = file.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = new ServicePolicyDto + { + Properties = new ServicePolicyDto.ServicePolicyContract + { + Value = data.ToString() + } + }; + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileServicePolicies)); + + logger.LogInformation("Getting service policies from {ServiceDirectory}...", serviceDirectory); + + return await ServicePolicyModule.ListPolicyFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Name, + new ServicePolicyDto + { + Properties = new ServicePolicyDto.ServicePolicyContract + { + Value = await file.ReadPolicy(cancellationToken) + } + })) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteServicePolicyModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteServicePolicyModels)); + + logger.LogInformation("Writing version set models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WritePolicyFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WritePolicyFile(ServicePolicyModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -75,38 +199,35 @@ private static async ValueTask WritePolicyFile(ServicePolicyModel model, Managem await policyFile.WritePolicy(model.Content, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static ServicePolicyDto GetDto(ServicePolicyModel model) => + new() + { + Properties = new ServicePolicyDto.ServicePolicyContract + { + Format = "rawxml", + Value = model.Content + } + }; +} + +file sealed class ValidatePublishedServicePoliciesHandler(ILogger logger, GetFileServicePolicies getFileResources, GetApimServicePolicies getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedServicePolicies)); - var expected = apimResources.MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published service policies in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = ServicePoliciesUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await ServicePolicyModule.ListPolicyFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Name, - new ServicePolicyDto - { - Properties = new ServicePolicyDto.ServicePolicyContract - { - Value = await file.ReadPolicy(cancellationToken) - } - })) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(ServicePolicyDto dto) => new { @@ -115,55 +236,94 @@ private static string NormalizeDto(ServicePolicyDto dto) => .Where(c => char.IsWhiteSpace(c) is false) .ToArray()) }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class ServicePolicyServices +{ + public static void ConfigureDeleteAllServicePolicies(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutServicePolicyModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedServicePolicies(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimServicePolicies(services); + ConfigureGetFileServicePolicies(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => ServicePolicyFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimServicePolicies(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ServicePolicyFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileServicePolicies(IServiceCollection services) { - var name = file.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteServicePolicyModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedServicePolicies(IServiceCollection services) + { + ConfigureGetFileServicePolicies(services); + ConfigureGetApimServicePolicies(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class ServicePolicy +{ + public static Gen GenerateUpdate(ServicePolicyModel original) => + from content in ServicePolicyModel.GenerateContent() + select original with { - using (contents) + Content = content + }; + + public static Gen GenerateOverride(ServicePolicyDto original) => + from content in ServicePolicyModel.GenerateContent() + select new ServicePolicyDto + { + Properties = new ServicePolicyDto.ServicePolicyContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = new ServicePolicyDto - { - Properties = new ServicePolicyDto.ServicePolicyContract - { - Value = data.ToString() - } - }; - return (name, dto); + Value = content } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static ServicePolicyDto GetDto(ServicePolicyModel model) => + new() + { + Properties = new ServicePolicyDto.ServicePolicyContract + { + Format = "rawxml", + Value = model.Content + } + }; } diff --git a/tools/code/integration.tests/Subscription.cs b/tools/code/integration.tests/Subscription.cs index 5a4f22d5..937ae56f 100644 --- a/tools/code/integration.tests/Subscription.cs +++ b/tools/code/integration.tests/Subscription.cs @@ -4,17 +4,45 @@ using CsCheck; using FluentAssertions; using LanguageExt; -using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; +internal delegate ValueTask DeleteAllSubscriptions(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file sealed class DeleteAllSubscriptionsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllSubscriptions)); + + logger.LogInformation("Deleting all subscriptions in {ServiceName}.", serviceName); + var serviceUri = getServiceUri(serviceName); + await SubscriptionsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +internal static class SubscriptionServices +{ + public static void ConfigureDeleteAllSubscriptions(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + internal static class Subscription { public static Gen GenerateUpdate(SubscriptionModel original) => diff --git a/tools/code/integration.tests/Tag.cs b/tools/code/integration.tests/Tag.cs index 12720a92..69e23551 100644 --- a/tools/code/integration.tests/Tag.cs +++ b/tools/code/integration.tests/Tag.cs @@ -4,37 +4,67 @@ using CsCheck; using FluentAssertions; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Tag +internal delegate ValueTask DeleteAllTags(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutTagModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedTags(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimTags(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileTags(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteTagModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedTags(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllTagsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(TagModel original) => - from displayName in TagModel.GenerateDisplayName() - select original with - { - DisplayName = displayName - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllTags)); - public static Gen GenerateOverride(TagDto original) => - from displayName in TagModel.GenerateDisplayName() - select new TagDto + logger.LogInformation("Deleting all tags in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await TagsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutTagModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutTagModels)); + + logger.LogInformation("Putting tag models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new TagDto.TagContract - { - DisplayName = displayName - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(TagModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = TagUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static TagDto GetDto(TagModel model) => new() @@ -44,29 +74,109 @@ private static TagDto GetDto(TagModel model) => DisplayName = model.DisplayName } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedTagsHandler(ILogger logger, GetApimTags getApimResources, GetFileTags getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedTags)); + + logger.LogInformation("Validating extracted tags in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(TagDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(TagModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimTagsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = TagUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimTags)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting tags from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = TagsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await TagsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileTagsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileTags)); + + logger.LogInformation("Getting tags from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => TagInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, TagInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileTags)); + + logger.LogInformation("Getting tags from {ServiceDirectory}...", serviceDirectory); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + return await TagModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteTagModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteTagModels)); + + logger.LogInformation("Writing tag models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(TagModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -76,81 +186,126 @@ private static async ValueTask WriteInformationFile(TagModel model, ManagementSe await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static TagDto GetDto(TagModel model) => + new() + { + Properties = new TagDto.TagContract + { + DisplayName = model.DisplayName + } + }; +} + +file sealed class ValidatePublishedTagsHandler(ILogger logger, GetFileTags getFileResources, GetApimTags getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedTags)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published tags in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = TagsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await TagModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(TagDto dto) => new { DisplayName = dto.Properties.DisplayName ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class TagServices +{ + public static void ConfigureDeleteAllTags(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutTagModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidateExtractedTags(IServiceCollection services) + { + ConfigureGetApimTags(services); + ConfigureGetFileTags(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetApimTags(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => TagInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetFileTags(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, TagInformationFile file, CancellationToken cancellationToken) + public static void ConfigureWriteTagModels(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureValidatePublishedTags(IServiceCollection services) + { + ConfigureGetFileTags(services); + ConfigureGetApimTags(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Tag +{ + public static Gen GenerateUpdate(TagModel original) => + from displayName in TagModel.GenerateDisplayName() + select original with { - using (contents) + DisplayName = displayName + }; + + public static Gen GenerateOverride(TagDto original) => + from displayName in TagModel.GenerateDisplayName() + select new TagDto + { + Properties = new TagDto.TagContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + DisplayName = displayName } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static TagDto GetDto(TagModel model) => + new() + { + Properties = new TagDto.TagContract + { + DisplayName = model.DisplayName + } + }; } diff --git a/tools/code/integration.tests/Test.cs b/tools/code/integration.tests/Test.cs new file mode 100644 index 00000000..af5031f9 --- /dev/null +++ b/tools/code/integration.tests/Test.cs @@ -0,0 +1,506 @@ +using common; +using common.tests; +using CsCheck; +using LanguageExt; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using publisher; +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace integration.tests; + +internal delegate ValueTask RunTests(CancellationToken cancellationToken); + +file delegate ValueTask TestExtractor(CancellationToken cancellationToken); + +file delegate ValueTask TestExtractThenPublish(CancellationToken cancellationToken); + +file delegate ValueTask TestPublisher(CancellationToken cancellationToken); + +file delegate ValueTask CleanUpTests(CancellationToken cancellationToken); + +file sealed class RunTestsHandler(ILogger logger, + ActivitySource activitySource, + TestExtractor testExtractor, + TestExtractThenPublish testExtractThenPublish, + TestPublisher testPublisher, + CleanUpTests cleanUpTests) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(RunTests)); + + logger.LogInformation("Running tests..."); + await testExtractor(cancellationToken); + await testExtractThenPublish(cancellationToken); + await testPublisher(cancellationToken); + await cleanUpTests(cancellationToken); + } +} + +file sealed class TestExtractorHandler(ILogger logger, + IConfiguration configuration, + ActivitySource activitySource, + CreateApimService createService, + EmptyApimService emptyService, + PutServiceModel putServiceModel, + RunExtractor runExtractor, + ValidateExtractorArtifacts validateExtractor, + DeleteApimService deleteApimService) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(TestExtractor)); + + logger.LogInformation("Testing extractor..."); + + var generator = Fixture.Generate(configuration); + await generator.SampleAsync(async fixture => await Run(fixture, cancellationToken), iter: 1); + } + + private async ValueTask Run(Fixture fixture, CancellationToken cancellationToken) + { + await createService(fixture.ServiceName, cancellationToken); + await emptyService(fixture.ServiceName, cancellationToken); + await putServiceModel(fixture.ServiceModel, fixture.ServiceName, cancellationToken); + await runExtractor(fixture.ExtractorOptions, fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + await validateExtractor(fixture.ExtractorOptions, fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + await CleanUp(fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + } + + private async ValueTask CleanUp(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + if (serviceName.ToString().StartsWith(Common.TestServiceNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + await deleteApimService(serviceName, cancellationToken); + } + + Common.DeleteServiceDirectory(serviceDirectory); + } + + private sealed record Fixture + { + public required ExtractorOptions ExtractorOptions { get; init; } + public required ServiceModel ServiceModel { get; init; } + public required ManagementServiceName ServiceName { get; init; } + public required ManagementServiceDirectory ServiceDirectory { get; init; } + + public static Gen Generate(IConfiguration configuration) => + from serviceModel in ServiceModel.Generate() + from extractorOptions in ExtractorOptions.Generate(serviceModel) + from serviceName in Common.useExistingInstance + ? Gen.Const(ManagementServiceName.From(configuration.GetValue("FIRST_API_MANAGEMENT_SERVICE_NAME"))) + : Common.GenerateManagementServiceName(Common.TestServiceNamePrefix) + from serviceDirectory in Common.GenerateManagementServiceDirectory() + select new Fixture + { + ExtractorOptions = extractorOptions, + ServiceModel = serviceModel, + ServiceName = serviceName, + ServiceDirectory = serviceDirectory + }; + } +} + +file sealed class TestExtractThenPublishHandler(ILogger logger, + IConfiguration configuration, + ActivitySource activitySource, + CreateApimService createService, + PutServiceModel putServiceModel, + RunExtractor runExtractor, + DeleteApimService deleteApimService, + RunPublisher runPublisher, + ValidatePublishedArtifacts validatePublisher) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(TestExtractThenPublish)); + + logger.LogInformation("Testing extracting, then publishing to a fresh instance..."); + + var generator = Fixture.Generate(configuration); + await generator.SampleAsync(async fixture => await Run(fixture, cancellationToken), iter: 1); + } + + private async ValueTask Run(Fixture fixture, CancellationToken cancellationToken) + { + await CreateExtractorArtifacts(fixture.ServiceModel, fixture.SourceServiceName, fixture.ServiceDirectory, cancellationToken); + await PublishToDestination(fixture.PublisherOptions, fixture.DestinationServiceName, fixture.ServiceDirectory, cancellationToken); + await validatePublisher(fixture.PublisherOptions, Option.None, fixture.DestinationServiceName, fixture.ServiceDirectory, cancellationToken); + await CleanUp(fixture.SourceServiceName, fixture.DestinationServiceName, fixture.ServiceDirectory, cancellationToken); + } + + private async ValueTask CreateExtractorArtifacts(ServiceModel serviceModel, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(CreateExtractorArtifacts)); + + logger.LogInformation("Creating extractor artifacts..."); + + await createService(serviceName, cancellationToken); + await putServiceModel(serviceModel, serviceName, cancellationToken); + await runExtractor(ExtractorOptions.NoFilter, serviceName, serviceDirectory, cancellationToken); + await deleteApimService(serviceName, cancellationToken); + } + + private async ValueTask PublishToDestination(PublisherOptions publisherOptions, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PublishToDestination)); + + logger.LogInformation("Publishing artifacts to destination..."); + await createService(serviceName, cancellationToken); + await runPublisher(publisherOptions, serviceName, serviceDirectory, Option.None, cancellationToken); + } + + private async ValueTask CleanUp(ManagementServiceName sourceServiceName, ManagementServiceName destinationServiceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + await new[] { sourceServiceName, destinationServiceName } + .Where(name => name.ToString().StartsWith(Common.TestServiceNamePrefix, StringComparison.OrdinalIgnoreCase)) + .IterParallel(deleteApimService.Invoke, cancellationToken); + + Common.DeleteServiceDirectory(serviceDirectory); + } + + private sealed record Fixture + { + public required PublisherOptions PublisherOptions { get; init; } + public required ServiceModel ServiceModel { get; init; } + public required ManagementServiceName SourceServiceName { get; init; } + public required ManagementServiceName DestinationServiceName { get; init; } + public required ManagementServiceDirectory ServiceDirectory { get; init; } + + public static Gen Generate(IConfiguration configuration) => + from serviceModel in ServiceModel.Generate() + from publisherOptions in PublisherOptions.Generate(serviceModel) + from sourceServiceName in GenerateManagementServiceName(configuration, "FIRST_API_MANAGEMENT_SERVICE_NAME") + from destinationServiceName in GenerateManagementServiceName(configuration, "SECOND_API_MANAGEMENT_SERVICE_NAME") + from serviceDirectory in Common.GenerateManagementServiceDirectory() + select new Fixture + { + PublisherOptions = publisherOptions, + ServiceModel = serviceModel, + SourceServiceName = sourceServiceName, + DestinationServiceName = destinationServiceName, + ServiceDirectory = serviceDirectory + }; + + private static Gen GenerateManagementServiceName(IConfiguration configuration, string configurationKey) => + Common.useExistingInstance + ? Gen.Const(ManagementServiceName.From(configuration.GetValue(configurationKey))) + : Common.GenerateManagementServiceName(Common.TestServiceNamePrefix); + } +} + +file sealed class TestPublisherHandler(ILogger logger, + IConfiguration configuration, + ActivitySource activitySource, + WriteServiceModelArtifacts writeArtifacts, + CreateApimService createService, + EmptyApimService emptyService, + RunPublisher runPublisher, + ValidatePublishedArtifacts validatePublisher, + WriteServiceModelCommits writeCommits, + DeleteApimService deleteApimService) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(TestPublisher)); + + logger.LogInformation("Testing publisher..."); + + var generator = Fixture.Generate(configuration); + await generator.SampleAsync(async fixture => await Run(fixture, cancellationToken), iter: 1); + } + + private async ValueTask Run(Fixture fixture, CancellationToken cancellationToken) + { + await PublishAllChangesAndValidate(fixture.PublisherOptions, fixture.ServiceModel, fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + await PublishCommitsAndValidate(fixture.PublisherOptions, fixture.CommitModels, fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + await CleanUp(fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + } + + private async ValueTask PublishAllChangesAndValidate(PublisherOptions publisherOptions, ServiceModel serviceModel, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + await writeArtifacts(serviceModel, serviceDirectory, cancellationToken); + await createService(serviceName, cancellationToken); + await emptyService(serviceName, cancellationToken); + await runPublisher(publisherOptions, serviceName, serviceDirectory, Option.None, cancellationToken); + await validatePublisher(publisherOptions, Option.None, serviceName, serviceDirectory, cancellationToken); + } + + private async ValueTask PublishCommitsAndValidate(PublisherOptions publisherOptions, IEnumerable serviceModels, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var commits = await writeCommits(serviceModels, serviceDirectory, cancellationToken); + await runPublisher(publisherOptions, serviceName, serviceDirectory, commits.HeadOrNone(), cancellationToken); + await validatePublisher(publisherOptions, commits.HeadOrNone(), serviceName, serviceDirectory, cancellationToken); + } + + private async ValueTask CleanUp(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + if (serviceName.ToString().StartsWith(Common.TestServiceNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + await deleteApimService(serviceName, cancellationToken); + } + + Common.DeleteServiceDirectory(serviceDirectory); + } + + private sealed record Fixture + { + public required PublisherOptions PublisherOptions { get; init; } + public required ServiceModel ServiceModel { get; init; } + public required ImmutableArray CommitModels { get; init; } + public required ManagementServiceName ServiceName { get; init; } + public required ManagementServiceDirectory ServiceDirectory { get; init; } + + public static Gen Generate(IConfiguration configuration) => + from serviceModel in ServiceModel.Generate() + from commitModels in GenerateCommitModels(serviceModel) + from publisherOptions in PublisherOptions.Generate(serviceModel) + from serviceName in GenerateManagementServiceName(configuration, "FIRST_API_MANAGEMENT_SERVICE_NAME") + from serviceDirectory in Common.GenerateManagementServiceDirectory() + select new Fixture + { + PublisherOptions = publisherOptions, + ServiceModel = serviceModel, + CommitModels = commitModels, + ServiceName = serviceName, + ServiceDirectory = serviceDirectory + }; + + private static Gen GenerateManagementServiceName(IConfiguration configuration, string configurationKey) => + Common.useExistingInstance + ? Gen.Const(ManagementServiceName.From(configuration.GetValue(configurationKey))) + : Common.GenerateManagementServiceName(Common.TestServiceNamePrefix); + + private static Gen> GenerateCommitModels(ServiceModel initialModel) => + from list in Gen.Int.List[1, 10] + from aggregate in list.Aggregate(Gen.Const(ImmutableArray.Empty), + (gen, _) => from commits in gen + let lastCommit = commits.LastOrDefault(initialModel) + from newCommit in GenerateUpdatedServiceModel(lastCommit, ChangeParameters.All) + select commits.Add(newCommit)) + select aggregate; + + private static Gen GenerateUpdatedServiceModel(ServiceModel initialModel, ChangeParameters changeParameters) => + from namedValues in GenerateNewSet(initialModel.NamedValues, NamedValueModel.GenerateSet(), NamedValue.GenerateUpdate, changeParameters) + from tags in GenerateNewSet(initialModel.Tags, TagModel.GenerateSet(), Tag.GenerateUpdate, changeParameters) + from versionSets in GenerateNewSet(initialModel.VersionSets, VersionSetModel.GenerateSet(), VersionSet.GenerateUpdate, changeParameters) + from backends in GenerateNewSet(initialModel.Backends, BackendModel.GenerateSet(), Backend.GenerateUpdate, changeParameters) + from loggers in GenerateNewSet(initialModel.Loggers, LoggerModel.GenerateSet(), Logger.GenerateUpdate, changeParameters) + from diagnostics in from diagnosticSet in GenerateNewSet(initialModel.Diagnostics, DiagnosticModel.GenerateSet(), Diagnostic.GenerateUpdate, changeParameters) + from updatedDiagnostics in ServiceModel.UpdateDiagnostics(diagnosticSet, loggers) + select updatedDiagnostics + from policyFragments in GenerateNewSet(initialModel.PolicyFragments, PolicyFragmentModel.GenerateSet(), PolicyFragment.GenerateUpdate, changeParameters) + from servicePolicies in GenerateNewSet(initialModel.ServicePolicies, ServicePolicyModel.GenerateSet(), ServicePolicy.GenerateUpdate, changeParameters) + from groups in GenerateNewSet(initialModel.Groups, GroupModel.GenerateSet(), Group.GenerateUpdate, changeParameters) + from apis in from apiSet in GenerateNewSet(initialModel.Apis, ApiModel.GenerateSet(), Api.GenerateUpdate, changeParameters) + from updatedApis in ServiceModel.UpdateApis(apiSet, versionSets, tags) + select updatedApis + from products in from productSet in GenerateNewSet(initialModel.Products, ProductModel.GenerateSet(), Product.GenerateUpdate, changeParameters) + from updatedProductGroups in ServiceModel.UpdateProducts(productSet, groups, tags, apis) + select updatedProductGroups + from gateways in from gatewaySet in GenerateNewSet(initialModel.Gateways, GatewayModel.GenerateSet(), Gateway.GenerateUpdate, changeParameters) + from updatedGatewayApis in ServiceModel.UpdateGateways(gatewaySet, apis) + select updatedGatewayApis + from subscriptions in from subscriptionSet in GenerateNewSet(initialModel.Subscriptions, SubscriptionModel.GenerateSet(), Subscription.GenerateUpdate, changeParameters) + from updatedSubscriptions in ServiceModel.UpdateSubscriptions(subscriptionSet, products, apis) + select updatedSubscriptions + select initialModel with + { + NamedValues = namedValues, + Tags = tags, + Gateways = gateways, + VersionSets = versionSets, + Backends = backends, + Loggers = loggers, + Diagnostics = diagnostics, + PolicyFragments = policyFragments, + ServicePolicies = servicePolicies, + Products = products, + Groups = groups, + Apis = apis, + Subscriptions = subscriptions + }; + + public static Gen> GenerateNewSet(FrozenSet original, Gen> newGen, Func> updateGen) => + GenerateNewSet(original, newGen, updateGen, ChangeParameters.All); + + private static Gen> GenerateNewSet(FrozenSet original, Gen> newGen, Func> updateGen, ChangeParameters changeParameters) + { + var generator = from originalItems in Gen.Const(original) + from itemsRemoved in changeParameters.Remove ? RemoveItems(originalItems) : Gen.Const(originalItems) + from itemsAdded in changeParameters.Add ? AddItems(itemsRemoved, newGen) : Gen.Const(itemsRemoved) + from itemsModified in changeParameters.Modify ? ModifyItems(itemsAdded, updateGen) : Gen.Const(itemsAdded) + select itemsModified; + + return changeParameters.MaxSize.Map(maxSize => generator.SelectMany(set => set.Count <= maxSize + ? generator + : from smallerSet in Gen.Shuffle(set.ToArray(), maxSize) + + select smallerSet.ToFrozenSet(set.Comparer))) + .IfNone(generator); + } + + private static Gen> RemoveItems(FrozenSet set) => + from itemsToRemove in Generator.SubFrozenSetOf(set) + select set.Except(itemsToRemove, set.Comparer).ToFrozenSet(set.Comparer); + + private static Gen> AddItems(FrozenSet set, Gen> gen) => + from itemsToAdd in gen + select set.Append(itemsToAdd).ToFrozenSet(set.Comparer); + + private static Gen> ModifyItems(FrozenSet set, Func> updateGen) => + from itemsToModify in Generator.SubFrozenSetOf(set) + from modifiedItems in itemsToModify.Select(updateGen).SequenceToImmutableArray() + select set.Except(itemsToModify).Append(modifiedItems).ToFrozenSet(set.Comparer); + + private sealed record ChangeParameters + { + public required bool Add { get; init; } + public required bool Modify { get; init; } + public required bool Remove { get; init; } + + public static ChangeParameters None { get; } = new() + { + Add = false, + Modify = false, + Remove = false + }; + + public static ChangeParameters All { get; } = new() + { + Add = true, + Modify = true, + Remove = true + }; + + public Option MaxSize { get; init; } = Option.None; + } + } +} + +file sealed class CleanUpTestsHandler(ILogger logger, + ActivitySource activitySource, + ListApimServiceNames listApimServiceNames, + DeleteApimService deleteApimService) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(CleanUpTests)); + + logger.LogInformation("Cleaning up test..."); + + await listApimServiceNames(cancellationToken) + .Where(name => name.ToString().StartsWith(Common.TestServiceNamePrefix, StringComparison.OrdinalIgnoreCase)) + .IterParallel(deleteApimService.Invoke, cancellationToken); + } +} + +internal static class TestServices +{ + public static void ConfigureRunTests(IServiceCollection services) + { + ConfigureTestExtractor(services); + ConfigureTestExtractThenPublish(services); + ConfigureTestPublisher(services); + ConfigureCleanUpTests(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureTestExtractor(IServiceCollection services) + { + ManagementServices.ConfigureCreateApimService(services); + ManagementServices.ConfigureEmptyApimService(services); + ManagementServices.ConfigurePutServiceModel(services); + ExtractorServices.ConfigureRunExtractor(services); + ExtractorServices.ConfigureValidateExtractorArtifacts(services); + ManagementServices.ConfigureDeleteApimService(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureTestExtractThenPublish(IServiceCollection services) + { + ManagementServices.ConfigureCreateApimService(services); + ManagementServices.ConfigurePutServiceModel(services); + ExtractorServices.ConfigureRunExtractor(services); + ManagementServices.ConfigureDeleteApimService(services); + PublisherServices.ConfigureRunPublisher(services); + PublisherServices.ConfigureValidatePublishedArtifacts(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureTestPublisher(IServiceCollection services) + { + ManagementServices.ConfigureWriteServiceModelArtifacts(services); + ManagementServices.ConfigureCreateApimService(services); + ManagementServices.ConfigureEmptyApimService(services); + PublisherServices.ConfigureRunPublisher(services); + PublisherServices.ConfigureValidatePublishedArtifacts(services); + ManagementServices.ConfigureWriteServiceModelCommits(services); + ManagementServices.ConfigureDeleteApimService(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureCleanUpTests(IServiceCollection services) + { + ManagementServices.ConfigureListApimServiceNames(services); + ManagementServices.ConfigureDeleteApimService(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +file sealed class Common +{ +#pragma warning disable CA1802 // Use literals where appropriate +#pragma warning disable CA1805 // Do not initialize unnecessarily + public static string TestServiceNamePrefix { get; } = "apiopsinttest-"; + public static readonly bool useExistingInstance = false; +#pragma warning restore CA1805 // Do not initialize unnecessarily +#pragma warning restore CA1802 // Use literals where appropriate + + public static void DeleteServiceDirectory(ManagementServiceDirectory serviceDirectory) => serviceDirectory.ToDirectoryInfo().ForceDelete(); + + // We want the name to change between tests, even if the seed is the same. + // This avoids soft-delete issues with APIM + public static Gen GenerateManagementServiceName(string prefix) => + from lorem in Generator.Lorem + let characters = lorem.Paragraphs(3) + .Where(char.IsLetterOrDigit) + .Select(char.ToLowerInvariant) + .ToArray() + from suffixCharacters in Gen.Shuffle(characters, 8) + let name = $"{prefix}{new string(suffixCharacters)}" + select ManagementServiceName.From(name); + + public static Gen GenerateManagementServiceDirectory() => + from lorem in Generator.Lorem + let characters = lorem.Paragraphs(3) + .Where(char.IsLetterOrDigit) + .Select(char.ToLowerInvariant) + .ToArray() + from suffixCharacters in Gen.Shuffle(characters, 8) + let name = $"apiops-{new string(suffixCharacters)}" + let path = Path.Combine(Path.GetTempPath(), name) + let directoryInfo = new DirectoryInfo(path) + select ManagementServiceDirectory.From(directoryInfo); +} \ No newline at end of file diff --git a/tools/code/integration.tests/Tests.cs b/tools/code/integration.tests/Tests.cs deleted file mode 100644 index 018a282c..00000000 --- a/tools/code/integration.tests/Tests.cs +++ /dev/null @@ -1,274 +0,0 @@ -using common; -using common.tests; -using CsCheck; -using DotNext.Collections.Generic; -using FluentAssertions; -using LanguageExt; -using NUnit.Framework; -using publisher; -using System; -using System.Collections.Frozen; -using System.Collections.Immutable; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace integration.tests; - - -[TestFixture] -public sealed class Tests -{ - private const string serviceNamePrefix = "apiopsinttest-"; - -#pragma warning disable CA1802 // Use literals where appropriate -#pragma warning disable CA1805 // Do not initialize unnecessarily - private static readonly bool useExistingInstance = false; -#pragma warning restore CA1805 // Do not initialize unnecessarily -#pragma warning restore CA1802 // Use literals where appropriate - - [Test] - public async Task Runs_as_expected() - { - AssertionOptions.FormattingOptions.MaxLines = 10000; - AssertionOptions.AssertEquivalencyUsing(options => options.ComparingRecordsByValue()); - - var cancellationToken = CancellationToken.None; - await OneTimeSetup(cancellationToken); - - await WriteProgress("Generating fixture..."); - var generator = from fixture in Fixture.Generate(serviceNamePrefix) - // Use configuration service name for special SKUs - select useExistingInstance - ? fixture with - { - FirstServiceName = Configuration.FirstServiceName, - SecondServiceName = Configuration.SecondServiceName - } - : fixture with - { - ServiceModel = fixture.ServiceModel with - { - Gateways = FrozenSet.Empty - }, - PublishAllChangesModel = fixture.PublishAllChangesModel with - { - Gateways = FrozenSet.Empty - }, - CommitModels = fixture.CommitModels.Select(model => model with - { - Gateways = FrozenSet.Empty - }).ToImmutableArray() - }; - - await generator.SampleAsync(async fixture => - { - // 1. Set up the management service - await CreateManagementServices(fixture, cancellationToken); - DeleteServiceDirectory(fixture); - - // 2. Run the extractor and validate its artifacts - await PutServiceModel(fixture, cancellationToken); - await RunExtractor(fixture, cancellationToken); - await ValidateExtractorArtifacts(fixture, cancellationToken); - - // 3. Publish the extracted changes to the empty second service - await PublishExtractedArtifactsToSecondService(fixture, cancellationToken); - await ValidatePublishedExtractedArtifacts(fixture, cancellationToken); - await CleanUpExtractorResources(fixture, cancellationToken); - - // 3. Make changes to the extracted artifacts, publish the changes, then validate - await WriteFirstChange(fixture, cancellationToken); - await PublishFirstChange(fixture, cancellationToken); - await ValidatePublishedFirstChange(fixture, cancellationToken); - - // 4. Write commits, publish changes in a specific commit, then validate - var commits = await WriteCommitArtifacts(fixture, cancellationToken); - await TestCommits(fixture, commits, cancellationToken); - - await CleanUp(fixture, cancellationToken); - }, iter: 1, seed: "0000KOIPe036", threads: useExistingInstance ? 1 : -1); - } - - private static async ValueTask WriteProgress(string message) => - await TestContext.Progress.WriteLineAsync($"{DateTime.Now:O}: {message}"); - - private static async ValueTask OneTimeSetup(CancellationToken cancellationToken) - { - var serviceProviderUri = Configuration.ManagementServiceProviderUri; - var pipeline = Configuration.HttpPipeline; - - await WriteProgress("Deleting management services..."); - await ServiceModule.DeleteManagementServices(serviceProviderUri, serviceNamePrefix, pipeline, cancellationToken); - } - - private static async ValueTask CreateManagementServices(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Creating management services..."); - - var firstServiceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - var secondServiceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); - - if (useExistingInstance) - { - await Task.WhenAll(ServiceModule.DeleteAll(firstServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask(), - ServiceModule.DeleteAll(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask()); - } - else - { - await Task.WhenAll(ServiceModule.CreateManagementService(firstServiceUri, Configuration.Location, Configuration.HttpPipeline, cancellationToken).AsTask(), - ServiceModule.CreateManagementService(secondServiceUri, Configuration.Location, Configuration.HttpPipeline, cancellationToken).AsTask()); - } - } - - private static void DeleteServiceDirectory(Fixture fixture) - { - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - } - - private static async ValueTask PutServiceModel(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Putting service model..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - await ServiceModule.Put(fixture.ServiceModel, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask RunExtractor(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Running extractor..."); - - var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Extractor.Run(fixture.ExtractorOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, cancellationToken); - } - - private static async ValueTask ValidateExtractorArtifacts(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Validating extractor artifacts..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - await ServiceModule.ValidateExtractedArtifacts(fixture.ExtractorOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask PublishExtractedArtifactsToSecondService(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Extracting all artifacts from first instance..."); - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - - var bearerToken = await Configuration.GetBearerToken(cancellationToken); - var extractorOptions = fixture.ExtractorOptions with - { - ApiNamesToExport = Option>.None, - ProductNamesToExport = Option>.None, - GroupNamesToExport = Option>.None, - SubscriptionNamesToExport = Option>.None, - BackendNamesToExport = Option>.None, - LoggerNamesToExport = Option>.None, - DiagnosticNamesToExport = Option>.None, - PolicyFragmentNamesToExport = Option>.None, - GatewayNamesToExport = Option>.None, - TagNamesToExport = Option>.None, - VersionSetNamesToExport = Option>.None, - NamedValueNamesToExport = Option>.None, - }; - await Extractor.Run(extractorOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, cancellationToken); - - await WriteProgress("Publishing extracted artifacts to second instance..."); - await Publisher.Run(fixture.PublisherOptions, fixture.SecondServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId: Option.None, cancellationToken); - } - - private static async ValueTask ValidatePublishedExtractedArtifacts(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Validating published extracted artifacts..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); - await ServiceModule.ValidatePublisherChanges(fixture.PublisherOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask CleanUpExtractorResources(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Cleaning up extractor resources..."); - - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - - var firstServiceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - var secondServiceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); - await Task.WhenAll(ServiceModule.DeleteAll(firstServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask(), - useExistingInstance - ? ServiceModule.DeleteAll(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask() - : ServiceModule.DeleteManagementService(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask()); - } - - private static async ValueTask WriteFirstChange(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Writing first change..."); - - var firstChange = fixture.PublishAllChangesModel; - await ServiceModule.WriteArtifacts(firstChange, fixture.ServiceDirectory, useExistingInstance, cancellationToken); - } - - private static async ValueTask PublishFirstChange(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Publishing first change..."); - - var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Publisher.Run(fixture.PublisherOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId: Option.None, cancellationToken); - } - - private static async ValueTask ValidatePublishedFirstChange(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Validating published first change..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - await ServiceModule.ValidatePublisherChanges(fixture.PublisherOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask> WriteCommitArtifacts(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Writing commit artifacts..."); - - return await ServiceModule.WriteCommitArtifacts(fixture.CommitModels, fixture.ServiceDirectory, useExistingInstance, cancellationToken); - } - - private static async ValueTask TestCommits(Fixture fixture, ImmutableArray commits, CancellationToken cancellationToken) => - await commits.Take(1) - .ForEachAsync(async (commit, cancellationToken) => - { - await PublishCommit(fixture, commit, cancellationToken); - await ValidatePublishedCommit(fixture, commit, cancellationToken); - }, cancellationToken); - - private static async ValueTask PublishCommit(Fixture fixture, CommitId commitId, CancellationToken cancellationToken) - { - await WriteProgress($"Publishing commit {commitId}..."); - - var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Publisher.Run(fixture.PublisherOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId, cancellationToken); - } - - private static async ValueTask ValidatePublishedCommit(Fixture fixture, CommitId commitId, CancellationToken cancellationToken) - { - await WriteProgress($"Validating published commit {commitId}..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - await ServiceModule.ValidatePublisherCommitChanges(fixture.PublisherOptions, commitId, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask CleanUp(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Cleaning up..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - - if (useExistingInstance) - { - await ServiceModule.DeleteAll(serviceUri, Configuration.HttpPipeline, cancellationToken); - } - else - { - await ServiceModule.DeleteManagementService(serviceUri, Configuration.HttpPipeline, cancellationToken); - } - - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - } -} \ No newline at end of file diff --git a/tools/code/integration.tests/VersionSet.cs b/tools/code/integration.tests/VersionSet.cs index 2fc298b9..e61e70c2 100644 --- a/tools/code/integration.tests/VersionSet.cs +++ b/tools/code/integration.tests/VersionSet.cs @@ -5,59 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class VersionSet +internal delegate ValueTask DeleteAllVersionSets(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutVersionSetModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedVersionSets(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimVersionSets(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileVersionSets(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteVersionSetModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedVersionSets(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllVersionSetsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(VersionSetModel original) => - from displayName in VersionSetModel.GenerateDisplayName() - from scheme in VersioningScheme.Generate() - from description in VersionSetModel.GenerateDescription().OptionOf() - select original with - { - DisplayName = displayName, - Scheme = scheme, - Description = description - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllVersionSets)); - public static Gen GenerateOverride(VersionSetDto original) => - from displayName in VersionSetModel.GenerateDisplayName() - from header in GenerateHeaderOverride(original) - from query in GenerateQueryOverride(original) - from description in VersionSetModel.GenerateDescription().OptionOf() - select new VersionSetDto - { - Properties = new VersionSetDto.VersionSetContract - { - DisplayName = displayName, - Description = description.ValueUnsafe(), - VersionHeaderName = header, - VersionQueryName = query - } - }; + logger.LogInformation("Deleting all version sets in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await VersionSetsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} - private static Gen GenerateHeaderOverride(VersionSetDto original) => - Gen.OneOf(Gen.Const(original.Properties.VersionHeaderName), - string.IsNullOrWhiteSpace(original.Properties.VersionHeaderName) - ? Gen.Const(() => null as string)! - : VersioningScheme.Header.GenerateHeaderName()); +file sealed class PutVersionSetModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutVersionSetModels)); - private static Gen GenerateQueryOverride(VersionSetDto original) => - Gen.OneOf(Gen.Const(original.Properties.VersionQueryName), - string.IsNullOrWhiteSpace(original.Properties.VersionQueryName) - ? Gen.Const(() => null as string)! - : VersioningScheme.Query.GenerateQueryName()); + logger.LogInformation("Putting version set models in {ServiceName}...", serviceName); + await models.IterParallel(async model => + { + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(VersionSetModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = VersionSetUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static VersionSetDto GetDto(VersionSetModel model) => new() @@ -77,29 +85,113 @@ private static VersionSetDto GetDto(VersionSetModel model) => } } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedVersionSetsHandler(ILogger logger, GetApimVersionSets getApimResources, GetFileVersionSets getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedVersionSets)); + + logger.LogInformation("Validating extracted version sets in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(VersionSetDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty, + VersionHeaderName = dto.Properties.VersionHeaderName ?? string.Empty, + VersionQueryName = dto.Properties.VersionQueryName ?? string.Empty, + VersioningScheme = dto.Properties.VersioningScheme ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(VersionSetModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimVersionSetsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = VersionSetUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimVersionSets)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting version sets from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = VersionSetsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await VersionSetsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileVersionSetsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileVersionSets)); + + logger.LogInformation("Getting version sets from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => VersionSetInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, VersionSetInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileVersionSets)); + + logger.LogInformation("Getting version sets from {ServiceDirectory}...", serviceDirectory); + + return await VersionSetModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteVersionSetModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteVersionSetModels)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Writing version set models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(VersionSetModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -109,33 +201,44 @@ private static async ValueTask WriteInformationFile(VersionSetModel model, Manag await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static VersionSetDto GetDto(VersionSetModel model) => + new() + { + Properties = new VersionSetDto.VersionSetContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe(), + VersionHeaderName = model.Scheme is VersioningScheme.Header header ? header.HeaderName : null, + VersionQueryName = model.Scheme is VersioningScheme.Query query ? query.QueryName : null, + VersioningScheme = model.Scheme switch + { + VersioningScheme.Header => "Header", + VersioningScheme.Query => "Query", + VersioningScheme.Segment => "Segment", + _ => null + } + } + }; +} + +file sealed class ValidatePublishedVersionSetsHandler(ILogger logger, GetFileVersionSets getFileResources, GetApimVersionSets getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedVersionSets)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published version sets in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = VersionSetsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await VersionSetModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(VersionSetDto dto) => new { @@ -145,49 +248,125 @@ private static string NormalizeDto(VersionSetDto dto) => VersionQueryName = dto.Properties.VersionQueryName ?? string.Empty, VersioningScheme = dto.Properties.VersioningScheme ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class VersionSetServices +{ + public static void ConfigureDeleteAllVersionSets(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutVersionSetModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedVersionSets(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimVersionSets(services); + ConfigureGetFileVersionSets(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => VersionSetInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimVersionSets(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, VersionSetInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileVersionSets(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteVersionSetModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedVersionSets(IServiceCollection services) + { + ConfigureGetFileVersionSets(services); + ConfigureGetApimVersionSets(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class VersionSet +{ + public static Gen GenerateUpdate(VersionSetModel original) => + from displayName in VersionSetModel.GenerateDisplayName() + from scheme in VersioningScheme.Generate() + from description in VersionSetModel.GenerateDescription().OptionOf() + select original with { - using (contents) + DisplayName = displayName, + Scheme = scheme, + Description = description + }; + + public static Gen GenerateOverride(VersionSetDto original) => + from displayName in VersionSetModel.GenerateDisplayName() + from header in GenerateHeaderOverride(original) + from query in GenerateQueryOverride(original) + from description in VersionSetModel.GenerateDescription().OptionOf() + select new VersionSetDto + { + Properties = new VersionSetDto.VersionSetContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + DisplayName = displayName, + Description = description.ValueUnsafe(), + VersionHeaderName = header, + VersionQueryName = query } - }); - } + }; + + private static Gen GenerateHeaderOverride(VersionSetDto original) => + Gen.OneOf(Gen.Const(original.Properties.VersionHeaderName), + string.IsNullOrWhiteSpace(original.Properties.VersionHeaderName) + ? Gen.Const(() => null as string)! + : VersioningScheme.Header.GenerateHeaderName()); + + private static Gen GenerateQueryOverride(VersionSetDto original) => + Gen.OneOf(Gen.Const(original.Properties.VersionQueryName), + string.IsNullOrWhiteSpace(original.Properties.VersionQueryName) + ? Gen.Const(() => null as string)! + : VersioningScheme.Query.GenerateQueryName()); + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static VersionSetDto GetDto(VersionSetModel model) => + new() + { + Properties = new VersionSetDto.VersionSetContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe(), + VersionHeaderName = model.Scheme is VersioningScheme.Header header ? header.HeaderName : null, + VersionQueryName = model.Scheme is VersioningScheme.Query query ? query.QueryName : null, + VersioningScheme = model.Scheme switch + { + VersioningScheme.Header => "Header", + VersioningScheme.Query => "Query", + VersioningScheme.Segment => "Segment", + _ => null + } + } + }; } diff --git a/tools/code/integration.tests/integration.tests.csproj b/tools/code/integration.tests/integration.tests.csproj index 5da383f3..6fce3f61 100644 --- a/tools/code/integration.tests/integration.tests.csproj +++ b/tools/code/integration.tests/integration.tests.csproj @@ -1,37 +1,25 @@  - - net8.0 - enable - true - CA2007,CA1707,CA1716,CA1724,CA1848 - latest-all - false - true - 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 - + + Exe + net8.0 + false + true + CA2007,CA1848,CA1812 + 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 + 8-all + enable + + + + + + - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - + + + + + diff --git a/tools/code/publisher/Common.cs b/tools/code/publisher/Common.cs index 1b440f94..2b5d2b9b 100644 --- a/tools/code/publisher/Common.cs +++ b/tools/code/publisher/Common.cs @@ -13,6 +13,7 @@ using Microsoft.IdentityModel.JsonWebTokens; using System; using System.Collections.Frozen; +using System.Diagnostics; using System.IO; using System.Reflection; using System.Threading; @@ -150,6 +151,7 @@ internal static class CommonServices { public static void Configure(IServiceCollection services) { + services.AddSingleton(GetActivitySource); services.TryAddSingleton(GetAzureEnvironment); services.TryAddSingleton(GetTokenCredential); services.TryAddSingleton(GetConfigurationJson); @@ -165,8 +167,12 @@ public static void Configure(IServiceCollection services) ConfigureGetArtifactsInPreviousCommit(services); services.ConfigureApimHttpClient(); + OpenTelemetryServices.Configure(services); } + private static ActivitySource GetActivitySource(IServiceProvider provider) => + new("ApiOps.Publisher"); + private static AzureEnvironment GetAzureEnvironment(IServiceProvider provider) { var configuration = provider.GetRequiredService(); diff --git a/tools/code/publisher/Git.cs b/tools/code/publisher/Git.cs index 3bfca618..270475ca 100644 --- a/tools/code/publisher/Git.cs +++ b/tools/code/publisher/Git.cs @@ -36,7 +36,8 @@ public static FrozenSet GetChangedFilesInCommit(DirectoryInfo reposito private static TreeChanges GetChanges(DirectoryInfo repositoryDirectory, CommitId commitId) { - using var repository = new Repository(repositoryDirectory.FullName); + var gitRootDirectory = repositoryDirectory.Parent!.FullName; + using var repository = new Repository(gitRootDirectory); var commit = GetCommit(repository, commitId); @@ -53,7 +54,8 @@ private static Commit GetCommit(Repository repository, CommitId commitId) => public static Option TryGetPreviousCommitId(DirectoryInfo repositoryDirectory, CommitId commitId) { - using var repository = new Repository(repositoryDirectory.FullName); + var gitRootDirectory = repositoryDirectory.Parent!.FullName; + using var repository = new Repository(gitRootDirectory); var commit = GetCommit(repository, commitId); @@ -66,7 +68,8 @@ public static Option TryGetPreviousCommitId(DirectoryInfo repositoryDi public static Option TryGetFileContentsInCommit(DirectoryInfo repositoryDirectory, FileInfo file, CommitId commitId) { - using var repository = new Repository(repositoryDirectory.FullName); + var gitRootDirectory = repositoryDirectory.Parent!.FullName; + using var repository = new Repository(gitRootDirectory); var relativePath = Path.GetRelativePath(repositoryDirectory.FullName, file.FullName); var relativePathString = Path.DirectorySeparatorChar == '\\' ? relativePath.Replace('\\', '/') @@ -81,7 +84,8 @@ public static Option TryGetFileContentsInCommit(DirectoryInfo repository public static FrozenSet GetExistingFilesInCommit(DirectoryInfo repositoryDirectory, CommitId commitId) { - using var repository = new Repository(repositoryDirectory.FullName); + var gitRootDirectory = repositoryDirectory.Parent!.FullName; + using var repository = new Repository(gitRootDirectory); var commit = GetCommit(repository, commitId); diff --git a/tools/code/publisher/publisher.csproj b/tools/code/publisher/publisher.csproj index bd29b26c..e58f3faf 100644 --- a/tools/code/publisher/publisher.csproj +++ b/tools/code/publisher/publisher.csproj @@ -2,8 +2,9 @@ net8.0 + false true - latest-all + 8-all CA1708,CA1724,CA1812,CA1848,CA2007,CA1034,CA1062 Exe enable @@ -13,7 +14,7 @@ - +