From 055e64172c06c06a9885bebe6aa11060521b7294 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 14:41:55 -0400 Subject: [PATCH 01/29] Update release to zip file --- .github/workflows/create_github_release.yaml | 419 ++++++++++--------- 1 file changed, 211 insertions(+), 208 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 0eb84826..8989a618 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -32,178 +32,23 @@ jobs: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Generating extractor..." - $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "extractor" "extractor.csproj" - $outputFolderPath = "${{ runner.temp }}" + Write-Information "Creating output directory..." + $outputFolderPath = Join-Path "${{ runner.temp }}" "extractor-output" + New-Item -Path "$outputFolderPath" -ItemType "Directory" + Write-Information "Publishing application..." + $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "extractor" "extractor.csproj" & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" if ($LASTEXITCODE -ne 0) { throw "Generating extractor failed."} + ## DELETE + Get-ChildItem -Path $outputFolderPath -Recurse - $exeFileExt = "${{ matrix.dotnet-runtime }}".Contains("win") ? ".exe" : "" - $exeFolderPath = Join-Path "$outputFolderPath" "extractor$exeFileExt" - $exeFileNameFinal = "${{ format('extractor.{0}', matrix.dotnet-runtime) }}$exeFileExt" - Rename-Item -Path "$exeFolderPath" -NewName $exeFileNameFinal - echo "EXTRACTOR_FILENAME=$exeFileNameFinal" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - - Write-Information "Execution complete." - shell: pwsh - env: - EXTRACTOR_FILENAME: - - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - path: ${{ format('{0}/{1}', runner.temp, env.EXTRACTOR_FILENAME) }} - - generate_publisher_artifacts: - name: Generate publisher artifacts - strategy: - matrix: - dotnet-runtime: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, win-x64, osx-arm64, osx-x64] - # Dynamically set the runner OS based on the .NET runtime - runs-on: ${{ fromJSON('{"linux-x64":"ubuntu-latest", "linux-arm64":"ubuntu-latest", "linux-musl-x64":"ubuntu-latest", "linux-musl-arm64":"ubuntu:latest", "win-x64":"windows-latest", "osx-arm64":"macos-latest", "osx-x64":"macos-latest"}')[matrix.dotnet-runtime] }} - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Setup .NET - uses: actions/setup-dotnet@v3 - with: - dotnet-version: 8.x - - - name: Generate executable - run: | - Set-StrictMode -Version Latest - $ErrorActionPreference = "Stop" - $VerbosePreference = "Continue" - $InformationPreference = "Continue" - - Write-Information "Generating publisher..." - $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "publisher" "publisher.csproj" - $outputFolderPath = "${{ runner.temp }}" - - & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" - if ($LASTEXITCODE -ne 0) { throw "Generating publisher failed."} - - $exeFileExt = "${{ matrix.dotnet-runtime }}".Contains("win") ? ".exe" : "" - $exeFolderPath = Join-Path "$outputFolderPath" "publisher$exeFileExt" - $exeFileNameFinal = "${{ format('publisher.{0}', matrix.dotnet-runtime) }}$exeFileExt" - Rename-Item -Path "$exeFolderPath" -NewName $exeFileNameFinal - echo "PUBLISHER_FILENAME=$exeFileNameFinal" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - - Write-Information "Execution complete." - shell: pwsh - env: - PUBLISHER_FILENAME: - - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - path: ${{ format('{0}/{1}', runner.temp, env.PUBLISHER_FILENAME) }} - - generate_github_pipeline_artifacts: - name: Generate GitHub artifacts - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Update versions in YAML - run: | - Set-StrictMode -Version Latest - $ErrorActionPreference = "Stop" - $VerbosePreference = "Continue" - $InformationPreference = "Continue" - - $runExtractorPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "run-extractor.yaml" - (Get-Content -Path "$runExtractorPath") | - ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | - Set-Content -Path "$runExtractorPath" - - $runPublisherWithEnvPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "run-publisher-with-env.yaml" - (Get-Content -Path "$runPublisherWithEnvPath") | - ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | - Set-Content -Path "$runPublisherWithEnvPath" - - Write-Information "Execution complete." - shell: pwsh - - - name: Setup artifact contents - run: | - Set-StrictMode -Version Latest - $ErrorActionPreference = "Stop" - $VerbosePreference = "Continue" - $InformationPreference = "Continue" - - $artifactsFolderPath = Join-Path "${{ runner.temp }}" "github_artifacts" - - Write-Information "Copying GitHub workflows..." - $sourceFolderPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "*" - $destinationFolderPath = Join-Path "$artifactsFolderPath" ".github" "workflows" - New-Item -Path "$destinationFolderPath" -ItemType "Directory" - Copy-Item -Path "$sourceFolderPath" -Destination "$destinationFolderPath" -Recurse - - Write-Information "Zipping artifacts..." - $destinationFilePath = Join-Path "${{ runner.temp }}" "Github.zip" - - # We use System.IO.Compression.ZipFile instead of Compress-Archive because of this issue: https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/66 - [System.IO.Compression.ZipFile]::CreateFromDirectory("$artifactsFolderPath", "$destinationFilePath") - - Write-Information "Execution complete." - shell: pwsh - - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - path: ${{ runner.temp }}/Github.zip - - generate_ado_pipeline_artifacts: - name: Generate Azure DevOps artifacts - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Update versions in YAML - run: | - Set-StrictMode -Version Latest - $ErrorActionPreference = "Stop" - $VerbosePreference = "Continue" - $InformationPreference = "Continue" - - $runExtractorPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "run-extractor.yaml" - (Get-Content -Path "$runExtractorPath") | - ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | - Set-Content -Path "$runExtractorPath" - - $runPublisherWithEnvPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "run-publisher-with-env.yaml" - (Get-Content -Path "$runPublisherWithEnvPath") | - ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | - Set-Content -Path "$runPublisherWithEnvPath" - - Write-Information "Execution complete." - shell: pwsh - - - name: Setup artifact contents - run: | - Set-StrictMode -Version Latest - $ErrorActionPreference = "Stop" - $VerbosePreference = "Continue" - $InformationPreference = "Continue" - - $artifactsFolderPath = Join-Path "${{ runner.temp }}" "azdo_artifacts" - - Write-Information "Copying Azure DevOps pipelines..." - $sourceFolderPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "*" - $destinationFolderPath = Join-Path "$artifactsFolderPath" "tools" "pipelines" - New-Item -Path "$destinationFolderPath" -ItemType "Directory" - Copy-Item -Path "$sourceFolderPath" -Destination "$destinationFolderPath" -Recurse - - Write-Information "Zipping artifacts..." - $destinationFilePath = Join-Path "${{ runner.temp }}" "Azure_DevOps.zip" - - # We use System.IO.Compression.ZipFile instead of Compress-Archive because of this issue: https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/66 - [System.IO.Compression.ZipFile]::CreateFromDirectory("$artifactsFolderPath", "$destinationFilePath") + Write-Information "Zipping application..." + $sourceFolderPath = Join-Path "$outputFolderPath" "*" + $extractorFileNameWithoutExtension = "${{ format('extractor.{0}', matrix.dotnet-runtime) }}" + $destinationFilePath = Join-Path "$outputFolderPath" "$($extractorFileNameWithoutExtension).zip" + Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath + "{ZIP_FILE_PATH}=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append Write-Information "Execution complete." shell: pwsh @@ -211,43 +56,201 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v3 with: - path: ${{ runner.temp }}/Azure_DevOps.zip - - generate_release: - name: Generate release - needs: - [ - generate_extractor_artifacts, - generate_publisher_artifacts, - generate_github_pipeline_artifacts, - generate_ado_pipeline_artifacts, - ] - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Download artifacts - uses: actions/download-artifact@v3 - - name: Create release - uses: softprops/action-gh-release@v1 - with: - files: | - ${{github.workspace}}/artifact/extractor.win-x64.exe - ${{github.workspace}}/artifact/extractor.linux-x64 - ${{github.workspace}}/artifact/extractor.linux-arm64 - ${{github.workspace}}/artifact/extractor.linux-musl-x64 - ${{github.workspace}}/artifact/extractor.linux-musl-arm64 - ${{github.workspace}}/artifact/extractor.osx-arm64 - ${{github.workspace}}/artifact/extractor.osx-x64 - ${{github.workspace}}/artifact/publisher.win-x64.exe - ${{github.workspace}}/artifact/publisher.linux-x64 - ${{github.workspace}}/artifact/publisher.linux-arm64 - ${{github.workspace}}/artifact/publisher.linux-musl-x64 - ${{github.workspace}}/artifact/publisher.linux-musl-arm64 - ${{github.workspace}}/artifact/publisher.osx-arm64 - ${{github.workspace}}/artifact/publisher.osx-x64 - ${{github.workspace}}/artifact/Github.zip - ${{github.workspace}}/artifact/Azure_DevOps.zip - name: APIOps Toolkit for Azure APIM ${{ github.event.inputs.Release_Version }} - tag_name: ${{ github.event.inputs.Release_Version }} - generate_release_notes: true + path: ${{ format('{0}/{1}', runner.temp, env.ZIP_FILE_PATH) }} + + # generate_publisher_artifacts: + # name: Generate publisher artifacts + # strategy: + # matrix: + # dotnet-runtime: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, win-x64, osx-arm64, osx-x64] + # # Dynamically set the runner OS based on the .NET runtime + # runs-on: ${{ fromJSON('{"linux-x64":"ubuntu-latest", "linux-arm64":"ubuntu-latest", "linux-musl-x64":"ubuntu-latest", "linux-musl-arm64":"ubuntu:latest", "win-x64":"windows-latest", "osx-arm64":"macos-latest", "osx-x64":"macos-latest"}')[matrix.dotnet-runtime] }} + # steps: + # - name: Checkout repository + # uses: actions/checkout@v3 + + # - name: Setup .NET + # uses: actions/setup-dotnet@v3 + # with: + # dotnet-version: 8.x + + # - name: Generate executable + # run: | + # Set-StrictMode -Version Latest + # $ErrorActionPreference = "Stop" + # $VerbosePreference = "Continue" + # $InformationPreference = "Continue" + + # Write-Information "Generating publisher..." + # $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "publisher" "publisher.csproj" + # $outputFolderPath = "${{ runner.temp }}" + + # & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" + # if ($LASTEXITCODE -ne 0) { throw "Generating publisher failed."} + + # $exeFileExt = "${{ matrix.dotnet-runtime }}".Contains("win") ? ".exe" : "" + # $exeFolderPath = Join-Path "$outputFolderPath" "publisher$exeFileExt" + # $exeFileNameFinal = "${{ format('publisher.{0}', matrix.dotnet-runtime) }}$exeFileExt" + # Rename-Item -Path "$exeFolderPath" -NewName $exeFileNameFinal + # echo "PUBLISHER_FILENAME=$exeFileNameFinal" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + + # Write-Information "Execution complete." + # shell: pwsh + # env: + # PUBLISHER_FILENAME: + + # - name: Upload artifact + # uses: actions/upload-artifact@v3 + # with: + # path: ${{ format('{0}/{1}', runner.temp, env.PUBLISHER_FILENAME) }} + + # generate_github_pipeline_artifacts: + # name: Generate GitHub artifacts + # runs-on: ubuntu-latest + # steps: + # - name: Checkout repository + # uses: actions/checkout@v3 + + # - name: Update versions in YAML + # run: | + # Set-StrictMode -Version Latest + # $ErrorActionPreference = "Stop" + # $VerbosePreference = "Continue" + # $InformationPreference = "Continue" + + # $runExtractorPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "run-extractor.yaml" + # (Get-Content -Path "$runExtractorPath") | + # ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | + # Set-Content -Path "$runExtractorPath" + + # $runPublisherWithEnvPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "run-publisher-with-env.yaml" + # (Get-Content -Path "$runPublisherWithEnvPath") | + # ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | + # Set-Content -Path "$runPublisherWithEnvPath" + + # Write-Information "Execution complete." + # shell: pwsh + + # - name: Setup artifact contents + # run: | + # Set-StrictMode -Version Latest + # $ErrorActionPreference = "Stop" + # $VerbosePreference = "Continue" + # $InformationPreference = "Continue" + + # $artifactsFolderPath = Join-Path "${{ runner.temp }}" "github_artifacts" + + # Write-Information "Copying GitHub workflows..." + # $sourceFolderPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "*" + # $destinationFolderPath = Join-Path "$artifactsFolderPath" ".github" "workflows" + # New-Item -Path "$destinationFolderPath" -ItemType "Directory" + # Copy-Item -Path "$sourceFolderPath" -Destination "$destinationFolderPath" -Recurse + + # Write-Information "Zipping artifacts..." + # $destinationFilePath = Join-Path "${{ runner.temp }}" "Github.zip" + + # # We use System.IO.Compression.ZipFile instead of Compress-Archive because of this issue: https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/66 + # [System.IO.Compression.ZipFile]::CreateFromDirectory("$artifactsFolderPath", "$destinationFilePath") + + # Write-Information "Execution complete." + # shell: pwsh + + # - name: Upload artifact + # uses: actions/upload-artifact@v3 + # with: + # path: ${{ runner.temp }}/Github.zip + + # generate_ado_pipeline_artifacts: + # name: Generate Azure DevOps artifacts + # runs-on: ubuntu-latest + # steps: + # - name: Checkout repository + # uses: actions/checkout@v3 + + # - name: Update versions in YAML + # run: | + # Set-StrictMode -Version Latest + # $ErrorActionPreference = "Stop" + # $VerbosePreference = "Continue" + # $InformationPreference = "Continue" + + # $runExtractorPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "run-extractor.yaml" + # (Get-Content -Path "$runExtractorPath") | + # ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | + # Set-Content -Path "$runExtractorPath" + + # $runPublisherWithEnvPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "run-publisher-with-env.yaml" + # (Get-Content -Path "$runPublisherWithEnvPath") | + # ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | + # Set-Content -Path "$runPublisherWithEnvPath" + + # Write-Information "Execution complete." + # shell: pwsh + + # - name: Setup artifact contents + # run: | + # Set-StrictMode -Version Latest + # $ErrorActionPreference = "Stop" + # $VerbosePreference = "Continue" + # $InformationPreference = "Continue" + + # $artifactsFolderPath = Join-Path "${{ runner.temp }}" "azdo_artifacts" + + # Write-Information "Copying Azure DevOps pipelines..." + # $sourceFolderPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "*" + # $destinationFolderPath = Join-Path "$artifactsFolderPath" "tools" "pipelines" + # New-Item -Path "$destinationFolderPath" -ItemType "Directory" + # Copy-Item -Path "$sourceFolderPath" -Destination "$destinationFolderPath" -Recurse + + # Write-Information "Zipping artifacts..." + # $destinationFilePath = Join-Path "${{ runner.temp }}" "Azure_DevOps.zip" + + # # We use System.IO.Compression.ZipFile instead of Compress-Archive because of this issue: https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/66 + # [System.IO.Compression.ZipFile]::CreateFromDirectory("$artifactsFolderPath", "$destinationFilePath") + + # Write-Information "Execution complete." + # shell: pwsh + + # - name: Upload artifact + # uses: actions/upload-artifact@v3 + # with: + # path: ${{ runner.temp }}/Azure_DevOps.zip + + # generate_release: + # name: Generate release + # needs: + # [ + # generate_extractor_artifacts, + # generate_publisher_artifacts, + # generate_github_pipeline_artifacts, + # generate_ado_pipeline_artifacts, + # ] + # runs-on: ubuntu-latest + # permissions: + # contents: write + # steps: + # - name: Download artifacts + # uses: actions/download-artifact@v3 + # - name: Create release + # uses: softprops/action-gh-release@v1 + # with: + # files: | + # ${{github.workspace}}/artifact/extractor.win-x64.exe + # ${{github.workspace}}/artifact/extractor.linux-x64 + # ${{github.workspace}}/artifact/extractor.linux-arm64 + # ${{github.workspace}}/artifact/extractor.linux-musl-x64 + # ${{github.workspace}}/artifact/extractor.linux-musl-arm64 + # ${{github.workspace}}/artifact/extractor.osx-arm64 + # ${{github.workspace}}/artifact/extractor.osx-x64 + # ${{github.workspace}}/artifact/publisher.win-x64.exe + # ${{github.workspace}}/artifact/publisher.linux-x64 + # ${{github.workspace}}/artifact/publisher.linux-arm64 + # ${{github.workspace}}/artifact/publisher.linux-musl-x64 + # ${{github.workspace}}/artifact/publisher.linux-musl-arm64 + # ${{github.workspace}}/artifact/publisher.osx-arm64 + # ${{github.workspace}}/artifact/publisher.osx-x64 + # ${{github.workspace}}/artifact/Github.zip + # ${{github.workspace}}/artifact/Azure_DevOps.zip + # name: APIOps Toolkit for Azure APIM ${{ github.event.inputs.Release_Version }} + # tag_name: ${{ github.event.inputs.Release_Version }} + # generate_release_notes: true From 849d62f1156b9681fe8aedd4d75c022fe7730381 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 15:11:26 -0400 Subject: [PATCH 02/29] Create publisher release with optimal compression --- .github/workflows/create_github_release.yaml | 111 ++++++++++--------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 8989a618..4840d2df 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -9,8 +9,57 @@ on: type: string jobs: - generate_extractor_artifacts: - name: Generate extractor artifacts +# generate_extractor_artifacts: +# name: Generate extractor artifacts +# strategy: +# matrix: +# dotnet-runtime: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, win-x64, osx-arm64, osx-x64] +# # Dynamically set the runner OS based on the .NET runtime +# runs-on: ${{ fromJSON('{"linux-x64":"ubuntu-latest", "linux-arm64":"ubuntu-latest", "linux-musl-x64":"ubuntu-latest", "linux-musl-arm64":"ubuntu:latest", "win-x64":"windows-latest", "osx-arm64":"macos-latest", "osx-x64":"macos-latest"}')[matrix.dotnet-runtime] }} +# steps: +# - name: Checkout repository +# uses: actions/checkout@v3 + +# - name: Setup .NET +# uses: actions/setup-dotnet@v3 +# with: +# dotnet-version: 8.x + +# - name: Generate executable +# run: | +# Set-StrictMode -Version Latest +# $ErrorActionPreference = "Stop" +# $VerbosePreference = "Continue" +# $InformationPreference = "Continue" + +# Write-Information "Creating output directory..." +# $outputFolderPath = Join-Path "${{ runner.temp }}" "extractor-output" +# New-Item -Path "$outputFolderPath" -ItemType "Directory" + +# Write-Information "Publishing application..." +# $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "extractor" "extractor.csproj" +# & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" +# if ($LASTEXITCODE -ne 0) { throw "Generating extractor failed."} +# ## DELETE +# Get-ChildItem -Path $outputFolderPath -Recurse + +# Write-Information "Zipping application..." +# $sourceFolderPath = Join-Path "$outputFolderPath" "*" +# $extractorFileNameWithoutExtension = "${{ format('extractor.{0}', matrix.dotnet-runtime) }}" +# $destinationFilePath = Join-Path "$outputFolderPath" "$($extractorFileNameWithoutExtension).zip" +# Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath +# "{ZIP_FILE_PATH}=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append + +# Write-Information "Execution complete." +# shell: pwsh + +# - name: Upload artifact +# uses: actions/upload-artifact@v3 +# with: +# path: ${{ format('{0}/{1}', runner.temp, env.ZIP_FILE_PATH) }} + + generate_publisher_artifacts: + name: Generate publisher artifacts strategy: matrix: dotnet-runtime: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, win-x64, osx-arm64, osx-x64] @@ -33,21 +82,21 @@ jobs: $InformationPreference = "Continue" Write-Information "Creating output directory..." - $outputFolderPath = Join-Path "${{ runner.temp }}" "extractor-output" + $outputFolderPath = Join-Path "${{ runner.temp }}" "publisher-output" New-Item -Path "$outputFolderPath" -ItemType "Directory" Write-Information "Publishing application..." - $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "extractor" "extractor.csproj" + $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "publisher" "publisher.csproj" & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" - if ($LASTEXITCODE -ne 0) { throw "Generating extractor failed."} + if ($LASTEXITCODE -ne 0) { throw "Generating publisher failed."} ## DELETE Get-ChildItem -Path $outputFolderPath -Recurse Write-Information "Zipping application..." $sourceFolderPath = Join-Path "$outputFolderPath" "*" - $extractorFileNameWithoutExtension = "${{ format('extractor.{0}', matrix.dotnet-runtime) }}" - $destinationFilePath = Join-Path "$outputFolderPath" "$($extractorFileNameWithoutExtension).zip" - Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath + $publisherFileNameWithoutExtension = "${{ format('publisher.{0}', matrix.dotnet-runtime) }}" + $destinationFilePath = Join-Path "$outputFolderPath" "$($publisherFileNameWithoutExtension).zip" + Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -CompressionLevel Optimal "{ZIP_FILE_PATH}=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append Write-Information "Execution complete." @@ -58,52 +107,6 @@ jobs: with: path: ${{ format('{0}/{1}', runner.temp, env.ZIP_FILE_PATH) }} - # generate_publisher_artifacts: - # name: Generate publisher artifacts - # strategy: - # matrix: - # dotnet-runtime: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, win-x64, osx-arm64, osx-x64] - # # Dynamically set the runner OS based on the .NET runtime - # runs-on: ${{ fromJSON('{"linux-x64":"ubuntu-latest", "linux-arm64":"ubuntu-latest", "linux-musl-x64":"ubuntu-latest", "linux-musl-arm64":"ubuntu:latest", "win-x64":"windows-latest", "osx-arm64":"macos-latest", "osx-x64":"macos-latest"}')[matrix.dotnet-runtime] }} - # steps: - # - name: Checkout repository - # uses: actions/checkout@v3 - - # - name: Setup .NET - # uses: actions/setup-dotnet@v3 - # with: - # dotnet-version: 8.x - - # - name: Generate executable - # run: | - # Set-StrictMode -Version Latest - # $ErrorActionPreference = "Stop" - # $VerbosePreference = "Continue" - # $InformationPreference = "Continue" - - # Write-Information "Generating publisher..." - # $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "publisher" "publisher.csproj" - # $outputFolderPath = "${{ runner.temp }}" - - # & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" - # if ($LASTEXITCODE -ne 0) { throw "Generating publisher failed."} - - # $exeFileExt = "${{ matrix.dotnet-runtime }}".Contains("win") ? ".exe" : "" - # $exeFolderPath = Join-Path "$outputFolderPath" "publisher$exeFileExt" - # $exeFileNameFinal = "${{ format('publisher.{0}', matrix.dotnet-runtime) }}$exeFileExt" - # Rename-Item -Path "$exeFolderPath" -NewName $exeFileNameFinal - # echo "PUBLISHER_FILENAME=$exeFileNameFinal" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - - # Write-Information "Execution complete." - # shell: pwsh - # env: - # PUBLISHER_FILENAME: - - # - name: Upload artifact - # uses: actions/upload-artifact@v3 - # with: - # path: ${{ format('{0}/{1}', runner.temp, env.PUBLISHER_FILENAME) }} - # generate_github_pipeline_artifacts: # name: Generate GitHub artifacts # runs-on: ubuntu-latest From 160c5debe1544e79cef25f03fd7f8a6124bd4961 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 15:23:11 -0400 Subject: [PATCH 03/29] Use latest upload action --- .github/workflows/create_github_release.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 4840d2df..6a6a63a3 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -40,8 +40,6 @@ jobs: # $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "extractor" "extractor.csproj" # & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" # if ($LASTEXITCODE -ne 0) { throw "Generating extractor failed."} -# ## DELETE -# Get-ChildItem -Path $outputFolderPath -Recurse # Write-Information "Zipping application..." # $sourceFolderPath = Join-Path "$outputFolderPath" "*" @@ -89,13 +87,12 @@ jobs: $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "publisher" "publisher.csproj" & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" if ($LASTEXITCODE -ne 0) { throw "Generating publisher failed."} - ## DELETE - Get-ChildItem -Path $outputFolderPath -Recurse Write-Information "Zipping application..." + ## DELETE + Get-ChildItem -Path $outputFolderPath -Recurse $sourceFolderPath = Join-Path "$outputFolderPath" "*" - $publisherFileNameWithoutExtension = "${{ format('publisher.{0}', matrix.dotnet-runtime) }}" - $destinationFilePath = Join-Path "$outputFolderPath" "$($publisherFileNameWithoutExtension).zip" + $destinationFilePath = Join-Path "$outputFolderPath" "publisher.${{ matrix.dotnet-runtime) }}.zip" Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -CompressionLevel Optimal "{ZIP_FILE_PATH}=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append @@ -103,8 +100,9 @@ jobs: shell: pwsh - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: + name: publisher-${{ matrix.dotnet-runtime }} path: ${{ format('{0}/{1}', runner.temp, env.ZIP_FILE_PATH) }} # generate_github_pipeline_artifacts: From f91dc4f2dada125e0ad5ba600561968b1080e0de Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 15:24:24 -0400 Subject: [PATCH 04/29] Remove extraneous character --- .github/workflows/create_github_release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 6a6a63a3..5558e810 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -92,7 +92,7 @@ jobs: ## DELETE Get-ChildItem -Path $outputFolderPath -Recurse $sourceFolderPath = Join-Path "$outputFolderPath" "*" - $destinationFilePath = Join-Path "$outputFolderPath" "publisher.${{ matrix.dotnet-runtime) }}.zip" + $destinationFilePath = Join-Path "$outputFolderPath" "publisher.${{ matrix.dotnet-runtime }}.zip" Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -CompressionLevel Optimal "{ZIP_FILE_PATH}=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append From aa7cf73a8ebd44f952c98e836ce6979c79071ac7 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 15:29:52 -0400 Subject: [PATCH 05/29] Fix upload path --- .github/workflows/create_github_release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 5558e810..7cc34706 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -94,7 +94,7 @@ jobs: $sourceFolderPath = Join-Path "$outputFolderPath" "*" $destinationFilePath = Join-Path "$outputFolderPath" "publisher.${{ matrix.dotnet-runtime }}.zip" Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -CompressionLevel Optimal - "{ZIP_FILE_PATH}=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append + "ZIP_FILE_PATH=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append Write-Information "Execution complete." shell: pwsh @@ -103,7 +103,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: publisher-${{ matrix.dotnet-runtime }} - path: ${{ format('{0}/{1}', runner.temp, env.ZIP_FILE_PATH) }} + path: ${{ env.ZIP_FILE_PATH) }} # generate_github_pipeline_artifacts: # name: Generate GitHub artifacts From 17439cac63844edfcea44c48fb00f02bc3d6c690 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 15:32:40 -0400 Subject: [PATCH 06/29] Remove extraneous path --- .github/workflows/create_github_release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 7cc34706..0e9b4908 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -103,7 +103,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: publisher-${{ matrix.dotnet-runtime }} - path: ${{ env.ZIP_FILE_PATH) }} + path: ${{ env.ZIP_FILE_PATH }} # generate_github_pipeline_artifacts: # name: Generate GitHub artifacts From 45343c5a7e5a261c0747c86c1f4cdd81170eec74 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 20:09:46 -0400 Subject: [PATCH 07/29] Include extractor artifacts --- .github/workflows/create_github_release.yaml | 95 ++++++++++---------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 0e9b4908..ffeb6958 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -9,52 +9,51 @@ on: type: string jobs: -# generate_extractor_artifacts: -# name: Generate extractor artifacts -# strategy: -# matrix: -# dotnet-runtime: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, win-x64, osx-arm64, osx-x64] -# # Dynamically set the runner OS based on the .NET runtime -# runs-on: ${{ fromJSON('{"linux-x64":"ubuntu-latest", "linux-arm64":"ubuntu-latest", "linux-musl-x64":"ubuntu-latest", "linux-musl-arm64":"ubuntu:latest", "win-x64":"windows-latest", "osx-arm64":"macos-latest", "osx-x64":"macos-latest"}')[matrix.dotnet-runtime] }} -# steps: -# - name: Checkout repository -# uses: actions/checkout@v3 - -# - name: Setup .NET -# uses: actions/setup-dotnet@v3 -# with: -# dotnet-version: 8.x - -# - name: Generate executable -# run: | -# Set-StrictMode -Version Latest -# $ErrorActionPreference = "Stop" -# $VerbosePreference = "Continue" -# $InformationPreference = "Continue" - -# Write-Information "Creating output directory..." -# $outputFolderPath = Join-Path "${{ runner.temp }}" "extractor-output" -# New-Item -Path "$outputFolderPath" -ItemType "Directory" - -# Write-Information "Publishing application..." -# $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "extractor" "extractor.csproj" -# & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" -# if ($LASTEXITCODE -ne 0) { throw "Generating extractor failed."} - -# Write-Information "Zipping application..." -# $sourceFolderPath = Join-Path "$outputFolderPath" "*" -# $extractorFileNameWithoutExtension = "${{ format('extractor.{0}', matrix.dotnet-runtime) }}" -# $destinationFilePath = Join-Path "$outputFolderPath" "$($extractorFileNameWithoutExtension).zip" -# Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -# "{ZIP_FILE_PATH}=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append - -# Write-Information "Execution complete." -# shell: pwsh - -# - name: Upload artifact -# uses: actions/upload-artifact@v3 -# with: -# path: ${{ format('{0}/{1}', runner.temp, env.ZIP_FILE_PATH) }} + generate_extractor_artifacts: + name: Generate extractor artifacts + strategy: + matrix: + dotnet-runtime: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, win-x64, osx-arm64, osx-x64] + # Dynamically set the runner OS based on the .NET runtime + runs-on: ${{ fromJSON('{"linux-x64":"ubuntu-latest", "linux-arm64":"ubuntu-latest", "linux-musl-x64":"ubuntu-latest", "linux-musl-arm64":"ubuntu-latest", "win-x64":"windows-latest", "osx-arm64":"macos-latest", "osx-x64":"macos-latest"}')[matrix.dotnet-runtime] }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 8.x + + - name: Generate executable + run: | + Set-StrictMode -Version Latest + $ErrorActionPreference = "Stop" + $VerbosePreference = "Continue" + $InformationPreference = "Continue" + + Write-Information "Creating output directory..." + $outputFolderPath = Join-Path "${{ runner.temp }}" "extractor-output" + New-Item -Path "$outputFolderPath" -ItemType "Directory" + + Write-Information "Publishing application..." + $sourcePath = Join-Path "${{ github.workspace }}" "tools" "code" "extractor" "extractor.csproj" + & dotnet publish "$sourcePath" --self-contained --runtime "${{ matrix.dotnet-runtime }}" -p:PublishSingleFile=true --output "$outputFolderPath" + if ($LASTEXITCODE -ne 0) { throw "Generating extractor failed."} + + Write-Information "Zipping application..." + $sourceFolderPath = Join-Path "$outputFolderPath" "*" + $destinationFilePath = Join-Path "$outputFolderPath" "extractor.${{ matrix.dotnet-runtime }}.zip" + Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -CompressionLevel Optimal + "ZIP_FILE_PATH=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append + + Write-Information "Execution complete." + shell: pwsh + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + path: ${{ format('{0}/{1}', runner.temp, env.ZIP_FILE_PATH) }} generate_publisher_artifacts: name: Generate publisher artifacts @@ -62,7 +61,7 @@ jobs: matrix: dotnet-runtime: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, win-x64, osx-arm64, osx-x64] # Dynamically set the runner OS based on the .NET runtime - runs-on: ${{ fromJSON('{"linux-x64":"ubuntu-latest", "linux-arm64":"ubuntu-latest", "linux-musl-x64":"ubuntu-latest", "linux-musl-arm64":"ubuntu:latest", "win-x64":"windows-latest", "osx-arm64":"macos-latest", "osx-x64":"macos-latest"}')[matrix.dotnet-runtime] }} + runs-on: ${{ fromJSON('{"linux-x64":"ubuntu-latest", "linux-arm64":"ubuntu-latest", "linux-musl-x64":"ubuntu-latest", "linux-musl-arm64":"ubuntu-latest", "win-x64":"windows-latest", "osx-arm64":"macos-latest", "osx-x64":"macos-latest"}')[matrix.dotnet-runtime] }} steps: - name: Checkout repository uses: actions/checkout@v3 @@ -89,8 +88,6 @@ jobs: if ($LASTEXITCODE -ne 0) { throw "Generating publisher failed."} Write-Information "Zipping application..." - ## DELETE - Get-ChildItem -Path $outputFolderPath -Recurse $sourceFolderPath = Join-Path "$outputFolderPath" "*" $destinationFilePath = Join-Path "$outputFolderPath" "publisher.${{ matrix.dotnet-runtime }}.zip" Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -CompressionLevel Optimal From 351c90bb1b680f9489473bb4142527d0385cde18 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 20:16:25 -0400 Subject: [PATCH 08/29] Fix extractor path --- .github/workflows/create_github_release.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index ffeb6958..4159fd61 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -51,9 +51,10 @@ jobs: shell: pwsh - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - path: ${{ format('{0}/{1}', runner.temp, env.ZIP_FILE_PATH) }} + name: extractor-${{ matrix.dotnet-runtime }} + path: ${{ env.ZIP_FILE_PATH }} generate_publisher_artifacts: name: Generate publisher artifacts From 7f403461fd97bead0ff0d5bf54711f99d25ebaa0 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 1 May 2024 20:46:50 -0400 Subject: [PATCH 09/29] Update pipelines with zip files --- .github/workflows/create_github_release.yaml | 306 +++++++++--------- tools/azdo_pipelines/run-extractor.yaml | 38 ++- .../run-publisher-with-env.yaml | 37 ++- tools/github_workflows/run-extractor.yaml | 72 +++-- .../run-publisher-with-env.yaml | 156 +++++---- 5 files changed, 342 insertions(+), 267 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 4159fd61..d8c89d2b 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -43,7 +43,7 @@ jobs: Write-Information "Zipping application..." $sourceFolderPath = Join-Path "$outputFolderPath" "*" - $destinationFilePath = Join-Path "$outputFolderPath" "extractor.${{ matrix.dotnet-runtime }}.zip" + $destinationFilePath = Join-Path "$outputFolderPath" "extractor-${{ matrix.dotnet-runtime }}.zip" Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -CompressionLevel Optimal "ZIP_FILE_PATH=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append @@ -90,7 +90,7 @@ jobs: Write-Information "Zipping application..." $sourceFolderPath = Join-Path "$outputFolderPath" "*" - $destinationFilePath = Join-Path "$outputFolderPath" "publisher.${{ matrix.dotnet-runtime }}.zip" + $destinationFilePath = Join-Path "$outputFolderPath" "publisher-${{ matrix.dotnet-runtime }}.zip" Compress-Archive -Path $sourceFolderPath -DestinationPath $destinationFilePath -CompressionLevel Optimal "ZIP_FILE_PATH=$destinationFilePath" | Out-File -FilePath $env:GITHUB_ENV -Append @@ -103,153 +103,155 @@ jobs: name: publisher-${{ matrix.dotnet-runtime }} path: ${{ env.ZIP_FILE_PATH }} - # generate_github_pipeline_artifacts: - # name: Generate GitHub artifacts - # runs-on: ubuntu-latest - # steps: - # - name: Checkout repository - # uses: actions/checkout@v3 - - # - name: Update versions in YAML - # run: | - # Set-StrictMode -Version Latest - # $ErrorActionPreference = "Stop" - # $VerbosePreference = "Continue" - # $InformationPreference = "Continue" - - # $runExtractorPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "run-extractor.yaml" - # (Get-Content -Path "$runExtractorPath") | - # ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | - # Set-Content -Path "$runExtractorPath" - - # $runPublisherWithEnvPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "run-publisher-with-env.yaml" - # (Get-Content -Path "$runPublisherWithEnvPath") | - # ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | - # Set-Content -Path "$runPublisherWithEnvPath" - - # Write-Information "Execution complete." - # shell: pwsh - - # - name: Setup artifact contents - # run: | - # Set-StrictMode -Version Latest - # $ErrorActionPreference = "Stop" - # $VerbosePreference = "Continue" - # $InformationPreference = "Continue" - - # $artifactsFolderPath = Join-Path "${{ runner.temp }}" "github_artifacts" - - # Write-Information "Copying GitHub workflows..." - # $sourceFolderPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "*" - # $destinationFolderPath = Join-Path "$artifactsFolderPath" ".github" "workflows" - # New-Item -Path "$destinationFolderPath" -ItemType "Directory" - # Copy-Item -Path "$sourceFolderPath" -Destination "$destinationFolderPath" -Recurse - - # Write-Information "Zipping artifacts..." - # $destinationFilePath = Join-Path "${{ runner.temp }}" "Github.zip" - - # # We use System.IO.Compression.ZipFile instead of Compress-Archive because of this issue: https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/66 - # [System.IO.Compression.ZipFile]::CreateFromDirectory("$artifactsFolderPath", "$destinationFilePath") - - # Write-Information "Execution complete." - # shell: pwsh - - # - name: Upload artifact - # uses: actions/upload-artifact@v3 - # with: - # path: ${{ runner.temp }}/Github.zip - - # generate_ado_pipeline_artifacts: - # name: Generate Azure DevOps artifacts - # runs-on: ubuntu-latest - # steps: - # - name: Checkout repository - # uses: actions/checkout@v3 - - # - name: Update versions in YAML - # run: | - # Set-StrictMode -Version Latest - # $ErrorActionPreference = "Stop" - # $VerbosePreference = "Continue" - # $InformationPreference = "Continue" - - # $runExtractorPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "run-extractor.yaml" - # (Get-Content -Path "$runExtractorPath") | - # ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | - # Set-Content -Path "$runExtractorPath" - - # $runPublisherWithEnvPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "run-publisher-with-env.yaml" - # (Get-Content -Path "$runPublisherWithEnvPath") | - # ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | - # Set-Content -Path "$runPublisherWithEnvPath" - - # Write-Information "Execution complete." - # shell: pwsh - - # - name: Setup artifact contents - # run: | - # Set-StrictMode -Version Latest - # $ErrorActionPreference = "Stop" - # $VerbosePreference = "Continue" - # $InformationPreference = "Continue" - - # $artifactsFolderPath = Join-Path "${{ runner.temp }}" "azdo_artifacts" - - # Write-Information "Copying Azure DevOps pipelines..." - # $sourceFolderPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "*" - # $destinationFolderPath = Join-Path "$artifactsFolderPath" "tools" "pipelines" - # New-Item -Path "$destinationFolderPath" -ItemType "Directory" - # Copy-Item -Path "$sourceFolderPath" -Destination "$destinationFolderPath" -Recurse - - # Write-Information "Zipping artifacts..." - # $destinationFilePath = Join-Path "${{ runner.temp }}" "Azure_DevOps.zip" - - # # We use System.IO.Compression.ZipFile instead of Compress-Archive because of this issue: https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/66 - # [System.IO.Compression.ZipFile]::CreateFromDirectory("$artifactsFolderPath", "$destinationFilePath") - - # Write-Information "Execution complete." - # shell: pwsh - - # - name: Upload artifact - # uses: actions/upload-artifact@v3 - # with: - # path: ${{ runner.temp }}/Azure_DevOps.zip - - # generate_release: - # name: Generate release - # needs: - # [ - # generate_extractor_artifacts, - # generate_publisher_artifacts, - # generate_github_pipeline_artifacts, - # generate_ado_pipeline_artifacts, - # ] - # runs-on: ubuntu-latest - # permissions: - # contents: write - # steps: - # - name: Download artifacts - # uses: actions/download-artifact@v3 - # - name: Create release - # uses: softprops/action-gh-release@v1 - # with: - # files: | - # ${{github.workspace}}/artifact/extractor.win-x64.exe - # ${{github.workspace}}/artifact/extractor.linux-x64 - # ${{github.workspace}}/artifact/extractor.linux-arm64 - # ${{github.workspace}}/artifact/extractor.linux-musl-x64 - # ${{github.workspace}}/artifact/extractor.linux-musl-arm64 - # ${{github.workspace}}/artifact/extractor.osx-arm64 - # ${{github.workspace}}/artifact/extractor.osx-x64 - # ${{github.workspace}}/artifact/publisher.win-x64.exe - # ${{github.workspace}}/artifact/publisher.linux-x64 - # ${{github.workspace}}/artifact/publisher.linux-arm64 - # ${{github.workspace}}/artifact/publisher.linux-musl-x64 - # ${{github.workspace}}/artifact/publisher.linux-musl-arm64 - # ${{github.workspace}}/artifact/publisher.osx-arm64 - # ${{github.workspace}}/artifact/publisher.osx-x64 - # ${{github.workspace}}/artifact/Github.zip - # ${{github.workspace}}/artifact/Azure_DevOps.zip - # name: APIOps Toolkit for Azure APIM ${{ github.event.inputs.Release_Version }} - # tag_name: ${{ github.event.inputs.Release_Version }} - # generate_release_notes: true + generate_github_pipeline_artifacts: + name: Generate GitHub artifacts + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Update versions in YAML + run: | + Set-StrictMode -Version Latest + $ErrorActionPreference = "Stop" + $VerbosePreference = "Continue" + $InformationPreference = "Continue" + + $runExtractorPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "run-extractor.yaml" + (Get-Content -Path "$runExtractorPath") | + ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | + Set-Content -Path "$runExtractorPath" + + $runPublisherWithEnvPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "run-publisher-with-env.yaml" + (Get-Content -Path "$runPublisherWithEnvPath") | + ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | + Set-Content -Path "$runPublisherWithEnvPath" + + Write-Information "Execution complete." + shell: pwsh + + - name: Setup artifact contents + run: | + Set-StrictMode -Version Latest + $ErrorActionPreference = "Stop" + $VerbosePreference = "Continue" + $InformationPreference = "Continue" + + $artifactsFolderPath = Join-Path "${{ runner.temp }}" "github_artifacts" + + Write-Information "Copying GitHub workflows..." + $sourceFolderPath = Join-Path "${{ github.workspace }}" "tools" "github_workflows" "*" + $destinationFolderPath = Join-Path "$artifactsFolderPath" ".github" "workflows" + New-Item -Path "$destinationFolderPath" -ItemType "Directory" + Copy-Item -Path "$sourceFolderPath" -Destination "$destinationFolderPath" -Recurse + + Write-Information "Zipping artifacts..." + $destinationFilePath = Join-Path "${{ runner.temp }}" "Github.zip" + + # We use System.IO.Compression.ZipFile instead of Compress-Archive because of this issue: https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/66 + [System.IO.Compression.ZipFile]::CreateFromDirectory("$artifactsFolderPath", "$destinationFilePath") + + Write-Information "Execution complete." + shell: pwsh + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + path: ${{ runner.temp }}/Github.zip + + generate_ado_pipeline_artifacts: + name: Generate Azure DevOps artifacts + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Update versions in YAML + run: | + Set-StrictMode -Version Latest + $ErrorActionPreference = "Stop" + $VerbosePreference = "Continue" + $InformationPreference = "Continue" + + $runExtractorPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "run-extractor.yaml" + (Get-Content -Path "$runExtractorPath") | + ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | + Set-Content -Path "$runExtractorPath" + + $runPublisherWithEnvPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "run-publisher-with-env.yaml" + (Get-Content -Path "$runPublisherWithEnvPath") | + ForEach-Object {$_ -Replace 'desired-version-goes-here', '${{ github.event.inputs.Release_Version }}'} | + Set-Content -Path "$runPublisherWithEnvPath" + + Write-Information "Execution complete." + shell: pwsh + + - name: Setup artifact contents + run: | + Set-StrictMode -Version Latest + $ErrorActionPreference = "Stop" + $VerbosePreference = "Continue" + $InformationPreference = "Continue" + + $artifactsFolderPath = Join-Path "${{ runner.temp }}" "azdo_artifacts" + + Write-Information "Copying Azure DevOps pipelines..." + $sourceFolderPath = Join-Path "${{ github.workspace }}" "tools" "azdo_pipelines" "*" + $destinationFolderPath = Join-Path "$artifactsFolderPath" "tools" "pipelines" + New-Item -Path "$destinationFolderPath" -ItemType "Directory" + Copy-Item -Path "$sourceFolderPath" -Destination "$destinationFolderPath" -Recurse + + Write-Information "Zipping artifacts..." + $destinationFilePath = Join-Path "${{ runner.temp }}" "Azure_DevOps.zip" + + # We use System.IO.Compression.ZipFile instead of Compress-Archive because of this issue: https://github.com/PowerShell/Microsoft.PowerShell.Archive/issues/66 + [System.IO.Compression.ZipFile]::CreateFromDirectory("$artifactsFolderPath", "$destinationFilePath") + + Write-Information "Execution complete." + shell: pwsh + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + path: ${{ runner.temp }}/Azure_DevOps.zip + + generate_release: + name: Generate release + needs: + [ + generate_extractor_artifacts, + generate_publisher_artifacts, + generate_github_pipeline_artifacts, + generate_ado_pipeline_artifacts, + ] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download artifacts + uses: actions/download-artifact@v3 + - name: Create release + uses: softprops/action-gh-release@v1 + with: + files: | + ${{github.workspace}}/artifact/extractor-linux-arm64.zip + ${{github.workspace}}/artifact/extractor-linux-musl-arm64.zip + ${{github.workspace}}/artifact/extractor-linux-musl-x64.zip + ${{github.workspace}}/artifact/extractor-linux-x64.zip + ${{github.workspace}}/artifact/extractor-osx-arm64.zip + ${{github.workspace}}/artifact/extractor-osx-x64.zip + ${{github.workspace}}/artifact/extractor.osx-x64.zip + ${{github.workspace}}/artifact/extractor-win-x64.zip + ${{github.workspace}}/artifact/publisher-linux-arm64.zip + ${{github.workspace}}/artifact/publisher-linux-musl-arm64.zip + ${{github.workspace}}/artifact/publisher-linux-musl-x64.zip + ${{github.workspace}}/artifact/publisher-linux-x64.zip + ${{github.workspace}}/artifact/publisher-osx-arm64.zip + ${{github.workspace}}/artifact/publisher-osx-x64.zip + ${{github.workspace}}/artifact/publisher.osx-x64.zip + ${{github.workspace}}/artifact/publisher-win-x64.zip + ${{github.workspace}}/artifact/Github.zip + ${{github.workspace}}/artifact/Azure_DevOps.zip + name: APIOps Toolkit for Azure APIM ${{ github.event.inputs.Release_Version }} + tag_name: ${{ github.event.inputs.Release_Version }} + generate_release_notes: true diff --git a/tools/azdo_pipelines/run-extractor.yaml b/tools/azdo_pipelines/run-extractor.yaml index 2f32e66d..48529fc0 100644 --- a/tools/azdo_pipelines/run-extractor.yaml +++ b/tools/azdo_pipelines/run-extractor.yaml @@ -91,33 +91,39 @@ stages: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Downloading extractor..." - $extractorFileName = "extractor.linux-x64" - $extractorFinalFileName = "extractor" - + Write-Information "Setting name variables..." + $releaseFileName = "extractor-linux-x64.zip" + $executableFileName = "extractor" + if ("$(Agent.OS)" -like "*win*") { - $extractorFileName = "extractor.win-x64.exe" - $extractorFinalFileName = "extractor.exe" + $releaseFileName = "extractor-win-x64.zip" + $executableFileName = "extractor.exe" } elseif ("$(Agent.OS)" -like "*mac*" -and "$(Agent.OSArchitecture)" -like "*arm*") { - $extractorFileName = "extractor.osx-arm64" + $releaseFileName = "extractor-osx-arm64.zip" } elseif ("$(Agent.OS)" -like "*mac*" -and "$(Agent.OSArchitecture)" -like "*x86_64*") { - $extractorFileName = "extractor.osx-x64" + $releaseFileName = "extractor-osx-x64.zip" } - - $uri = "https://github.com/Azure/apiops/releases/download/$(apiops_release_version)/$extractorFileName" - $destinationFilePath = Join-Path "$(Agent.TempDirectory)" $extractorFinalFileName - Invoke-WebRequest -Uri "$uri" -OutFile "$destinationFilePath" - if ("$(Agent.OS)" -like "*linux*") + Write-Information "Downloading release..." + $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" + $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" + + Write-Information "Extracting release..." + $executableFolderPath = Join-Path "${{ runner.temp }}" "extractor" + Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" + $executableFilePath = Join-Path "$executableFolderPath" $executableFileName + + Write-Information "Setting file permissions..." + if ("${{ runner.os }}" -like "*linux*") { - Write-Information "Setting file permissions..." - & chmod +x "$destinationFilePath" + & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} } - Write-Host "##vso[task.setvariable variable=EXTRACTOR_FILE_PATH]$destinationFilePath" + Write-Host "##vso[task.setvariable variable=EXTRACTOR_FILE_PATH]$executableFilePath" Write-Information "Execution complete." failOnStderr: true pwsh: true diff --git a/tools/azdo_pipelines/run-publisher-with-env.yaml b/tools/azdo_pipelines/run-publisher-with-env.yaml index e8e846df..29d00d82 100644 --- a/tools/azdo_pipelines/run-publisher-with-env.yaml +++ b/tools/azdo_pipelines/run-publisher-with-env.yaml @@ -87,32 +87,39 @@ steps: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Downloading publisher..." - $publisherFileName = "publisher.linux-x64" - $publisherFinalFileName = "publisher" + Write-Information "Setting name variables..." + $releaseFileName = "publisher-linux-x64.zip" + $executableFileName = "publisher" + if ("$(Agent.OS)" -like "*win*") { - $publisherFileName = "publisher.win-x64.exe" - $publisherFinalFileName = "publisher.exe" + $releaseFileName = "publisher-win-x64.zip" + $executableFileName = "publisher.exe" } elseif ("$(Agent.OS)" -like "*mac*" -and "$(Agent.OSArchitecture)" -like "*arm*") { - $publisherFileName = "publisher.osx-arm64" + $releaseFileName = "publisher-osx-arm64.zip" } elseif ("$(Agent.OS)" -like "*mac*" -and "$(Agent.OSArchitecture)" -like "*x86_64*") { - $publisherFileName = "publisher.osx-x64" + $releaseFileName = "publisher-osx-x64.zip" } - - $uri = "https://github.com/Azure/apiops/releases/download/$(apiops_release_version)/$publisherFileName" - $destinationFilePath = Join-Path "$(Agent.TempDirectory)" $publisherFinalFileName - Invoke-WebRequest -Uri "$uri" -OutFile "$destinationFilePath" - if ("$(Agent.OS)" -like "*linux*") + Write-Information "Downloading release..." + $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" + $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" + + Write-Information "Extracting release..." + $executableFolderPath = Join-Path "${{ runner.temp }}" "publisher" + Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" + $executableFilePath = Join-Path "$executableFolderPath" $executableFileName + + Write-Information "Setting file permissions..." + if ("${{ runner.os }}" -like "*linux*") { - Write-Information "Setting file permissions..." - & chmod +x "$destinationFilePath" + & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} } - Write-Host "##vso[task.setvariable variable=PUBLISHER_FILE_PATH]$destinationFilePath" + Write-Host "##vso[task.setvariable variable=PUBLISHER_FILE_PATH]$executableFilePath" Write-Information "Execution complete." failOnStderr: true pwsh: true diff --git a/tools/github_workflows/run-extractor.yaml b/tools/github_workflows/run-extractor.yaml index 6b1a9411..0c502ba3 100644 --- a/tools/github_workflows/run-extractor.yaml +++ b/tools/github_workflows/run-extractor.yaml @@ -47,32 +47,40 @@ jobs: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Downloading extractor..." - $extractorFileName = "extractor.linux-x64" - $extractorFinalFileName = "extractor" + Write-Information "Setting name variables..." + $releaseFileName = "extractor-linux-x64.zip" + $executableFileName = "extractor" + if ("${{ runner.os }}" -like "*win*") { - $extractorFileName = "extractor.win-x64.exe" - $extractorFinalFileName = "extractor.exe" + $releaseFileName = "extractor-win-x64.zip" + $executableFileName = "extractor.exe" } elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*") { - $extractorFileName = "extractor.osx-arm64" + $releaseFileName = "extractor-osx-arm64.zip" } elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*x86_64*") { - $extractorFileName = "extractor.osx-x64" + $releaseFileName = "extractor-osx-x64.zip" } - - $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$extractorFileName" - $destinationFilePath = Join-Path "${{ runner.temp }}" $extractorFinalFileName - Invoke-WebRequest -Uri "$uri" -OutFile "$destinationFilePath" + Write-Information "Downloading release..." + $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" + $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" + + Write-Information "Extracting release..." + $executableFolderPath = Join-Path "${{ runner.temp }}" "extractor" + Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" + $executableFilePath = Join-Path "$executableFolderPath" $executableFileName + + Write-Information "Setting file permissions..." if ("${{ runner.os }}" -like "*linux*") { - Write-Information "Setting file permissions..." - & chmod +x "$destinationFilePath" + & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} } - & "$destinationFilePath" + Write-Information "Running extractor..." + & "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Running extractor failed."} Write-Information "Execution complete." @@ -96,20 +104,40 @@ jobs: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Downloading extractor..." - $extractorFileName = "${{ runner.os }}" -like "*win*" ? "extractor.win-x64.exe" : "extractor.linux-x64.exe" - $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$extractorFileName" - $destinationFilePath = Join-Path "${{ runner.temp }}" "extractor.exe" - Invoke-WebRequest -Uri "$uri" -OutFile "$destinationFilePath" + Write-Information "Setting name variables..." + $releaseFileName = "extractor-linux-x64.zip" + $executableFileName = "extractor" + + if ("${{ runner.os }}" -like "*win*") { + $releaseFileName = "extractor-win-x64.zip" + $executableFileName = "extractor.exe" + } + elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*") { + $releaseFileName = "extractor-osx-arm64.zip" + } + elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*x86_64*") { + $releaseFileName = "extractor-osx-x64.zip" + } + + Write-Information "Downloading release..." + $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" + $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" + + Write-Information "Extracting release..." + $executableFolderPath = Join-Path "${{ runner.temp }}" "extractor" + Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" + $executableFilePath = Join-Path "$executableFolderPath" $executableFileName + Write-Information "Setting file permissions..." if ("${{ runner.os }}" -like "*linux*") { - Write-Information "Setting file permissions..." - & chmod +x "$destinationFilePath" + & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} } - & "$destinationFilePath" + Write-Information "Running extractor..." + & "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Running extractor failed."} Write-Information "Execution complete." diff --git a/tools/github_workflows/run-publisher-with-env.yaml b/tools/github_workflows/run-publisher-with-env.yaml index ee37ca35..c8c66525 100644 --- a/tools/github_workflows/run-publisher-with-env.yaml +++ b/tools/github_workflows/run-publisher-with-env.yaml @@ -68,32 +68,40 @@ jobs: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Downloading publisher..." - $publisherFileName = "publisher.linux-x64" - $publisherFinalFileName = "publisher" + Write-Information "Setting name variables..." + $releaseFileName = "publisher-linux-x64.zip" + $executableFileName = "publisher" + if ("${{ runner.os }}" -like "*win*") { - $publisherFileName = "publisher.win-x64.exe" - $publisherFinalFileName = "publisher.exe" + $releaseFileName = "publisher-win-x64.zip" + $executableFileName = "publisher.exe" } elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*") { - $publisherFileName = "publisher.osx-arm64" + $releaseFileName = "publisher-osx-arm64.zip" } elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*x86_64*") { - $publisherFileName = "publisher.osx-x64" + $releaseFileName = "publisher-osx-x64.zip" } - - $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$publisherFileName" - $destinationFilePath = Join-Path "${{ runner.temp }}" $publisherFinalFileName - Invoke-WebRequest -Uri "$uri" -OutFile "$destinationFilePath" + Write-Information "Downloading release..." + $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" + $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" + + Write-Information "Extracting release..." + $executableFolderPath = Join-Path "${{ runner.temp }}" "publisher" + Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" + $executableFilePath = Join-Path "$executableFolderPath" $executableFileName + + Write-Information "Setting file permissions..." if ("${{ runner.os }}" -like "*linux*") { - Write-Information "Setting file permissions..." - & chmod +x "$destinationFilePath" + & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} } - & "$destinationFilePath" + Write-Information "Running publisher..." + & "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Running publisher failed."} Write-Information "Execution complete." @@ -115,32 +123,40 @@ jobs: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Downloading publisher..." - $publisherFileName = "publisher.linux-x64" - $publisherFinalFileName = "publisher" - if("${{ runner.os }}" -like "*win*"){ - $publisherFileName = "publisher.win-x64.exe" - $publisherFinalFileName = "publisher.exe" + Write-Information "Setting name variables..." + $releaseFileName = "publisher-linux-x64.zip" + $executableFileName = "publisher" + + if ("${{ runner.os }}" -like "*win*") { + $releaseFileName = "publisher-win-x64.zip" + $executableFileName = "publisher.exe" } - elseif("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*"){ - $publisherFileName = "publisher.osx-arm64" + elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*") { + $releaseFileName = "publisher-osx-arm64.zip" } elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*x86_64*") { - $publisherFileName = "publisher.osx-x64" + $releaseFileName = "publisher-osx-x64.zip" } - - $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$publisherFileName" - $destinationFilePath = Join-Path "${{ runner.temp }}" $publisherFinalFileName - Invoke-WebRequest -Uri "$uri" -OutFile "$destinationFilePath" + Write-Information "Downloading release..." + $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" + $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" + + Write-Information "Extracting release..." + $executableFolderPath = Join-Path "${{ runner.temp }}" "publisher" + Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" + $executableFilePath = Join-Path "$executableFolderPath" $executableFileName + + Write-Information "Setting file permissions..." if ("${{ runner.os }}" -like "*linux*") { - Write-Information "Setting file permissions..." - & chmod +x "$destinationFilePath" + & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} } - & "$destinationFilePath" + Write-Information "Running publisher..." + & "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Running publisher failed."} Write-Information "Execution complete." @@ -164,32 +180,40 @@ jobs: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Downloading publisher..." - $publisherFileName = "publisher.linux-x64" - $publisherFinalFileName = "publisher" - if("${{ runner.os }}" -like "*win*"){ - $publisherFileName = "publisher.win-x64.exe" - $publisherFinalFileName = "publisher.exe" + Write-Information "Setting name variables..." + $releaseFileName = "publisher-linux-x64.zip" + $executableFileName = "publisher" + + if ("${{ runner.os }}" -like "*win*") { + $releaseFileName = "publisher-win-x64.zip" + $executableFileName = "publisher.exe" } - elseif("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*"){ - $publisherFileName = "publisher.osx-arm64" + elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*") { + $releaseFileName = "publisher-osx-arm64.zip" } elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*x86_64*") { - $publisherFileName = "publisher.osx-x64" + $releaseFileName = "publisher-osx-x64.zip" } - - $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$publisherFileName" - $destinationFilePath = Join-Path "${{ runner.temp }}" $publisherFinalFileName - Invoke-WebRequest -Uri "$uri" -OutFile "$destinationFilePath" + Write-Information "Downloading release..." + $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" + $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" + + Write-Information "Extracting release..." + $executableFolderPath = Join-Path "${{ runner.temp }}" "publisher" + Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" + $executableFilePath = Join-Path "$executableFolderPath" $executableFileName + + Write-Information "Setting file permissions..." if ("${{ runner.os }}" -like "*linux*") { - Write-Information "Setting file permissions..." - & chmod +x "$destinationFilePath" + & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} } - & "$destinationFilePath" + Write-Information "Running publisher..." + & "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Running publisher failed."} Write-Information "Execution complete." @@ -212,32 +236,40 @@ jobs: $VerbosePreference = "Continue" $InformationPreference = "Continue" - Write-Information "Downloading publisher..." - $publisherFileName = "publisher.linux-x64" - $publisherFinalFileName = "publisher" - if("${{ runner.os }}" -like "*win*"){ - $publisherFileName = "publisher.win-x64.exe" - $publisherFinalFileName = "publisher.exe" + Write-Information "Setting name variables..." + $releaseFileName = "publisher-linux-x64.zip" + $executableFileName = "publisher" + + if ("${{ runner.os }}" -like "*win*") { + $releaseFileName = "publisher-win-x64.zip" + $executableFileName = "publisher.exe" } - elseif("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*"){ - $publisherFileName = "publisher.osx-arm64" + elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*arm*") { + $releaseFileName = "publisher-osx-arm64.zip" } elseif ("${{ runner.os }}" -like "*mac*" -and "${{ runner.arch }}" -like "*x86_64*") { - $publisherFileName = "publisher.osx-x64" + $releaseFileName = "publisher-osx-x64.zip" } - - $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$publisherFileName" - $destinationFilePath = Join-Path "${{ runner.temp }}" $publisherFinalFileName - Invoke-WebRequest -Uri "$uri" -OutFile "$destinationFilePath" + Write-Information "Downloading release..." + $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" + $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" + + Write-Information "Extracting release..." + $executableFolderPath = Join-Path "${{ runner.temp }}" "publisher" + Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" + $executableFilePath = Join-Path "$executableFolderPath" $executableFileName + + Write-Information "Setting file permissions..." if ("${{ runner.os }}" -like "*linux*") { - Write-Information "Setting file permissions..." - & chmod +x "$destinationFilePath" + & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} } - & "$destinationFilePath" + Write-Information "Running publisher..." + & "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Running publisher failed."} Write-Information "Execution complete." From 197d9a3ed777d587cd0d5fb5eaf59771cce535e7 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Thu, 2 May 2024 12:36:59 -0400 Subject: [PATCH 10/29] Update artifact task versions --- .github/workflows/create_github_release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index d8c89d2b..3a08d418 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -155,7 +155,7 @@ jobs: shell: pwsh - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: ${{ runner.temp }}/Github.zip @@ -211,7 +211,7 @@ jobs: shell: pwsh - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: ${{ runner.temp }}/Azure_DevOps.zip @@ -229,7 +229,7 @@ jobs: contents: write steps: - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Create release uses: softprops/action-gh-release@v1 with: From 648e0a772b2540f95535f94585e2e9f5016b1826 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Thu, 2 May 2024 12:42:23 -0400 Subject: [PATCH 11/29] Create names for artifacts --- .github/workflows/create_github_release.yaml | 38 ++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 3a08d418..546fa861 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -157,6 +157,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: + name: github path: ${{ runner.temp }}/Github.zip generate_ado_pipeline_artifacts: @@ -213,6 +214,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4 with: + name: ado path: ${{ runner.temp }}/Azure_DevOps.zip generate_release: @@ -234,24 +236,24 @@ jobs: uses: softprops/action-gh-release@v1 with: files: | - ${{github.workspace}}/artifact/extractor-linux-arm64.zip - ${{github.workspace}}/artifact/extractor-linux-musl-arm64.zip - ${{github.workspace}}/artifact/extractor-linux-musl-x64.zip - ${{github.workspace}}/artifact/extractor-linux-x64.zip - ${{github.workspace}}/artifact/extractor-osx-arm64.zip - ${{github.workspace}}/artifact/extractor-osx-x64.zip - ${{github.workspace}}/artifact/extractor.osx-x64.zip - ${{github.workspace}}/artifact/extractor-win-x64.zip - ${{github.workspace}}/artifact/publisher-linux-arm64.zip - ${{github.workspace}}/artifact/publisher-linux-musl-arm64.zip - ${{github.workspace}}/artifact/publisher-linux-musl-x64.zip - ${{github.workspace}}/artifact/publisher-linux-x64.zip - ${{github.workspace}}/artifact/publisher-osx-arm64.zip - ${{github.workspace}}/artifact/publisher-osx-x64.zip - ${{github.workspace}}/artifact/publisher.osx-x64.zip - ${{github.workspace}}/artifact/publisher-win-x64.zip - ${{github.workspace}}/artifact/Github.zip - ${{github.workspace}}/artifact/Azure_DevOps.zip + ${{github.workspace}}/extractor-linux-arm64/extractor-linux-arm64.zip + ${{github.workspace}}/extractor-linux-musl-arm64/extractor-linux-musl-arm64.zip + ${{github.workspace}}/extractor-linux-musl-x64/extractor-linux-musl-x64.zip + ${{github.workspace}}/extractor-linux-x64/extractor-linux-x64.zip + ${{github.workspace}}/extractor-osx-arm64/extractor-osx-arm64.zip + ${{github.workspace}}/extractor-osx-x64/extractor-osx-x64.zip + ${{github.workspace}}/extractor.osx-x64/extractor.osx-x64.zip + ${{github.workspace}}/extractor-win-x64/extractor-win-x64.zip + ${{github.workspace}}/publisher-linux-arm64/publisher-linux-arm64.zip + ${{github.workspace}}/publisher-linux-musl-arm64/publisher-linux-musl-arm64.zip + ${{github.workspace}}/publisher-linux-musl-x64./publisher-linux-musl-x64.zip + ${{github.workspace}}/publisher-linux-x64/publisher-linux-x64.zip + ${{github.workspace}}/publisher-osx-arm64/publisher-osx-arm64.zip + ${{github.workspace}}/publisher-osx-x64./publisher-osx-x64.zip + ${{github.workspace}}/publisher.osx-x64/publisher.osx-x64.zip + ${{github.workspace}}/publisher-win-x64/publisher-win-x64.zip + ${{github.workspace}}/github/Github.zip + ${{github.workspace}}/ado/Azure_DevOps.zip name: APIOps Toolkit for Azure APIM ${{ github.event.inputs.Release_Version }} tag_name: ${{ github.event.inputs.Release_Version }} generate_release_notes: true From 4fb9e8bb4db4acb8a69b7cc2d9736994377a53cf Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Thu, 2 May 2024 12:43:04 -0400 Subject: [PATCH 12/29] Remove extraneous dots --- .github/workflows/create_github_release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create_github_release.yaml b/.github/workflows/create_github_release.yaml index 546fa861..b1ac7b52 100644 --- a/.github/workflows/create_github_release.yaml +++ b/.github/workflows/create_github_release.yaml @@ -246,10 +246,10 @@ jobs: ${{github.workspace}}/extractor-win-x64/extractor-win-x64.zip ${{github.workspace}}/publisher-linux-arm64/publisher-linux-arm64.zip ${{github.workspace}}/publisher-linux-musl-arm64/publisher-linux-musl-arm64.zip - ${{github.workspace}}/publisher-linux-musl-x64./publisher-linux-musl-x64.zip + ${{github.workspace}}/publisher-linux-musl-x64/publisher-linux-musl-x64.zip ${{github.workspace}}/publisher-linux-x64/publisher-linux-x64.zip ${{github.workspace}}/publisher-osx-arm64/publisher-osx-arm64.zip - ${{github.workspace}}/publisher-osx-x64./publisher-osx-x64.zip + ${{github.workspace}}/publisher-osx-x64/publisher-osx-x64.zip ${{github.workspace}}/publisher.osx-x64/publisher.osx-x64.zip ${{github.workspace}}/publisher-win-x64/publisher-win-x64.zip ${{github.workspace}}/github/Github.zip From 2ac9645ff7cc97f2c53e735ab069f40d214cf9e1 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Thu, 2 May 2024 13:02:53 -0400 Subject: [PATCH 13/29] Updated ADO extractor YAML with correct ADO variables --- tools/azdo_pipelines/run-extractor.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/azdo_pipelines/run-extractor.yaml b/tools/azdo_pipelines/run-extractor.yaml index 48529fc0..d8899125 100644 --- a/tools/azdo_pipelines/run-extractor.yaml +++ b/tools/azdo_pipelines/run-extractor.yaml @@ -107,17 +107,17 @@ stages: } Write-Information "Downloading release..." - $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" - $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + $uri = "https://github.com/Azure/apiops/releases/download/$(apiops_release_version)/$releaseFileName" + $downloadFilePath = Join-Path ""$(Agent.TempDirectory)" $releaseFileName Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" Write-Information "Extracting release..." - $executableFolderPath = Join-Path "${{ runner.temp }}" "extractor" + $executableFolderPath = Join-Path ""$(Agent.TempDirectory)" "extractor" Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" $executableFilePath = Join-Path "$executableFolderPath" $executableFileName Write-Information "Setting file permissions..." - if ("${{ runner.os }}" -like "*linux*") + if ("$(Agent.OS)" -like "*linux*") { & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} From e2bbb039b4cf513031fb6fc6f3430b47e2134d5c Mon Sep 17 00:00:00 2001 From: Wael Kdouh Date: Thu, 2 May 2024 13:09:12 -0400 Subject: [PATCH 14/29] Fix path in ADO extractor YAML --- tools/azdo_pipelines/run-extractor.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/azdo_pipelines/run-extractor.yaml b/tools/azdo_pipelines/run-extractor.yaml index d8899125..6d055376 100644 --- a/tools/azdo_pipelines/run-extractor.yaml +++ b/tools/azdo_pipelines/run-extractor.yaml @@ -108,11 +108,11 @@ stages: Write-Information "Downloading release..." $uri = "https://github.com/Azure/apiops/releases/download/$(apiops_release_version)/$releaseFileName" - $downloadFilePath = Join-Path ""$(Agent.TempDirectory)" $releaseFileName + $downloadFilePath = Join-Path "$(Agent.TempDirectory)" $releaseFileName Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" Write-Information "Extracting release..." - $executableFolderPath = Join-Path ""$(Agent.TempDirectory)" "extractor" + $executableFolderPath = Join-Path "$(Agent.TempDirectory)" "extractor" Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" $executableFilePath = Join-Path "$executableFolderPath" $executableFileName From b041988d64d232740e593310ec026b2fc3c8dc90 Mon Sep 17 00:00:00 2001 From: Wael Kdouh Date: Thu, 2 May 2024 13:19:46 -0400 Subject: [PATCH 15/29] Update download and extraction paths in run-publisher-with-env.yaml --- tools/azdo_pipelines/run-publisher-with-env.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/azdo_pipelines/run-publisher-with-env.yaml b/tools/azdo_pipelines/run-publisher-with-env.yaml index 29d00d82..b188d4af 100644 --- a/tools/azdo_pipelines/run-publisher-with-env.yaml +++ b/tools/azdo_pipelines/run-publisher-with-env.yaml @@ -103,17 +103,17 @@ steps: } Write-Information "Downloading release..." - $uri = "https://github.com/Azure/apiops/releases/download/${{ env.apiops_release_version }}/$releaseFileName" - $downloadFilePath = Join-Path "${{ runner.temp }}" $releaseFileName + $uri = "https://github.com/Azure/apiops/releases/download/$(apiops_release_version)/$releaseFileName" + $downloadFilePath = Join-Path "$(Agent.TempDirectory)" $releaseFileName Invoke-WebRequest -Uri "$uri" -OutFile "$downloadFilePath" Write-Information "Extracting release..." - $executableFolderPath = Join-Path "${{ runner.temp }}" "publisher" + $executableFolderPath = Join-Path "$(Agent.TempDirectory)" "publisher" Expand-Archive -Path "$downloadFilePath" -DestinationPath "$executableFolderPath" $executableFilePath = Join-Path "$executableFolderPath" $executableFileName Write-Information "Setting file permissions..." - if ("${{ runner.os }}" -like "*linux*") + if ("$(Agent.OS)" -like "*linux*") { & chmod +x "$executableFilePath" if ($LASTEXITCODE -ne 0) { throw "Setting file permissions failed."} From 27af424f9c68076b522ef1153973d6e6d5f4e5c8 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Fri, 3 May 2024 16:33:35 -0400 Subject: [PATCH 16/29] Add test that extracts, then publishes to an empty instance --- tools/code/common.tests/Service.cs | 4 +- tools/code/integration.tests/Configuration.cs | 22 ++-- tools/code/integration.tests/Extractor.cs | 2 +- tools/code/integration.tests/Fixture.cs | 30 +++-- tools/code/integration.tests/Service.cs | 2 +- tools/code/integration.tests/Subscription.cs | 3 +- tools/code/integration.tests/Tests.cs | 105 +++++++++++++----- 7 files changed, 115 insertions(+), 53 deletions(-) diff --git a/tools/code/common.tests/Service.cs b/tools/code/common.tests/Service.cs index c05ef4a8..1dc49f8c 100644 --- a/tools/code/common.tests/Service.cs +++ b/tools/code/common.tests/Service.cs @@ -9,7 +9,6 @@ namespace common.tests; public record ServiceModel { - public required ManagementServiceName Name { get; init; } public required FrozenSet NamedValues { get; init; } public required FrozenSet Tags { get; init; } public required FrozenSet Gateways { get; init; } @@ -24,7 +23,7 @@ public record ServiceModel public required FrozenSet Subscriptions { get; init; } public required FrozenSet Apis { get; init; } - public static Gen Generate(ManagementServiceName name) => + public static Gen Generate() => from namedValues in NamedValueModel.GenerateSet() from tags in TagModel.GenerateSet() from versionSets in VersionSetModel.GenerateSet() @@ -50,7 +49,6 @@ from updatedSubscriptions in UpdateSubscriptions(originalSubscriptions, products select updatedSubscriptions select new ServiceModel { - Name = name, NamedValues = namedValues, Tags = tags, Gateways = gateways, diff --git a/tools/code/integration.tests/Configuration.cs b/tools/code/integration.tests/Configuration.cs index 501074f1..d508ad1e 100644 --- a/tools/code/integration.tests/Configuration.cs +++ b/tools/code/integration.tests/Configuration.cs @@ -8,7 +8,6 @@ using LanguageExt.UnsafeValueAccess; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; using Microsoft.IdentityModel.JsonWebTokens; using System; using System.Threading; @@ -45,11 +44,17 @@ internal static class Configuration private static readonly Lazy httpPipeline = new(() => GetHttpPipeline(TokenCredential, AzureEnvironment)); public static HttpPipeline HttpPipeline => httpPipeline.Value; - private static readonly Lazy serviceName = new(() => GetManagementServiceName(Default)); - public static ManagementServiceName ServiceName => serviceName.Value; + private static readonly Lazy firstManagementServiceName = new(() => GetFirstManagementServiceName(Default)); + public static ManagementServiceName FirstServiceName => firstManagementServiceName.Value; - private static readonly Lazy serviceUri = new(() => GetManagementServiceUri(ServiceName)); - public static ManagementServiceUri ServiceUri => serviceUri.Value; + private static readonly Lazy firstServiceUri = new(() => GetManagementServiceUri(FirstServiceName)); + public static ManagementServiceUri FirstServiceUri => firstServiceUri.Value; + + private static readonly Lazy secondManagementServiceName = new(() => GetSecondManagementServiceName(Default)); + public static ManagementServiceName SecondServiceName => secondManagementServiceName.Value; + + private static readonly Lazy secondServiceUri = new(() => GetManagementServiceUri(SecondServiceName)); + public static ManagementServiceUri SecondServiceUri => secondServiceUri.Value; private static IConfiguration CreateDefault() => new ConfigurationBuilder().AddEnvironmentVariables() @@ -156,6 +161,9 @@ private static async ValueTask GetBearerToken(TokenCredential tokenCrede return token.Token; } - private static ManagementServiceName GetManagementServiceName(IConfiguration configuration) => - ManagementServiceName.From(configuration.GetValue("TEST_API_MANAGEMENT_SERVICE_NAME")); + private static ManagementServiceName GetFirstManagementServiceName(IConfiguration configuration) => + ManagementServiceName.From(configuration.GetValue("FIRST_API_MANAGEMENT_SERVICE_NAME")); + + private static ManagementServiceName GetSecondManagementServiceName(IConfiguration configuration) => + ManagementServiceName.From(configuration.GetValue("SECOND_API_MANAGEMENT_SERVICE_NAME")); } \ No newline at end of file diff --git a/tools/code/integration.tests/Extractor.cs b/tools/code/integration.tests/Extractor.cs index a6caaf7b..78a3b8de 100644 --- a/tools/code/integration.tests/Extractor.cs +++ b/tools/code/integration.tests/Extractor.cs @@ -27,7 +27,7 @@ public static async ValueTask Run(ExtractorOptions options, ManagementServiceNam ["AZURE_SUBSCRIPTION_ID"] = subscriptionId, ["AZURE_RESOURCE_GROUP_NAME"] = resourceGroupName, ["AZURE_BEARER_TOKEN"] = bearerToken, - ["Logging:LogLevel:Default"] = "Information" + ["Logging:LogLevel:Default"] = "Trace" }; var optionsJson = options.ToJsonObject(); diff --git a/tools/code/integration.tests/Fixture.cs b/tools/code/integration.tests/Fixture.cs index 4926c00a..f7adc76c 100644 --- a/tools/code/integration.tests/Fixture.cs +++ b/tools/code/integration.tests/Fixture.cs @@ -12,6 +12,8 @@ namespace integration.tests; internal sealed record Fixture { + public required ManagementServiceName FirstServiceName { get; init; } + public required ManagementServiceName SecondServiceName { get; init; } public required ServiceModel ServiceModel { get; init; } public required ServiceModel PublishAllChangesModel { get; init; } public required ImmutableArray CommitModels { get; init; } @@ -20,15 +22,18 @@ internal sealed record Fixture public required PublisherOptions PublisherOptions { get; init; } public static Gen Generate(string managementServiceNamePrefix) => - from serviceName in GenerateManagementServiceName(managementServiceNamePrefix) - from serviceModel in ServiceModel.Generate(serviceName) + from firstServiceName in GenerateManagementServiceName(managementServiceNamePrefix) + from secondServiceName in GenerateManagementServiceName(managementServiceNamePrefix) + from serviceModel in ServiceModel.Generate() from publishAllChangesModel in GeneratePublishAllChangesModel(serviceModel) from commitModels in GenerateCommitModels(serviceModel) - let serviceDirectory = GetManagementServiceDirectory(serviceModel.Name) + from serviceDirectory in GetManagementServiceDirectory() from extractorOptions in ExtractorOptions.Generate(serviceModel) from publisherOptions in PublisherOptions.Generate(serviceModel) select new Fixture { + FirstServiceName = firstServiceName, + SecondServiceName = secondServiceName, ServiceModel = serviceModel, PublishAllChangesModel = publishAllChangesModel, CommitModels = commitModels, @@ -39,7 +44,7 @@ from publisherOptions in PublisherOptions.Generate(serviceModel) // We want the name to change between tests, even if the seed is the same. // This avoids soft-delete issues with APIM - private static Gen GenerateManagementServiceName(string prefix) => + public static Gen GenerateManagementServiceName(string prefix) => from lorem in Generator.Lorem let characters = lorem.Paragraphs(3) .Where(char.IsLetterOrDigit) @@ -49,12 +54,17 @@ from suffixCharacters in Gen.Shuffle(characters, 8) let name = $"{prefix}{new string(suffixCharacters)}" select ManagementServiceName.From(name); - private static ManagementServiceDirectory GetManagementServiceDirectory(ManagementServiceName serviceName) - { - var path = Path.Combine(Path.GetTempPath(), serviceName.ToString()); - var directoryInfo = new DirectoryInfo(path); - return ManagementServiceDirectory.From(directoryInfo); - } + private static Gen GetManagementServiceDirectory() => + from lorem in Generator.Lorem + let characters = lorem.Paragraphs(3) + .Where(char.IsLetterOrDigit) + .Select(char.ToLowerInvariant) + .ToArray() + from suffixCharacters in Gen.Shuffle(characters, 8) + let name = $"apiops-{new string(suffixCharacters)}" + let path = Path.Combine(Path.GetTempPath(), name) + let directoryInfo = new DirectoryInfo(path) + select ManagementServiceDirectory.From(directoryInfo); private static Gen GeneratePublishAllChangesModel(ServiceModel originalModel) { diff --git a/tools/code/integration.tests/Service.cs b/tools/code/integration.tests/Service.cs index 34e437c5..d492a829 100644 --- a/tools/code/integration.tests/Service.cs +++ b/tools/code/integration.tests/Service.cs @@ -37,7 +37,7 @@ await pipeline.ListJsonObjects(serviceProviderUri, cancellationToken) public static async ValueTask DeleteManagementService(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => await pipeline.DeleteResource(serviceUri.ToUri(), waitForCompletion: true, cancellationToken); - public static async ValueTask CreateManagementService(ServiceModel model, ManagementServiceUri serviceUri, string location, HttpPipeline pipeline, CancellationToken cancellationToken) + public static async ValueTask CreateManagementService(ManagementServiceUri serviceUri, string location, HttpPipeline pipeline, CancellationToken cancellationToken) { var body = BinaryData.FromObjectAsJson(new { diff --git a/tools/code/integration.tests/Subscription.cs b/tools/code/integration.tests/Subscription.cs index 17a3f727..36011d15 100644 --- a/tools/code/integration.tests/Subscription.cs +++ b/tools/code/integration.tests/Subscription.cs @@ -136,7 +136,8 @@ private static async ValueTask ValidatePublisherChanges(IDictionary name != SubscriptionName.From("master")); actual.Should().BeEquivalentTo(expected); } diff --git a/tools/code/integration.tests/Tests.cs b/tools/code/integration.tests/Tests.cs index 35135b4c..68cbc6ce 100644 --- a/tools/code/integration.tests/Tests.cs +++ b/tools/code/integration.tests/Tests.cs @@ -1,4 +1,5 @@ -using common.tests; +using common; +using common.tests; using CsCheck; using DotNext.Collections.Generic; using FluentAssertions; @@ -41,10 +42,8 @@ public async Task Runs_as_expected() select useExistingInstance ? fixture with { - ServiceModel = fixture.ServiceModel with - { - Name = Configuration.ServiceName - } + FirstServiceName = Configuration.FirstServiceName, + SecondServiceName = Configuration.SecondServiceName } : fixture with { @@ -65,13 +64,17 @@ select useExistingInstance await generator.SampleAsync(async fixture => { // 1. Set up the management service - await CreateManagementService(fixture, cancellationToken); - await PutServiceModel(fixture, cancellationToken); + await CreateManagementServices(fixture, cancellationToken); DeleteServiceDirectory(fixture); - //// 2. Run the extractor and validate its artifacts + // 2. Run the extractor and validate its artifacts + await PutServiceModel(fixture, cancellationToken); await RunExtractor(fixture, cancellationToken); await ValidateExtractorArtifacts(fixture, cancellationToken); + + // 3. Publish the extracted changes to the empty second service + await PublishExtractedArtifactsToSecondService(fixture, cancellationToken); + await ValidatePublishedExtractedArtifacts(fixture, cancellationToken); await CleanUpExtractorResources(fixture, cancellationToken); // 3. Make changes to the extracted artifacts, publish the changes, then validate @@ -84,7 +87,7 @@ await generator.SampleAsync(async fixture => await TestCommits(fixture, commits, cancellationToken); await CleanUp(fixture, cancellationToken); - }, iter: 1, threads: useExistingInstance ? 1 : -1); + }, iter: 1, seed: "00008284ypj1", threads: useExistingInstance ? 1 : -1); } private static async ValueTask WriteProgress(string message) => @@ -99,59 +102,101 @@ private static async ValueTask OneTimeSetup(CancellationToken cancellationToken) await ServiceModule.DeleteManagementServices(serviceProviderUri, serviceNamePrefix, pipeline, cancellationToken); } - private static async ValueTask CreateManagementService(Fixture fixture, CancellationToken cancellationToken) + private static async ValueTask CreateManagementServices(Fixture fixture, CancellationToken cancellationToken) { - await WriteProgress("Creating management service..."); + await WriteProgress("Creating management services..."); - var serviceUri = Configuration.GetManagementServiceUri(fixture.ServiceModel.Name); + var firstServiceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); + var secondServiceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); if (useExistingInstance) { - await ServiceModule.DeleteAll(serviceUri, Configuration.HttpPipeline, cancellationToken); - + await Task.WhenAll(ServiceModule.DeleteAll(firstServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask(), + ServiceModule.DeleteAll(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask()); } else { - await ServiceModule.CreateManagementService(fixture.ServiceModel, serviceUri, Configuration.Location, Configuration.HttpPipeline, cancellationToken); + await Task.WhenAll(ServiceModule.CreateManagementService(firstServiceUri, Configuration.Location, Configuration.HttpPipeline, cancellationToken).AsTask(), + ServiceModule.CreateManagementService(secondServiceUri, Configuration.Location, Configuration.HttpPipeline, cancellationToken).AsTask()); } } + private static void DeleteServiceDirectory(Fixture fixture) + { + ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); + } + private static async ValueTask PutServiceModel(Fixture fixture, CancellationToken cancellationToken) { await WriteProgress("Putting service model..."); - var serviceUri = Configuration.GetManagementServiceUri(fixture.ServiceModel.Name); + var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); await ServiceModule.Put(fixture.ServiceModel, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); } - private static void DeleteServiceDirectory(Fixture fixture) - { - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - } - private static async ValueTask RunExtractor(Fixture fixture, CancellationToken cancellationToken) { await WriteProgress("Running extractor..."); var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Extractor.Run(fixture.ExtractorOptions, fixture.ServiceModel.Name, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, cancellationToken); + await Extractor.Run(fixture.ExtractorOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, cancellationToken); } private static async ValueTask ValidateExtractorArtifacts(Fixture fixture, CancellationToken cancellationToken) { await WriteProgress("Validating extractor artifacts..."); - var serviceUri = Configuration.GetManagementServiceUri(fixture.ServiceModel.Name); + var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); await ServiceModule.ValidateExtractedArtifacts(fixture.ExtractorOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); } + private static async ValueTask PublishExtractedArtifactsToSecondService(Fixture fixture, CancellationToken cancellationToken) + { + await WriteProgress("Extracting all artifacts from first instance..."); + ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); + + var bearerToken = await Configuration.GetBearerToken(cancellationToken); + var extractorOptions = fixture.ExtractorOptions with + { + ApiNamesToExport = Option>.None, + ProductNamesToExport = Option>.None, + GroupNamesToExport = Option>.None, + SubscriptionNamesToExport = Option>.None, + BackendNamesToExport = Option>.None, + LoggerNamesToExport = Option>.None, + DiagnosticNamesToExport = Option>.None, + PolicyFragmentNamesToExport = Option>.None, + GatewayNamesToExport = Option>.None, + TagNamesToExport = Option>.None, + VersionSetNamesToExport = Option>.None, + NamedValueNamesToExport = Option>.None, + }; + await Extractor.Run(extractorOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, cancellationToken); + + await WriteProgress("Publishing extracted artifacts to second instance..."); + await Publisher.Run(fixture.PublisherOptions, fixture.SecondServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId: Option.None, cancellationToken); + } + + private static async ValueTask ValidatePublishedExtractedArtifacts(Fixture fixture, CancellationToken cancellationToken) + { + await WriteProgress("Validating published extracted artifacts..."); + + var serviceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); + await ServiceModule.ValidatePublisherChanges(fixture.PublisherOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); + } + private static async ValueTask CleanUpExtractorResources(Fixture fixture, CancellationToken cancellationToken) { await WriteProgress("Cleaning up extractor resources..."); - var serviceUri = Configuration.GetManagementServiceUri(fixture.ServiceModel.Name); - await ServiceModule.DeleteAll(serviceUri, Configuration.HttpPipeline, cancellationToken); ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); + + var firstServiceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); + var secondServiceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); + await Task.WhenAll(ServiceModule.DeleteAll(firstServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask(), + useExistingInstance + ? ServiceModule.DeleteAll(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask() + : ServiceModule.DeleteManagementService(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask()); } private static async ValueTask WriteFirstChange(Fixture fixture, CancellationToken cancellationToken) @@ -167,14 +212,14 @@ private static async ValueTask PublishFirstChange(Fixture fixture, CancellationT await WriteProgress("Publishing first change..."); var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Publisher.Run(fixture.PublisherOptions, fixture.ServiceModel.Name, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId: Option.None, cancellationToken); + await Publisher.Run(fixture.PublisherOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId: Option.None, cancellationToken); } private static async ValueTask ValidatePublishedFirstChange(Fixture fixture, CancellationToken cancellationToken) { await WriteProgress("Validating published first change..."); - var serviceUri = Configuration.GetManagementServiceUri(fixture.ServiceModel.Name); + var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); await ServiceModule.ValidatePublisherChanges(fixture.PublisherOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); } @@ -198,14 +243,14 @@ private static async ValueTask PublishCommit(Fixture fixture, CommitId commitId, await WriteProgress($"Publishing commit {commitId}..."); var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Publisher.Run(fixture.PublisherOptions, fixture.ServiceModel.Name, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId, cancellationToken); + await Publisher.Run(fixture.PublisherOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId, cancellationToken); } private static async ValueTask ValidatePublishedCommit(Fixture fixture, CommitId commitId, CancellationToken cancellationToken) { await WriteProgress($"Validating published commit {commitId}..."); - var serviceUri = Configuration.GetManagementServiceUri(fixture.ServiceModel.Name); + var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); await ServiceModule.ValidatePublisherCommitChanges(fixture.PublisherOptions, commitId, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); } @@ -213,7 +258,7 @@ private static async ValueTask CleanUp(Fixture fixture, CancellationToken cancel { await WriteProgress("Cleaning up..."); - var serviceUri = Configuration.GetManagementServiceUri(fixture.ServiceModel.Name); + var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); if (useExistingInstance) { From 9ae70ef64553a7fdd415f23b3adbedd597e52fe7 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Fri, 3 May 2024 16:34:23 -0400 Subject: [PATCH 17/29] Only get current versions in API associations --- tools/code/common/GatewayApi.cs | 3 ++- tools/code/common/ProductApi.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/code/common/GatewayApi.cs b/tools/code/common/GatewayApi.cs index 48dbaf3b..3440eb33 100644 --- a/tools/code/common/GatewayApi.cs +++ b/tools/code/common/GatewayApi.cs @@ -125,7 +125,8 @@ public static class GatewayApiModule public static IAsyncEnumerable ListNames(this GatewayApisUri uri, HttpPipeline pipeline, CancellationToken cancellationToken) => pipeline.ListJsonObjects(uri.ToUri(), cancellationToken) .Select(jsonObject => jsonObject.GetStringProperty("name")) - .Select(ApiName.From); + .Select(ApiName.From) + .Where(ApiName.IsNotRevisioned); public static IAsyncEnumerable<(ApiName Name, GatewayApiDto Dto)> List(this GatewayApisUri gatewayApisUri, HttpPipeline pipeline, CancellationToken cancellationToken) => gatewayApisUri.ListNames(pipeline, cancellationToken) diff --git a/tools/code/common/ProductApi.cs b/tools/code/common/ProductApi.cs index 1d2ed0f7..e147c999 100644 --- a/tools/code/common/ProductApi.cs +++ b/tools/code/common/ProductApi.cs @@ -125,7 +125,8 @@ public static class ProductApiModule public static IAsyncEnumerable ListNames(this ProductApisUri uri, HttpPipeline pipeline, CancellationToken cancellationToken) => pipeline.ListJsonObjects(uri.ToUri(), cancellationToken) .Select(jsonObject => jsonObject.GetStringProperty("name")) - .Select(ApiName.From); + .Select(ApiName.From) + .Where(ApiName.IsNotRevisioned); public static IAsyncEnumerable<(ApiName Name, ProductApiDto Dto)> List(this ProductApisUri productApisUri, HttpPipeline pipeline, CancellationToken cancellationToken) => productApisUri.ListNames(pipeline, cancellationToken) From 6ed4bd1b829258edbdbcc3191a7ce070d7cc7320 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Mon, 6 May 2024 11:29:05 -0400 Subject: [PATCH 18/29] API updates --- tools/code/common/Api.cs | 61 ++++++++++++++++++++------- tools/code/common/Json.cs | 4 +- tools/code/integration.tests/Api.cs | 11 +++-- tools/code/integration.tests/Tests.cs | 2 +- tools/code/publisher/Api.cs | 21 +++++++-- 5 files changed, 73 insertions(+), 26 deletions(-) diff --git a/tools/code/common/Api.cs b/tools/code/common/Api.cs index 77f1738f..4e9447af 100644 --- a/tools/code/common/Api.cs +++ b/tools/code/common/Api.cs @@ -456,11 +456,33 @@ public static async ValueTask> TryGetSpecificationContents(th return await apiUri.TryGetGraphQlSchema(pipeline, cancellationToken); } - var exportUri = GetExportUri(apiUri, specification); - var downloadUri = await GetSpecificationDownloadUri(exportUri, pipeline, cancellationToken); + BinaryData? content; + try + { + var exportUri = GetExportUri(apiUri, specification, includeLink: true); + var downloadUri = await GetSpecificationDownloadUri(exportUri, pipeline, cancellationToken); + + var nonAuthenticatedHttpPipeline = HttpPipelineBuilder.Build(ClientOptions.Default); + content = await nonAuthenticatedHttpPipeline.GetContent(downloadUri, cancellationToken); + } + // If we can't download the specification through the download link, get it directly. + catch (HttpRequestException exception) when (exception.StatusCode == HttpStatusCode.InternalServerError) + { + // Don't export XML specifications, as the non-link exports cannot be reimported. + if (specification is ApiSpecification.Wsdl or ApiSpecification.Wadl) + { + return Option.None; + } - var nonAuthenticatedHttpPipeline = HttpPipelineBuilder.Build(ClientOptions.Default); - var content = await nonAuthenticatedHttpPipeline.GetContent(downloadUri, cancellationToken); + var exportUri = GetExportUri(apiUri, specification, includeLink: false); + var json = await pipeline.GetJsonObject(exportUri, cancellationToken); + var contentString = json.GetProperty("value") switch + { + JsonValue jsonValue => jsonValue.ToString(), + var node => node.ToJsonString(JsonObjectExtensions.SerializerOptions) + }; + content = BinaryData.FromString(contentString); + } // APIM exports OpenApiV2 to JSON. Convert to YAML if needed. if (specification is ApiSpecification.OpenApi openApi && openApi.Format is OpenApiFormat.Yaml && openApi.Version is OpenApiVersion.V2) @@ -472,28 +494,35 @@ public static async ValueTask> TryGetSpecificationContents(th return content; } - private static Uri GetExportUri(ApiUri apiUri, ApiSpecification specification) + private static Uri GetExportUri(ApiUri apiUri, ApiSpecification specification, bool includeLink) { - var format = specification switch + var format = GetExportFormat(specification, includeLink); + + return apiUri.ToUri() + .SetQueryParam("format", format) + .SetQueryParam("export", "true") + .SetQueryParam("api-version", "2022-09-01-preview") + .ToUri(); + } + + private static string GetExportFormat(ApiSpecification specification, bool includeLink) + { + var formatWithoutLink = specification switch { - ApiSpecification.Wadl => "wadl-link", - ApiSpecification.Wsdl => "wsdl-link", + ApiSpecification.Wadl => "wadl", + ApiSpecification.Wsdl => "wsdl", ApiSpecification.OpenApi openApiSpecification => (openApiSpecification.Version, openApiSpecification.Format) switch { - (OpenApiVersion.V2, _) => "swagger-link", - (OpenApiVersion.V3, OpenApiFormat.Yaml) => "openapi-link", - (OpenApiVersion.V3, OpenApiFormat.Json) => "openapi+json-link", + (OpenApiVersion.V2, _) => "swagger", + (OpenApiVersion.V3, OpenApiFormat.Yaml) => "openapi", + (OpenApiVersion.V3, OpenApiFormat.Json) => "openapi+json", _ => throw new NotSupportedException() }, _ => throw new NotSupportedException() }; - return apiUri.ToUri() - .SetQueryParam("format", format) - .SetQueryParam("export", "true") - .SetQueryParam("api-version", "2022-09-01-preview") - .ToUri(); + return includeLink ? $"{formatWithoutLink}-link" : formatWithoutLink; } private static async ValueTask GetSpecificationDownloadUri(Uri exportUri, HttpPipeline pipeline, CancellationToken cancellationToken) diff --git a/tools/code/common/Json.cs b/tools/code/common/Json.cs index 1ad33dfe..f75d8aaa 100644 --- a/tools/code/common/Json.cs +++ b/tools/code/common/Json.cs @@ -338,8 +338,8 @@ public static Option TryAsGuid(this JsonValue? jsonValue) => jsonValue is not null && jsonValue.TryGetValue(out var guid) ? guid : jsonValue.TryAsString() - .Bind(x => Guid.TryParse(x, out var guidString) - ? guidString + .Bind(x => Guid.TryParse(x, out var guidFromString) + ? guidFromString : Option.None); public static Option TryAsAbsoluteUri(this JsonValue? jsonValue) => diff --git a/tools/code/integration.tests/Api.cs b/tools/code/integration.tests/Api.cs index bb14fb38..56c110d9 100644 --- a/tools/code/integration.tests/Api.cs +++ b/tools/code/integration.tests/Api.cs @@ -326,9 +326,10 @@ private static string NormalizeDto(ApiDto dto) => Path = dto.Properties.Path ?? string.Empty, RevisionDescription = dto.Properties.ApiRevisionDescription ?? string.Empty, Revision = dto.Properties.ApiRevision ?? string.Empty, - ServiceUrl = Uri.TryCreate(dto.Properties.ServiceUrl, UriKind.Absolute, out var uri) - ? uri.RemovePath().ToString() - : string.Empty + // Disabling this check because there are too many edge cases //TODO - Investigate + //ServiceUrl = Uri.TryCreate(dto.Properties.ServiceUrl, UriKind.Absolute, out var uri) + // ? uri.RemovePath().ToString() + // : string.Empty }.ToString()!; private static async ValueTask ValidateExtractedSpecificationFiles(IDictionary expectedResources, Option defaultApiSpecification, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) @@ -338,12 +339,16 @@ private static async ValueTask ValidateExtractedSpecificationFiles(IDictionary (file.Parent.Name, file.Specification)) + // Skip XML specification files. Sometimes they get extracted, other times they fail. + .Where(file => file.Specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl)) .ToFrozenDictionary(cancellationToken); actual.Should().BeEquivalentTo(expected); diff --git a/tools/code/integration.tests/Tests.cs b/tools/code/integration.tests/Tests.cs index 68cbc6ce..24973cd5 100644 --- a/tools/code/integration.tests/Tests.cs +++ b/tools/code/integration.tests/Tests.cs @@ -87,7 +87,7 @@ await generator.SampleAsync(async fixture => await TestCommits(fixture, commits, cancellationToken); await CleanUp(fixture, cancellationToken); - }, iter: 1, seed: "00008284ypj1", threads: useExistingInstance ? 1 : -1); + }, iter: 1, threads: useExistingInstance ? 1 : -1); } private static async ValueTask WriteProgress(string message) => diff --git a/tools/code/publisher/Api.cs b/tools/code/publisher/Api.cs index 3f97e40f..1d5f431a 100644 --- a/tools/code/publisher/Api.cs +++ b/tools/code/publisher/Api.cs @@ -181,8 +181,8 @@ private async ValueTask Put(ApiName name, ApiDto dto, CancellationToken cancella { // Put prerequisites await PutVersionSet(dto, cancellationToken); + await PutCurrentRevision(name, dto, cancellationToken); - await correctApimRevisionNumber(name, dto, cancellationToken); await putInApim(name, dto, cancellationToken); } @@ -190,6 +190,19 @@ private async ValueTask PutVersionSet(ApiDto dto, CancellationToken cancellation await Common.TryGetVersionSetName(dto) .IterTask(putVersionSet.Invoke, cancellationToken); + private async ValueTask PutCurrentRevision(ApiName name, ApiDto dto, CancellationToken cancellationToken) + { + if (ApiName.IsRevisioned(name)) + { + var rootName = ApiName.GetRootName(name); + await Handle(rootName, cancellationToken); + } + else + { + await correctApimRevisionNumber(name, dto, cancellationToken); + } + } + public void Dispose() => semaphore.Dispose(); } @@ -501,9 +514,9 @@ public async ValueTask Handle(ApiName name, ApiRevisionNumber revisionNumber, Ca } file sealed class DeleteApiHandler(IEnumerable onDeletingHandlers, - ILoggerFactory loggerFactory, - FindApiDto findDto, - DeleteApiFromApim deleteFromApim) : IDisposable + ILoggerFactory loggerFactory, + FindApiDto findDto, + DeleteApiFromApim deleteFromApim) : IDisposable { private readonly ILogger logger = Common.GetLogger(loggerFactory); private readonly ApiSemaphore semaphore = new(); From 26816f51226d77f7b3576ddad7b578b857acc082 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Tue, 7 May 2024 12:43:20 -0400 Subject: [PATCH 19/29] Remove AllowTracing from subscription test --- tools/code/common.tests/Subscription.cs | 5 +---- tools/code/integration.tests/Service.cs | 2 +- tools/code/integration.tests/Subscription.cs | 13 ++++--------- tools/code/integration.tests/Tests.cs | 2 +- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/tools/code/common.tests/Subscription.cs b/tools/code/common.tests/Subscription.cs index fe202add..aacd5c32 100644 --- a/tools/code/common.tests/Subscription.cs +++ b/tools/code/common.tests/Subscription.cs @@ -36,18 +36,15 @@ public sealed record SubscriptionModel public required SubscriptionName Name { get; init; } public required string DisplayName { get; init; } public required SubscriptionScope Scope { get; init; } - public Option AllowTracing { get; init; } public static Gen Generate() => from name in GenerateName() from scope in SubscriptionScope.Generate() - from allowTracing in Gen.Bool.OptionOf() select new SubscriptionModel { Name = name, DisplayName = name.ToString(), - Scope = scope, - AllowTracing = allowTracing + Scope = scope }; public static Gen GenerateName() => diff --git a/tools/code/integration.tests/Service.cs b/tools/code/integration.tests/Service.cs index d492a829..72a036e1 100644 --- a/tools/code/integration.tests/Service.cs +++ b/tools/code/integration.tests/Service.cs @@ -35,7 +35,7 @@ await pipeline.ListJsonObjects(serviceProviderUri, cancellationToken) } public static async ValueTask DeleteManagementService(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await pipeline.DeleteResource(serviceUri.ToUri(), waitForCompletion: true, cancellationToken); + await pipeline.DeleteResource(serviceUri.ToUri(), waitForCompletion: false, cancellationToken); public static async ValueTask CreateManagementService(ManagementServiceUri serviceUri, string location, HttpPipeline pipeline, CancellationToken cancellationToken) { diff --git a/tools/code/integration.tests/Subscription.cs b/tools/code/integration.tests/Subscription.cs index 36011d15..5a4f22d5 100644 --- a/tools/code/integration.tests/Subscription.cs +++ b/tools/code/integration.tests/Subscription.cs @@ -22,19 +22,16 @@ from displayName in SubscriptionModel.GenerateDisplayName() from allowTracing in Gen.Bool.OptionOf() select original with { - DisplayName = displayName, - AllowTracing = allowTracing + DisplayName = displayName }; public static Gen GenerateOverride(SubscriptionDto original) => from displayName in SubscriptionModel.GenerateDisplayName() - from allowTracing in Gen.Bool.OptionOf() select new SubscriptionDto { Properties = new SubscriptionDto.SubscriptionContract { - DisplayName = displayName, - AllowTracing = allowTracing.ValueUnsafe() + DisplayName = displayName } }; @@ -52,8 +49,7 @@ private static SubscriptionDto GetDto(SubscriptionModel model) => SubscriptionScope.Product product => $"/products/{product.Name}", SubscriptionScope.Api api => $"/apis/{api.Name}", _ => throw new InvalidOperationException($"Scope {model.Scope} not supported.") - }, - AllowTracing = model.AllowTracing.ValueUnsafe() + } } }; @@ -120,8 +116,7 @@ private static string NormalizeDto(SubscriptionDto dto) => new { DisplayName = dto.Properties.DisplayName ?? string.Empty, - Scope = string.Join('/', dto.Properties.Scope?.Split('/')?.TakeLast(2)?.ToArray() ?? []), - AllowTracing = dto.Properties.AllowTracing ?? false + Scope = string.Join('/', dto.Properties.Scope?.Split('/')?.TakeLast(2)?.ToArray() ?? []) }.ToString()!; public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) diff --git a/tools/code/integration.tests/Tests.cs b/tools/code/integration.tests/Tests.cs index 24973cd5..018a282c 100644 --- a/tools/code/integration.tests/Tests.cs +++ b/tools/code/integration.tests/Tests.cs @@ -87,7 +87,7 @@ await generator.SampleAsync(async fixture => await TestCommits(fixture, commits, cancellationToken); await CleanUp(fixture, cancellationToken); - }, iter: 1, threads: useExistingInstance ? 1 : -1); + }, iter: 1, seed: "0000KOIPe036", threads: useExistingInstance ? 1 : -1); } private static async ValueTask WriteProgress(string message) => From 52c211586633f86bc9af0425b2a4a99a06b13877 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 8 May 2024 11:03:44 -0400 Subject: [PATCH 20/29] Extend extraction check to related resources --- tools/code/common/Api.cs | 5 ++++ tools/code/common/Diagnostic.cs | 6 ++++- tools/code/common/Subscription.cs | 12 +++++++++ tools/code/extractor/Api.cs | 39 ++++++++++++++++++++++------ tools/code/extractor/Diagnostic.cs | 20 ++++++++------ tools/code/extractor/GatewayApi.cs | 5 +++- tools/code/extractor/Logger.cs | 4 +-- tools/code/extractor/Product.cs | 4 +-- tools/code/extractor/ProductApi.cs | 5 +++- tools/code/extractor/Subscription.cs | 33 ++++++++++++++--------- tools/code/extractor/VersionSet.cs | 4 +-- tools/code/publisher/Api.cs | 11 +++----- tools/code/publisher/Diagnostic.cs | 11 +++----- tools/code/publisher/Subscription.cs | 24 +++++------------ 14 files changed, 112 insertions(+), 71 deletions(-) diff --git a/tools/code/common/Api.cs b/tools/code/common/Api.cs index 4e9447af..6cc84026 100644 --- a/tools/code/common/Api.cs +++ b/tools/code/common/Api.cs @@ -762,4 +762,9 @@ public static FileInfo ToFileInfo(this ApiSpecificationFile file) => public static async ValueTask ReadContents(this ApiSpecificationFile file, CancellationToken cancellationToken) => await file.ToFileInfo().ReadAsBinaryData(cancellationToken); + + public static Option TryGetVersionSetName(ApiDto dto) => + from versionSetId in Prelude.Optional(dto.Properties.ApiVersionSetId) + from versionSetNameString in versionSetId.Split('/').LastOrNone() + select VersionSetName.From(versionSetNameString); } \ No newline at end of file diff --git a/tools/code/common/Diagnostic.cs b/tools/code/common/Diagnostic.cs index 65bc90cc..a1cffd54 100644 --- a/tools/code/common/Diagnostic.cs +++ b/tools/code/common/Diagnostic.cs @@ -7,7 +7,6 @@ using System.IO; using System.Linq; using System.Net; -using System.Text.Json.Nodes; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; @@ -308,4 +307,9 @@ public static async ValueTask ReadDto(this DiagnosticInformationF var content = await file.ToFileInfo().ReadAsBinaryData(cancellationToken); return content.ToObjectFromJson(); } + + public static Option TryGetLoggerName(DiagnosticDto dto) => + from loggerId in Prelude.Optional(dto.Properties.LoggerId) + from loggerNameString in loggerId.Split('/').LastOrNone() + select LoggerName.From(loggerNameString); } \ No newline at end of file diff --git a/tools/code/common/Subscription.cs b/tools/code/common/Subscription.cs index 5c8706f7..cd071721 100644 --- a/tools/code/common/Subscription.cs +++ b/tools/code/common/Subscription.cs @@ -228,4 +228,16 @@ public static async ValueTask ReadDto(this SubscriptionInformat var content = await file.ToFileInfo().ReadAsBinaryData(cancellationToken); return content.ToObjectFromJson(); } + + public static Option TryGetApiName(SubscriptionDto dto) => + from scope in Prelude.Optional(dto.Properties.Scope) + where scope.Contains("/apis/", StringComparison.OrdinalIgnoreCase) + from apiNameString in scope.Split('/').LastOrNone() + select ApiName.From(apiNameString); + + public static Option TryGetProductName(SubscriptionDto dto) => + from scope in Prelude.Optional(dto.Properties.Scope) + where scope.Contains("/products/", StringComparison.OrdinalIgnoreCase) + from productNameString in scope.Split('/').LastOrNone() + select ProductName.From(productNameString); } \ No newline at end of file diff --git a/tools/code/extractor/Api.cs b/tools/code/extractor/Api.cs index c711c768..0e5dbd56 100644 --- a/tools/code/extractor/Api.cs +++ b/tools/code/extractor/Api.cs @@ -17,7 +17,9 @@ namespace extractor; file delegate IAsyncEnumerable<(ApiName Name, ApiDto Dto, Option<(ApiSpecification Specification, BinaryData Contents)> SpecificationOption)> ListApis(CancellationToken cancellationToken); -file delegate bool ShouldExtractApi(ApiName name); +internal delegate bool ShouldExtractApiName(ApiName name); + +file delegate bool ShouldExtractApiDto(ApiDto dto); file delegate ValueTask WriteApiArtifacts(ApiName name, ApiDto dto, Option<(ApiSpecification Specification, BinaryData Contents)> specificationOption, CancellationToken cancellationToken); @@ -26,7 +28,8 @@ namespace extractor; file delegate ValueTask WriteApiSpecificationFile(ApiName name, ApiSpecification specification, BinaryData contents, CancellationToken cancellationToken); file sealed class ExtractApisHandler(ListApis list, - ShouldExtractApi shouldExtract, + ShouldExtractApiName shouldExtractName, + ShouldExtractApiDto shouldExtractDto, WriteApiArtifacts writeArtifacts, ExtractApiPolicies extractApiPolicies, ExtractApiTags extractApiTags, @@ -34,7 +37,8 @@ file sealed class ExtractApisHandler(ListApis list, { public async ValueTask Handle(CancellationToken cancellationToken) => await list(cancellationToken) - .Where(api => shouldExtract(api.Name)) + .Where(api => shouldExtractName(api.Name)) + .Where(api => shouldExtractDto(api.Dto)) // Group APIs by version set (https://github.com/Azure/apiops/issues/316). // We'll process each group in parallel, but each API within a group sequentially. .GroupBy(api => api.Dto.Properties.ApiVersionSetId ?? Guid.NewGuid().ToString()) @@ -136,7 +140,7 @@ private Option TryGetSpecification(ApiDto dto) => }; } -file sealed class ShouldExtractApiHandler(ShouldExtractFactory shouldExtractFactory) +file sealed class ShouldExtractApiNameHandler(ShouldExtractFactory shouldExtractFactory) { public bool Handle(ApiName name) { @@ -145,6 +149,15 @@ public bool Handle(ApiName name) } } +file sealed class ShouldExtractApiDtoHandler(ShouldExtractVersionSet shouldExtractVersionSet) +{ + public bool Handle(ApiDto dto) => + // Don't extract if its version set should not be extracted + ApiModule.TryGetVersionSetName(dto) + .Map(shouldExtractVersionSet.Invoke) + .IfNone(true); +} + file sealed class WriteApiArtifactsHandler(WriteApiInformationFile writeInformationFile, WriteApiSpecificationFile writeSpecificationFile) { @@ -191,7 +204,8 @@ internal static class ApiServices public static void ConfigureExtractApis(IServiceCollection services) { ConfigureListApis(services); - ConfigureShouldExtractApi(services); + ConfigureShouldExtractApiName(services); + ConfigureShouldExtractApiDto(services); ConfigureWriteApiArtifacts(services); ApiPolicyServices.ConfigureExtractApiPolicies(services); ApiTagServices.ConfigureExtractApiTags(services); @@ -200,16 +214,25 @@ public static void ConfigureExtractApis(IServiceCollection services) services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } + private static void ConfigureListApis(IServiceCollection services) { services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static void ConfigureShouldExtractApi(IServiceCollection services) + public static void ConfigureShouldExtractApiName(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureShouldExtractApiDto(IServiceCollection services) + { + VersionSetServices.ConfigureShouldExtractVersionSet(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } private static void ConfigureWriteApiArtifacts(IServiceCollection services) diff --git a/tools/code/extractor/Diagnostic.cs b/tools/code/extractor/Diagnostic.cs index 2002d1a9..9d82184c 100644 --- a/tools/code/extractor/Diagnostic.cs +++ b/tools/code/extractor/Diagnostic.cs @@ -15,7 +15,7 @@ namespace extractor; file delegate IAsyncEnumerable<(DiagnosticName Name, DiagnosticDto Dto)> ListDiagnostics(CancellationToken cancellationToken); -file delegate bool ShouldExtractDiagnostic(DiagnosticName name); +file delegate bool ShouldExtractDiagnostic(DiagnosticName name, DiagnosticDto dto); file delegate ValueTask WriteDiagnosticArtifacts(DiagnosticName name, DiagnosticDto dto, CancellationToken cancellationToken); @@ -25,7 +25,7 @@ file sealed class ExtractDiagnosticsHandler(ListDiagnostics list, ShouldExtractD { public async ValueTask Handle(CancellationToken cancellationToken) => await list(cancellationToken) - .Where(diagnostic => shouldExtract(diagnostic.Name)) + .Where(diagnostic => shouldExtract(diagnostic.Name, diagnostic.Dto)) .IterParallel(async diagnostic => await writeArtifacts(diagnostic.Name, diagnostic.Dto, cancellationToken), cancellationToken); } @@ -36,13 +36,15 @@ file sealed class ListDiagnosticsHandler(ManagementServiceUri serviceUri, HttpPi DiagnosticsUri.From(serviceUri).List(pipeline, cancellationToken); } -file sealed class ShouldExtractDiagnosticHandler(ShouldExtractFactory shouldExtractFactory) +file sealed class ShouldExtractDiagnosticHandler(ShouldExtractFactory shouldExtractFactory, ShouldExtractLogger shouldExtractLogger) { - public bool Handle(DiagnosticName name) - { - var shouldExtract = shouldExtractFactory.Create(); - return shouldExtract(name); - } + public bool Handle(DiagnosticName name, DiagnosticDto dto) => + // Check name from configuration override + shouldExtractFactory.Create().Invoke(name) + // Don't extract diagnostic if its logger should not be extracted + && DiagnosticModule.TryGetLoggerName(dto) + .Map(shouldExtractLogger.Invoke) + .IfNone(true); } file sealed class WriteDiagnosticArtifactsHandler(WriteDiagnosticInformationFile writeInformationFile) @@ -86,6 +88,8 @@ private static void ConfigureListDiagnostics(IServiceCollection services) private static void ConfigureShouldExtractDiagnostic(IServiceCollection services) { + LoggerServices.ConfigureShouldExtractLogger(services); + services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } diff --git a/tools/code/extractor/GatewayApi.cs b/tools/code/extractor/GatewayApi.cs index 7e7ebe70..bc3ebabd 100644 --- a/tools/code/extractor/GatewayApi.cs +++ b/tools/code/extractor/GatewayApi.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -17,10 +18,11 @@ namespace extractor; file delegate ValueTask WriteGatewayApiInformationFile(ApiName name, GatewayApiDto dto, GatewayName gatewayName, CancellationToken cancellationToken); -file sealed class ExtractGatewayApisHandler(ListGatewayApis list, WriteGatewayApiArtifacts writeArtifacts) +file sealed class ExtractGatewayApisHandler(ListGatewayApis list, ShouldExtractApiName shouldExtractApi, WriteGatewayApiArtifacts writeArtifacts) { public async ValueTask Handle(GatewayName gatewayName, CancellationToken cancellationToken) => await list(gatewayName, cancellationToken) + .Where(api => shouldExtractApi(api.Name)) .IterParallel(async gatewayapi => await writeArtifacts(gatewayapi.Name, gatewayapi.Dto, gatewayName, cancellationToken), cancellationToken); } @@ -57,6 +59,7 @@ internal static class GatewayApiServices public static void ConfigureExtractGatewayApis(IServiceCollection services) { ConfigureListGatewayApis(services); + ApiServices.ConfigureShouldExtractApiName(services); ConfigureWriteGatewayApiArtifacts(services); services.TryAddSingleton(); diff --git a/tools/code/extractor/Logger.cs b/tools/code/extractor/Logger.cs index eff86398..ac879091 100644 --- a/tools/code/extractor/Logger.cs +++ b/tools/code/extractor/Logger.cs @@ -15,7 +15,7 @@ namespace extractor; file delegate IAsyncEnumerable<(LoggerName Name, LoggerDto Dto)> ListLoggers(CancellationToken cancellationToken); -file delegate bool ShouldExtractLogger(LoggerName name); +internal delegate bool ShouldExtractLogger(LoggerName name); file delegate ValueTask WriteLoggerArtifacts(LoggerName name, LoggerDto dto, CancellationToken cancellationToken); @@ -84,7 +84,7 @@ private static void ConfigureListLoggers(IServiceCollection services) services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static void ConfigureShouldExtractLogger(IServiceCollection services) + public static void ConfigureShouldExtractLogger(IServiceCollection services) { services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); diff --git a/tools/code/extractor/Product.cs b/tools/code/extractor/Product.cs index b0deccee..852d341d 100644 --- a/tools/code/extractor/Product.cs +++ b/tools/code/extractor/Product.cs @@ -15,7 +15,7 @@ namespace extractor; file delegate IAsyncEnumerable<(ProductName Name, ProductDto Dto)> ListProducts(CancellationToken cancellationToken); -file delegate bool ShouldExtractProduct(ProductName name); +internal delegate bool ShouldExtractProduct(ProductName name); file delegate ValueTask WriteProductArtifacts(ProductName name, ProductDto dto, CancellationToken cancellationToken); @@ -103,7 +103,7 @@ private static void ConfigureListProducts(IServiceCollection services) services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static void ConfigureShouldExtractProduct(IServiceCollection services) + public static void ConfigureShouldExtractProduct(IServiceCollection services) { services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); diff --git a/tools/code/extractor/ProductApi.cs b/tools/code/extractor/ProductApi.cs index e355d7df..4f9bcb04 100644 --- a/tools/code/extractor/ProductApi.cs +++ b/tools/code/extractor/ProductApi.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -17,10 +18,11 @@ namespace extractor; file delegate ValueTask WriteProductApiInformationFile(ApiName name, ProductApiDto dto, ProductName productName, CancellationToken cancellationToken); -file sealed class ExtractProductApisHandler(ListProductApis list, WriteProductApiArtifacts writeArtifacts) +file sealed class ExtractProductApisHandler(ListProductApis list, ShouldExtractApiName shouldExtractApi, WriteProductApiArtifacts writeArtifacts) { public async ValueTask Handle(ProductName productName, CancellationToken cancellationToken) => await list(productName, cancellationToken) + .Where(api => shouldExtractApi(api.Name)) .IterParallel(async productapi => await writeArtifacts(productapi.Name, productapi.Dto, productName, cancellationToken), cancellationToken); } @@ -58,6 +60,7 @@ public static void ConfigureExtractProductApis(IServiceCollection services) { ConfigureListProductApis(services); ConfigureWriteProductApiArtifacts(services); + ApiServices.ConfigureShouldExtractApiName(services); services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); diff --git a/tools/code/extractor/Subscription.cs b/tools/code/extractor/Subscription.cs index e65de1da..a71e3e94 100644 --- a/tools/code/extractor/Subscription.cs +++ b/tools/code/extractor/Subscription.cs @@ -4,7 +4,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; -using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -16,19 +15,17 @@ namespace extractor; file delegate IAsyncEnumerable<(SubscriptionName Name, SubscriptionDto Dto)> ListSubscriptions(CancellationToken cancellationToken); -file delegate bool ShouldExtractSubscription(SubscriptionName name); +file delegate bool ShouldExtractSubscription(SubscriptionName name, SubscriptionDto dto); file delegate ValueTask WriteSubscriptionArtifacts(SubscriptionName name, SubscriptionDto dto, CancellationToken cancellationToken); file delegate ValueTask WriteSubscriptionInformationFile(SubscriptionName name, SubscriptionDto dto, CancellationToken cancellationToken); -file sealed class ExtractSubscriptionsHandler(ListSubscriptions list, ShouldExtractSubscription shouldExtract, WriteSubscriptionArtifacts writeArtifacts) +file sealed class ExtractSubscriptionsHandler(ListSubscriptions list, ShouldExtractSubscription shouldExtractSubscription, WriteSubscriptionArtifacts writeArtifacts) { public async ValueTask Handle(CancellationToken cancellationToken) => await list(cancellationToken) - // Skip master subscription - .Where(subscription => subscription.Name != SubscriptionName.From("master")) - .Where(subscription => shouldExtract(subscription.Name)) + .Where(subscription => shouldExtractSubscription(subscription.Name, subscription.Dto)) .IterParallel(async subscription => await writeArtifacts(subscription.Name, subscription.Dto, cancellationToken), cancellationToken); } @@ -39,13 +36,22 @@ file sealed class ListSubscriptionsHandler(ManagementServiceUri serviceUri, Http SubscriptionsUri.From(serviceUri).List(pipeline, cancellationToken); } -file sealed class ShouldExtractSubscriptionHandler(ShouldExtractFactory shouldExtractFactory) +file sealed class ShouldExtractSubscriptionHandler(ShouldExtractFactory shouldExtractFactory, ShouldExtractApiName shouldExtractApi, ShouldExtractProduct shouldExtractProduct) { - public bool Handle(SubscriptionName name) - { - var shouldExtract = shouldExtractFactory.Create(); - return shouldExtract(name); - } + public bool Handle(SubscriptionName name, SubscriptionDto dto) => + // Don't extract the master subscription + name != SubscriptionName.From("master") + // Check name from configuration override + && shouldExtractFactory.Create().Invoke(name) + // Don't extract subscription if its API should not be extracted + && SubscriptionModule.TryGetApiName(dto) + .Map(shouldExtractApi.Invoke) + .IfNone(true) + // Don't extract subscription if its product should not be extracted + && SubscriptionModule.TryGetProductName(dto) + .Map(shouldExtractProduct.Invoke) + .IfNone(true); + } file sealed class WriteSubscriptionArtifactsHandler(WriteSubscriptionInformationFile writeInformationFile) @@ -89,6 +95,9 @@ private static void ConfigureListSubscriptions(IServiceCollection services) private static void ConfigureShouldExtractSubscription(IServiceCollection services) { + ApiServices.ConfigureShouldExtractApiName(services); + ProductServices.ConfigureShouldExtractProduct(services); + services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } diff --git a/tools/code/extractor/VersionSet.cs b/tools/code/extractor/VersionSet.cs index 43914499..2dc7b836 100644 --- a/tools/code/extractor/VersionSet.cs +++ b/tools/code/extractor/VersionSet.cs @@ -15,7 +15,7 @@ namespace extractor; file delegate IAsyncEnumerable<(VersionSetName Name, VersionSetDto Dto)> ListVersionSets(CancellationToken cancellationToken); -file delegate bool ShouldExtractVersionSet(VersionSetName name); +internal delegate bool ShouldExtractVersionSet(VersionSetName name); file delegate ValueTask WriteVersionSetArtifacts(VersionSetName name, VersionSetDto dto, CancellationToken cancellationToken); @@ -84,7 +84,7 @@ private static void ConfigureListVersionSets(IServiceCollection services) services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static void ConfigureShouldExtractVersionSet(IServiceCollection services) + public static void ConfigureShouldExtractVersionSet(IServiceCollection services) { services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); diff --git a/tools/code/publisher/Api.cs b/tools/code/publisher/Api.cs index 1d5f431a..38dd6070 100644 --- a/tools/code/publisher/Api.cs +++ b/tools/code/publisher/Api.cs @@ -187,8 +187,8 @@ private async ValueTask Put(ApiName name, ApiDto dto, CancellationToken cancella } private async ValueTask PutVersionSet(ApiDto dto, CancellationToken cancellationToken) => - await Common.TryGetVersionSetName(dto) - .IterTask(putVersionSet.Invoke, cancellationToken); + await ApiModule.TryGetVersionSetName(dto) + .IterTask(putVersionSet.Invoke, cancellationToken); private async ValueTask PutCurrentRevision(ApiName name, ApiDto dto, CancellationToken cancellationToken) { @@ -593,7 +593,7 @@ await getDtosInPreviousCommit() var dtoOption = await kvp.Value(cancellationToken); return from dto in dtoOption - from versionSetName in Common.TryGetVersionSetName(dto) + from versionSetName in ApiModule.TryGetVersionSetName(dto) select (VersionSetName: versionSetName, ApiName: kvp.Key); }) .GroupBy(x => x.VersionSetName, x => x.ApiName) @@ -740,11 +740,6 @@ public static ILogger GetLogger(ILoggerFactory loggerFactory) => YamlOpenApiSpecificationFile.Name }.ToFrozenSet(); - public static Option TryGetVersionSetName(ApiDto dto) => - from versionSetId in Prelude.Optional(dto.Properties.ApiVersionSetId) - from versionSetNameString in versionSetId.Split('/').LastOrNone() - select VersionSetName.From(versionSetNameString); - public static ApiRevisionNumber GetRevisionNumber(ApiDto dto) => ApiRevisionNumber.TryFrom(dto.Properties.ApiRevision) .IfNone(() => ApiRevisionNumber.From(1)); diff --git a/tools/code/publisher/Diagnostic.cs b/tools/code/publisher/Diagnostic.cs index c4e3cee7..56667fc4 100644 --- a/tools/code/publisher/Diagnostic.cs +++ b/tools/code/publisher/Diagnostic.cs @@ -144,8 +144,8 @@ private async ValueTask Put(DiagnosticName name, DiagnosticDto dto, Cancellation } private async ValueTask PutLogger(DiagnosticDto dto, CancellationToken cancellationToken) => - await Common.TryGetLoggerName(dto) - .IterTask(putLogger.Invoke, cancellationToken); + await DiagnosticModule.TryGetLoggerName(dto) + .IterTask(putLogger.Invoke, cancellationToken); public void Dispose() => semaphore.Dispose(); } @@ -225,7 +225,7 @@ await getDtosInPreviousCommit() var dtoOption = await kvp.Value(cancellationToken); return from dto in dtoOption - from loggerName in Common.TryGetLoggerName(dto) + from loggerName in DiagnosticModule.TryGetLoggerName(dto) select (LoggerName: loggerName, DiagnosticName: kvp.Key); }) .GroupBy(x => x.LoggerName, x => x.DiagnosticName) @@ -360,9 +360,4 @@ file static class Common { public static ILogger GetLogger(ILoggerFactory factory) => factory.CreateLogger("DiagnosticPublisher"); - - public static Option TryGetLoggerName(DiagnosticDto dto) => - from loggerId in Prelude.Optional(dto.Properties.LoggerId) - from loggerNameString in loggerId.Split('/').LastOrNone() - select LoggerName.From(loggerNameString); } \ No newline at end of file diff --git a/tools/code/publisher/Subscription.cs b/tools/code/publisher/Subscription.cs index bac396f5..1abfdebb 100644 --- a/tools/code/publisher/Subscription.cs +++ b/tools/code/publisher/Subscription.cs @@ -149,12 +149,12 @@ private async ValueTask Put(SubscriptionName name, SubscriptionDto dto, Cancella } private async ValueTask PutProduct(SubscriptionDto dto, CancellationToken cancellationToken) => - await Common.TryGetProductName(dto) - .IterTask(putProduct.Invoke, cancellationToken); + await SubscriptionModule.TryGetProductName(dto) + .IterTask(putProduct.Invoke, cancellationToken); private async ValueTask PutApi(SubscriptionDto dto, CancellationToken cancellationToken) => - await Common.TryGetApiName(dto) - .IterTask(putApi.Invoke, cancellationToken); + await SubscriptionModule.TryGetApiName(dto) + .IterTask(putApi.Invoke, cancellationToken); public void Dispose() => semaphore.Dispose(); } @@ -234,7 +234,7 @@ await getDtosInPreviousCommit() var dtoOption = await kvp.Value(cancellationToken); return from dto in dtoOption - from productName in Common.TryGetProductName(dto) + from productName in SubscriptionModule.TryGetProductName(dto) select (ProductName: productName, SubscriptionName: kvp.Key); }) .GroupBy(x => x.ProductName, x => x.SubscriptionName) @@ -301,7 +301,7 @@ await getDtosInPreviousCommit() var dtoOption = await kvp.Value(cancellationToken); return from dto in dtoOption - from apiName in Common.TryGetApiName(dto) + from apiName in SubscriptionModule.TryGetApiName(dto) select (ApiName: apiName, SubscriptionName: kvp.Key); }) .GroupBy(x => x.ApiName, x => x.SubscriptionName) @@ -420,16 +420,4 @@ file static class Common { public static ILogger GetLogger(ILoggerFactory factory) => factory.CreateLogger("SubscriptionPublisher"); - - public static Option TryGetProductName(SubscriptionDto dto) => - from scope in Prelude.Optional(dto.Properties.Scope) - where scope.Contains("/products/", StringComparison.OrdinalIgnoreCase) - from productNameString in scope.Split('/').LastOrNone() - select ProductName.From(productNameString); - - public static Option TryGetApiName(SubscriptionDto dto) => - from scope in Prelude.Optional(dto.Properties.Scope) - where scope.Contains("/apis/", StringComparison.OrdinalIgnoreCase) - from apiNameString in scope.Split('/').LastOrNone() - select ApiName.From(apiNameString); } \ No newline at end of file From 0e83e7f48fb40e8573f7014fe74045eae85010e1 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Wed, 8 May 2024 11:10:28 -0400 Subject: [PATCH 21/29] Bump package versions --- tools/code/common/common.csproj | 6 +++--- tools/code/publisher/publisher.csproj | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/code/common/common.csproj b/tools/code/common/common.csproj index d8c3e816..9222e806 100644 --- a/tools/code/common/common.csproj +++ b/tools/code/common/common.csproj @@ -10,8 +10,8 @@ - - + + @@ -24,7 +24,7 @@ - + diff --git a/tools/code/publisher/publisher.csproj b/tools/code/publisher/publisher.csproj index ed0922f9..bd29b26c 100644 --- a/tools/code/publisher/publisher.csproj +++ b/tools/code/publisher/publisher.csproj @@ -13,7 +13,7 @@ - + From b2eccb8b303a379a7aae7904bffbc3d93b589f8d Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Fri, 24 May 2024 13:39:58 -0400 Subject: [PATCH 22/29] Change integration tests + Git searching --- tools/code/aspire/Program.cs | 15 + .../aspire/Properties/launchSettings.json | 31 + .../code/aspire/appsettings.Development.json | 8 + tools/code/aspire/aspire.csproj | 19 + tools/code/code.sln | 16 +- tools/code/common.tests/common.tests.csproj | 5 +- tools/code/common/OpenTelemetry.cs | 42 ++ tools/code/common/common.csproj | 13 +- tools/code/extractor/Common.cs | 6 + tools/code/extractor/extractor.csproj | 3 +- tools/code/integration.tests/Api.cs | 650 +++++++++++------- tools/code/integration.tests/App.cs | 30 + tools/code/integration.tests/Backend.cs | 321 ++++++--- tools/code/integration.tests/Common.cs | 114 +++ tools/code/integration.tests/Configuration.cs | 338 ++++----- tools/code/integration.tests/Diagnostic.cs | 344 ++++++--- tools/code/integration.tests/Extractor.cs | 183 ++++- tools/code/integration.tests/Fixture.cs | 4 +- tools/code/integration.tests/Gateway.cs | 29 + tools/code/integration.tests/Group.cs | 320 ++++++--- tools/code/integration.tests/Logger.cs | 382 +++++++--- .../integration.tests/ManagementService.cs | 462 +++++++++++++ tools/code/integration.tests/NamedValue.cs | 334 ++++++--- .../code/integration.tests/PolicyFragment.cs | 321 ++++++--- tools/code/integration.tests/Product.cs | 368 ++++++---- tools/code/integration.tests/Program.cs | 65 ++ tools/code/integration.tests/Publisher.cs | 219 ++++-- tools/code/integration.tests/Service.cs | 266 ------- tools/code/integration.tests/ServicePolicy.cs | 334 ++++++--- tools/code/integration.tests/Subscription.cs | 30 +- tools/code/integration.tests/Tag.cs | 307 +++++++-- tools/code/integration.tests/Test.cs | 506 ++++++++++++++ tools/code/integration.tests/Tests.cs | 274 -------- tools/code/integration.tests/VersionSet.cs | 373 +++++++--- .../integration.tests.csproj | 52 +- tools/code/publisher/Common.cs | 6 + tools/code/publisher/Git.cs | 12 +- tools/code/publisher/publisher.csproj | 5 +- 38 files changed, 4855 insertions(+), 1952 deletions(-) create mode 100644 tools/code/aspire/Program.cs create mode 100644 tools/code/aspire/Properties/launchSettings.json create mode 100644 tools/code/aspire/appsettings.Development.json create mode 100644 tools/code/aspire/aspire.csproj create mode 100644 tools/code/common/OpenTelemetry.cs create mode 100644 tools/code/integration.tests/App.cs create mode 100644 tools/code/integration.tests/Common.cs create mode 100644 tools/code/integration.tests/ManagementService.cs create mode 100644 tools/code/integration.tests/Program.cs delete mode 100644 tools/code/integration.tests/Service.cs create mode 100644 tools/code/integration.tests/Test.cs delete mode 100644 tools/code/integration.tests/Tests.cs diff --git a/tools/code/aspire/Program.cs b/tools/code/aspire/Program.cs new file mode 100644 index 00000000..e4b493f6 --- /dev/null +++ b/tools/code/aspire/Program.cs @@ -0,0 +1,15 @@ +using Aspire.Hosting; +using Projects; + +internal static class Program +{ + private static void Main(string[] args) + { + var builder = DistributedApplication.CreateBuilder(args); + + builder.AddProject("integration-tests"); + //.WithEnvironment("CSCHECK_SEED", "0000KOIPe036"); + + builder.Build().Run(); + } +} \ No newline at end of file diff --git a/tools/code/aspire/Properties/launchSettings.json b/tools/code/aspire/Properties/launchSettings.json new file mode 100644 index 00000000..88bd7039 --- /dev/null +++ b/tools/code/aspire/Properties/launchSettings.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17016;http://localhost:15029", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21043", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22139", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:15029", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19296", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20230", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + } + } +} diff --git a/tools/code/aspire/appsettings.Development.json b/tools/code/aspire/appsettings.Development.json new file mode 100644 index 00000000..0c208ae9 --- /dev/null +++ b/tools/code/aspire/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/tools/code/aspire/aspire.csproj b/tools/code/aspire/aspire.csproj new file mode 100644 index 00000000..71e7c345 --- /dev/null +++ b/tools/code/aspire/aspire.csproj @@ -0,0 +1,19 @@ + + + + Exe + net8.0 + enable + true + 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 + + + + + + + + + + + diff --git a/tools/code/code.sln b/tools/code/code.sln index 989c524b..8dfdd6e5 100644 --- a/tools/code/code.sln +++ b/tools/code/code.sln @@ -11,7 +11,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "publisher", "publisher\publ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "common.tests", "common.tests\common.tests.csproj", "{D1ED4FC5-8C52-4380-8520-BC46F475A4F8}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "integration.tests", "integration.tests\integration.tests.csproj", "{76339A55-86A2-4CBE-991B-515DED712804}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "integration.tests", "integration.tests\integration.tests.csproj", "{05923D65-3595-445A-8EB3-5ECC1F99A727}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "aspire", "aspire\aspire.csproj", "{C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -35,10 +37,14 @@ Global {D1ED4FC5-8C52-4380-8520-BC46F475A4F8}.Debug|Any CPU.Build.0 = Debug|Any CPU {D1ED4FC5-8C52-4380-8520-BC46F475A4F8}.Release|Any CPU.ActiveCfg = Release|Any CPU {D1ED4FC5-8C52-4380-8520-BC46F475A4F8}.Release|Any CPU.Build.0 = Release|Any CPU - {76339A55-86A2-4CBE-991B-515DED712804}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {76339A55-86A2-4CBE-991B-515DED712804}.Debug|Any CPU.Build.0 = Debug|Any CPU - {76339A55-86A2-4CBE-991B-515DED712804}.Release|Any CPU.ActiveCfg = Release|Any CPU - {76339A55-86A2-4CBE-991B-515DED712804}.Release|Any CPU.Build.0 = Release|Any CPU + {05923D65-3595-445A-8EB3-5ECC1F99A727}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05923D65-3595-445A-8EB3-5ECC1F99A727}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05923D65-3595-445A-8EB3-5ECC1F99A727}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05923D65-3595-445A-8EB3-5ECC1F99A727}.Release|Any CPU.Build.0 = Release|Any CPU + {C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C0AD9089-77B8-4E2F-ADD3-1F0846B8D370}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/tools/code/common.tests/common.tests.csproj b/tools/code/common.tests/common.tests.csproj index a33f307b..d5beb1ff 100644 --- a/tools/code/common.tests/common.tests.csproj +++ b/tools/code/common.tests/common.tests.csproj @@ -2,15 +2,16 @@ net8.0 + false true - latest-all + 8-all CA1034,CA1062,CA1724,CA2007,CA1848,CA1716 enable - + diff --git a/tools/code/common/OpenTelemetry.cs b/tools/code/common/OpenTelemetry.cs new file mode 100644 index 00000000..6a874bcb --- /dev/null +++ b/tools/code/common/OpenTelemetry.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Instrumentation.Http; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using System.Diagnostics; + +namespace common; + +public static class OpenTelemetryServices +{ + public static void Configure(IServiceCollection services) + { + var sourceName = services.BuildServiceProvider().GetService()?.Name ?? "ApiOps.*"; + + services.AddOpenTelemetry() + .WithMetrics(metrics => metrics.AddHttpClientInstrumentation() + .AddRuntimeInstrumentation() + .AddMeter("Azure.*")) + .WithTracing(tracing => tracing.AddHttpClientInstrumentation(ConfigureHttpClientTraceInstrumentationOptions) + .AddSource("Azure.*") + .AddSource(sourceName) + .SetSampler()); + + var configuration = services.BuildServiceProvider().GetRequiredService(); + configuration.TryGetValue("OTEL_EXPORTER_OTLP_ENDPOINT") + .Iter(_ => + { + services.AddLogging(builder => builder.AddOpenTelemetry()); + services.Configure(logging => logging.AddOtlpExporter()) + .ConfigureOpenTelemetryMeterProvider(metrics => metrics.AddOtlpExporter()) + .ConfigureOpenTelemetryTracerProvider(tracing => tracing.AddOtlpExporter()); + }); + } + + private static void ConfigureHttpClientTraceInstrumentationOptions(HttpClientTraceInstrumentationOptions options) + { + options.FilterHttpRequestMessage = (_) => Activity.Current?.Parent?.Source?.Name != "Azure.Core.Http"; + } +} diff --git a/tools/code/common/common.csproj b/tools/code/common/common.csproj index 9222e806..eb08ee52 100644 --- a/tools/code/common/common.csproj +++ b/tools/code/common/common.csproj @@ -3,7 +3,8 @@ net8.0 true - latest-all + 8-all + false CA1034,CA1062,CA1724,CA2007,CA1848 enable @@ -16,12 +17,16 @@ - + - + - + + + + + diff --git a/tools/code/extractor/Common.cs b/tools/code/extractor/Common.cs index d0a19133..9fbc3219 100644 --- a/tools/code/extractor/Common.cs +++ b/tools/code/extractor/Common.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.JsonWebTokens; using System; +using System.Diagnostics; using System.IO; using System.Reflection; @@ -20,6 +21,7 @@ internal static class CommonServices { public static void Configure(IServiceCollection services) { + services.AddSingleton(GetActivitySource); services.AddSingleton(GetAzureEnvironment); services.AddSingleton(GetTokenCredential); services.AddSingleton(GetHttpPipeline); @@ -28,8 +30,12 @@ public static void Configure(IServiceCollection services) services.AddSingleton(GetManagementServiceDirectory); services.AddSingleton(GetConfigurationJson); services.AddSingleton(); + OpenTelemetryServices.Configure(services); } + private static ActivitySource GetActivitySource(IServiceProvider provider) => + new("ApiOps.Extractor"); + private static AzureEnvironment GetAzureEnvironment(IServiceProvider provider) { var configuration = provider.GetRequiredService(); diff --git a/tools/code/extractor/extractor.csproj b/tools/code/extractor/extractor.csproj index 659093a1..ed8018c3 100644 --- a/tools/code/extractor/extractor.csproj +++ b/tools/code/extractor/extractor.csproj @@ -2,8 +2,9 @@ net8.0 + false true - latest-all + 8-all CA1708,CA1724,CA1812,CA1848,CA2007,CA1034,CA1062 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 Exe diff --git a/tools/code/integration.tests/Api.cs b/tools/code/integration.tests/Api.cs index 56c110d9..244bc013 100644 --- a/tools/code/integration.tests/Api.cs +++ b/tools/code/integration.tests/Api.cs @@ -6,121 +6,60 @@ using Flurl; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Api -{ - public static Gen GenerateUpdate(ApiModel original) => - from revisions in GenerateRevisionUpdates(original.Revisions, original.Type, original.Name) - select original with - { - Revisions = revisions - }; +internal delegate ValueTask DeleteAllApis(ManagementServiceName serviceName, CancellationToken cancellationToken); - private static Gen> GenerateRevisionUpdates(FrozenSet revisions, ApiType type, ApiName name) - { - var newGen = ApiRevision.GenerateSet(type, name); - var updateGen = (ApiRevision revision) => GenerateRevisionUpdate(revision, type); +internal delegate ValueTask PutApiModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); - return Fixture.GenerateNewSet(revisions, newGen, updateGen); - } +internal delegate ValueTask ValidateExtractedApis(Option> apiNamesOption, Option defaultApiSpecification, Option> versionSetNamesOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); - private static Gen GenerateRevisionUpdate(ApiRevision revision, ApiType type) => - from serviceUri in type is ApiType.Soap or ApiType.WebSocket - ? Gen.Const(revision.ServiceUri) - : ApiRevision.GenerateServiceUri(type) - from description in ApiRevision.GenerateDescription().OptionOf() - select revision with - { - ServiceUri = serviceUri.ValueUnsafe(), - Description = description.ValueUnsafe() - }; +file delegate ValueTask> GetApimApis(ManagementServiceName serviceName, CancellationToken cancellationToken); - public static Gen GenerateOverride(ApiDto original) => - from serviceUrl in (original.Properties.Type ?? original.Properties.ApiType) switch - { - "websocket" or "soap" => Gen.Const(original.Properties.ServiceUrl), - _ => Generator.AbsoluteUri.Select(uri => (string?)uri.ToString()) - } - from revisionDescription in ApiRevision.GenerateDescription().OptionOf() - select new ApiDto() - { - Properties = new ApiDto.ApiCreateOrUpdateProperties - { - ServiceUrl = serviceUrl, - ApiRevisionDescription = revisionDescription.ValueUnsafe() - } - }; +file delegate ValueTask> TryGetApimGraphQlSchema(ApiName name, ManagementServiceName serviceName, CancellationToken cancellationToken); - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.SelectMany(model => model.Revisions.Select((revision, index) => - { - var apiName = GetApiName(model.Name, revision, index); - var dto = GetDto(model.Name, model.Type, model.Path, model.Version, revision); - return (apiName, dto); - })) - .Where(api => ApiName.IsNotRevisioned(api.apiName)) - .ToFrozenDictionary(); +file delegate ValueTask> GetFileApis(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); - private static ApiName GetApiName(ApiName name, ApiRevision revision, int index) +internal delegate ValueTask WriteApiModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedApis(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllApisHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var rootApiName = ApiName.GetRootName(name); + using var _ = activitySource.StartActivity(nameof(DeleteAllApis)); - return index == 0 ? rootApiName : ApiName.GetRevisionedName(rootApiName, revision.Number); + logger.LogInformation("Deleting all APIs in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await ApisUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); } +} - private static ApiDto GetDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => - new ApiDto() - { - Properties = new ApiDto.ApiCreateOrUpdateProperties - { - // APIM sets the description to null when it imports for SOAP APIs. - DisplayName = name.ToString(), - Path = path, - ApiType = type switch - { - ApiType.Http => null, - ApiType.Soap => "soap", - ApiType.GraphQl => null, - ApiType.WebSocket => null, - _ => throw new NotSupportedException() - }, - Type = type switch - { - ApiType.Http => "http", - ApiType.Soap => "soap", - ApiType.GraphQl => "graphql", - ApiType.WebSocket => "websocket", - _ => throw new NotSupportedException() - }, - Protocols = type switch - { - ApiType.Http => ["http", "https"], - ApiType.Soap => ["http", "https"], - ApiType.GraphQl => ["http", "https"], - ApiType.WebSocket => ["ws", "wss"], - _ => throw new NotSupportedException() - }, - ServiceUrl = revision.ServiceUri.ValueUnsafe()?.ToString(), - ApiRevisionDescription = revision.Description.ValueUnsafe(), - ApiRevision = $"{revision.Number.ToInt()}", - ApiVersion = version.Map(version => version.Version).ValueUnsafe(), - ApiVersionSetId = version.Map(version => $"/apiVersionSets/{version.VersionSetName}").ValueUnsafe() - } - }; +file sealed class PutApiModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutApiModels)); - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => + logger.LogInformation("Putting API models in {ServiceName}...", serviceName); await models.IterParallel(async model => { - async ValueTask putRevision(ApiRevision revision) => await Put(model.Name, model.Type, model.Path, model.Version, revision, serviceUri, pipeline, cancellationToken); + var serviceUri = getServiceUri(serviceName); + async ValueTask putRevision(ApiRevision revision) => await Put(model.Name, model.Type, model.Path, model.Version, revision, serviceUri, cancellationToken); // Put first revision to make sure it's the current revision. await model.Revisions.HeadOrNone().IterTask(putRevision); @@ -128,8 +67,9 @@ await models.IterParallel(async model => // Put other revisions await model.Revisions.Skip(1).IterParallel(putRevision, cancellationToken); }, cancellationToken); + } - private static async ValueTask Put(ApiName name, ApiType type, string path, Option version, ApiRevision revision, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private async ValueTask Put(ApiName name, ApiType type, string path, Option version, ApiRevision revision, ManagementServiceUri serviceUri, CancellationToken cancellationToken) { var rootName = ApiName.GetRootName(name); var dto = GetDto(rootName, type, path, version, revision); @@ -147,7 +87,7 @@ private static async ValueTask Put(ApiName name, ApiType type, string path, Opti await ApiTag.Put(revision.Tags, revisionedName, serviceUri, pipeline, cancellationToken); } - private static ApiDto GetPutDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => + private static ApiDto GetDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => new ApiDto() { Properties = new ApiDto.ApiCreateOrUpdateProperties @@ -171,17 +111,6 @@ private static ApiDto GetPutDto(ApiName name, ApiType type, string path, Option< ApiType.WebSocket => "websocket", _ => throw new NotSupportedException() }, - Format = revision.Specification.IsSome - ? type switch - { - ApiType.Http => "openapi+json", - ApiType.Soap => "wsdl", - _ => null - } - : null, - Value = type is ApiType.Http or ApiType.Soap - ? revision.Specification.ValueUnsafe() - : null, Protocols = type switch { ApiType.Http => ["http", "https"], @@ -197,9 +126,190 @@ private static ApiDto GetPutDto(ApiName name, ApiType type, string path, Option< ApiVersionSetId = version.Map(version => $"/apiVersionSets/{version.VersionSetName}").ValueUnsafe() } }; +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await ApisUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class ValidateExtractedApisHandler(ILogger logger, GetApimApis getApimResources, TryGetApimGraphQlSchema tryGetApimGraphQlSchema, GetFileApis getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> apiNamesOption, Option defaultApiSpecification, Option> versionSetNamesOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedApis)); + + logger.LogInformation("Validating extracted APIs in {ServiceName}...", serviceName); + + var expected = await GetExpectedResources(apiNamesOption, versionSetNamesOption, serviceName, cancellationToken); + + await ValidateExtractedInformationFiles(expected, serviceDirectory, cancellationToken); + await ValidateExtractedSpecificationFiles(expected, defaultApiSpecification, serviceName, serviceDirectory, cancellationToken); + } + + private async ValueTask> GetExpectedResources(Option> apiNamesOption, Option> versionSetNamesOption, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var apimResources = await getApimResources(serviceName, cancellationToken); + + return apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, apiNamesOption)) + .WhereValue(dto => ApiModule.TryGetVersionSetName(dto) + .Map(name => ExtractorOptions.ShouldExtract(name, versionSetNamesOption)) + .IfNone(true)); + } + + private async ValueTask ValidateExtractedInformationFiles(IDictionary expectedResources, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = expectedResources.MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + private static string NormalizeDto(ApiDto dto) => + new + { + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Path = dto.Properties.Path ?? string.Empty, + RevisionDescription = dto.Properties.ApiRevisionDescription ?? string.Empty, + Revision = dto.Properties.ApiRevision ?? string.Empty, + ServiceUrl = Uri.TryCreate(dto.Properties.ServiceUrl, UriKind.Absolute, out var uri) + ? uri.RemovePath().ToString() + : string.Empty + }.ToString()!; + + private async ValueTask ValidateExtractedSpecificationFiles(IDictionary expectedResources, Option defaultApiSpecification, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var expected = await expectedResources.ToAsyncEnumerable() + .Choose(async kvp => + { + var name = kvp.Key; + return from specification in await GetExpectedApiSpecification(name, kvp.Value, defaultApiSpecification, serviceName, cancellationToken) + // Skip XML specification files. Sometimes they get extracted, other times they fail. + where specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl) + select (name, specification); + }) + .ToFrozenDictionary(cancellationToken); + + var actual = await ApiModule.ListSpecificationFiles(serviceDirectory, cancellationToken) + .Select(file => (file.Parent.Name, file.Specification)) + // Skip XML specification files. Sometimes they get extracted, other times they fail. + .Where(file => file.Specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl)) + .ToFrozenDictionary(cancellationToken); + + actual.Should().BeEquivalentTo(expected); + } + + private async ValueTask> GetExpectedApiSpecification(ApiName name, ApiDto dto, Option defaultApiSpecification, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + switch (dto.Properties.ApiType ?? dto.Properties.Type) + { + case "graphql": + var specificationContents = await tryGetApimGraphQlSchema(name, serviceName, cancellationToken); + return specificationContents.Map(contents => new ApiSpecification.GraphQl() as ApiSpecification); + case "soap": + return new ApiSpecification.Wsdl(); + case "websocket": + return Option.None; + default: +#pragma warning disable CA1849 // Call async methods when in an async method + return defaultApiSpecification.IfNone(() => new ApiSpecification.OpenApi + { + Format = new OpenApiFormat.Yaml(), + Version = new OpenApiVersion.V3() + }); +#pragma warning restore CA1849 // Call async methods when in an async method + } + } +} + +file sealed class GetApimApisHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetApimApis)); + + logger.LogInformation("Getting APIs from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = ApisUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class TryGetApimGraphQlSchemaHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ApiName name, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(TryGetApimGraphQlSchema)); + + logger.LogInformation("Getting GraphQL schema for {ApiName} from {ServiceName}...", name, serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = ApiUri.From(name, serviceUri); + + return await uri.TryGetGraphQlSchema(pipeline, cancellationToken); + } +} + +file sealed class GetFileApisHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileApis)); + + logger.LogInformation("Getting apis from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => ApiInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ApiInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileApis)); + + logger.LogInformation("Getting apis from {ServiceDirectory}...", serviceDirectory); + + return await ApiModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteApiModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteApiModels)); + + logger.LogInformation("Writing api models to {ServiceDirectory}...", serviceDirectory); + await models.IterParallel(async model => + { + await WriteRevisionArtifacts(model, serviceDirectory, cancellationToken); + }, cancellationToken); + } public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => await models.IterParallel(async model => @@ -228,6 +338,13 @@ private static async ValueTask WriteInformationFile(ApiName name, ApiType type, await informationFile.WriteDto(dto, cancellationToken); } + private static ApiName GetApiName(ApiName name, ApiRevision revision, int index) + { + var rootApiName = ApiName.GetRootName(name); + + return index == 0 ? rootApiName : ApiName.GetRevisionedName(rootApiName, revision.Number); + } + private static async ValueTask WriteSpecificationFile(ApiName name, ApiType type, ApiRevision revision, int index, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { var specificationOption = from contents in revision.Specification @@ -253,72 +370,65 @@ await specificationOption.IterTask(async x => }); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, Option defaultApiSpecification, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var expectedResources = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)); - - await ValidateExtractedInformationFiles(expectedResources, serviceDirectory, cancellationToken); - await ValidateExtractedSpecificationFiles(expectedResources, defaultApiSpecification, serviceDirectory, serviceUri, pipeline, cancellationToken); - - await expectedResources.Keys.IterParallel(async name => + private static ApiDto GetDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => + new ApiDto() { - await ApiPolicy.ValidateExtractedArtifacts(serviceDirectory, name, serviceUri, pipeline, cancellationToken); - await ApiTag.ValidateExtractedArtifacts(serviceDirectory, name, serviceUri, pipeline, cancellationToken); - }, cancellationToken); - } - - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = ApisUri.From(serviceUri); - - return await uri.List(pipeline, cancellationToken) - .Select(api => - { - var normalizedName = NormalizeName(api.Name, api.Dto); - return (normalizedName, api.Dto); - }) - .ToFrozenDictionary(cancellationToken); - } + Properties = new ApiDto.ApiCreateOrUpdateProperties + { + // APIM sets the description to null when it imports for SOAP APIs. + DisplayName = name.ToString(), + Path = path, + ApiType = type switch + { + ApiType.Http => null, + ApiType.Soap => "soap", + ApiType.GraphQl => null, + ApiType.WebSocket => null, + _ => throw new NotSupportedException() + }, + Type = type switch + { + ApiType.Http => "http", + ApiType.Soap => "soap", + ApiType.GraphQl => "graphql", + ApiType.WebSocket => "websocket", + _ => throw new NotSupportedException() + }, + Protocols = type switch + { + ApiType.Http => ["http", "https"], + ApiType.Soap => ["http", "https"], + ApiType.GraphQl => ["http", "https"], + ApiType.WebSocket => ["ws", "wss"], + _ => throw new NotSupportedException() + }, + ServiceUrl = revision.ServiceUri.ValueUnsafe()?.ToString(), + ApiRevisionDescription = revision.Description.ValueUnsafe(), + ApiRevision = $"{revision.Number.ToInt()}", + ApiVersion = version.Map(version => version.Version).ValueUnsafe(), + ApiVersionSetId = version.Map(version => $"/apiVersionSets/{version.VersionSetName}").ValueUnsafe() + } + }; +} - // APIM has an issue where it sometimes returns duplicate API names. - private static ApiName NormalizeName(ApiName name, ApiDto dto) +file sealed class ValidatePublishedApisHandler(ILogger logger, GetFileApis getFileResources, GetApimApis getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - if (dto.Properties.IsCurrent is true) - { - return name; - } - - if (ApiName.IsRevisioned(name)) - { - return name; - } - - var revisionNumber = ApiRevisionNumber.TryFrom(dto.Properties.ApiRevision) - .IfNone(() => throw new InvalidOperationException("Could not get revision number.")); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedApis)); - var rootName = ApiName.GetRootName(name); + logger.LogInformation("Validating published apis in {ServiceDirectory}...", serviceDirectory); - return ApiName.GetRevisionedName(rootName, revisionNumber); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask ValidateExtractedInformationFiles(IDictionary expectedResources, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) - { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - - var expected = expectedResources.MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await ApiModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(ApiDto dto) => new { @@ -331,106 +441,172 @@ private static string NormalizeDto(ApiDto dto) => // ? uri.RemovePath().ToString() // : string.Empty }.ToString()!; +} - private static async ValueTask ValidateExtractedSpecificationFiles(IDictionary expectedResources, Option defaultApiSpecification, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class ApiServices +{ + public static void ConfigureDeleteAllApis(IServiceCollection services) { - var expected = await expectedResources.ToAsyncEnumerable() - .Choose(async kvp => - { - var name = kvp.Key; - return from specification in await GetExpectedApiSpecification(name, kvp.Value, defaultApiSpecification, serviceUri, pipeline, cancellationToken) - // Skip XML specification files. Sometimes they get extracted, other times they fail. - where specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl) - select (name, specification); - }) - .ToFrozenDictionary(cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var actual = await ApiModule.ListSpecificationFiles(serviceDirectory, cancellationToken) - .Select(file => (file.Parent.Name, file.Specification)) - // Skip XML specification files. Sometimes they get extracted, other times they fail. - .Where(file => file.Specification is not (ApiSpecification.Wsdl or ApiSpecification.Wadl)) - .ToFrozenDictionary(cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - actual.Should().BeEquivalentTo(expected); + public static void ConfigurePutApiModels(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetExpectedApiSpecification(ApiName name, ApiDto dto, Option defaultApiSpecification, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedApis(IServiceCollection services) { - switch (dto.Properties.ApiType ?? dto.Properties.Type) - { - case "graphql": - var specificationContents = await ApiUri.From(name, serviceUri) - .TryGetGraphQlSchema(pipeline, cancellationToken); - return specificationContents.Map(contents => new ApiSpecification.GraphQl() as ApiSpecification); - case "soap": - return new ApiSpecification.Wsdl(); - case "websocket": - return Option.None; - default: -#pragma warning disable CA1849 // Call async methods when in an async method - return defaultApiSpecification.IfNone(() => new ApiSpecification.OpenApi - { - Format = new OpenApiFormat.Yaml(), - Version = new OpenApiVersion.V3() - }); -#pragma warning restore CA1849 // Call async methods when in an async method - } + ConfigureGetApimApis(services); + ConfigureTryGetApimGraphQlSchema(services); + ConfigureGetFileApis(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetApimApis(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - await fileResources.Keys.IterParallel(async name => - { - await ApiPolicy.ValidatePublisherChanges(name, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ApiTag.ValidatePublisherChanges(name, serviceDirectory, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureTryGetApimGraphQlSchema(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetFileApis(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - await fileResources.Keys.IterParallel(async name => - { - await ApiPolicy.ValidatePublisherCommitChanges(name, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ApiTag.ValidatePublisherCommitChanges(name, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + public static void ConfigureWriteApiModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => ApiInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + public static void ConfigureValidatePublishedApis(IServiceCollection services) + { + ConfigureGetFileApis(services); + ConfigureGetApimApis(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ApiInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Api +{ + public static Gen GenerateUpdate(ApiModel original) => + from revisions in GenerateRevisionUpdates(original.Revisions, original.Type, original.Name) + select original with + { + Revisions = revisions + }; + + private static Gen> GenerateRevisionUpdates(FrozenSet revisions, ApiType type, ApiName name) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + var newGen = ApiRevision.GenerateSet(type, name); + var updateGen = (ApiRevision revision) => GenerateRevisionUpdate(revision, type); - return await contentsOption.MapTask(async contents => + return Fixture.GenerateNewSet(revisions, newGen, updateGen); + } + + private static Gen GenerateRevisionUpdate(ApiRevision revision, ApiType type) => + from serviceUri in type is ApiType.Soap or ApiType.WebSocket + ? Gen.Const(revision.ServiceUri) + : ApiRevision.GenerateServiceUri(type) + from description in ApiRevision.GenerateDescription().OptionOf() + select revision with { - using (contents) + ServiceUri = serviceUri.ValueUnsafe(), + Description = description.ValueUnsafe() + }; + + public static Gen GenerateOverride(ApiDto original) => + from serviceUrl in (original.Properties.Type ?? original.Properties.ApiType) switch + { + "websocket" or "soap" => Gen.Const(original.Properties.ServiceUrl), + _ => Generator.AbsoluteUri.Select(uri => (string?)uri.ToString()) + } + from revisionDescription in ApiRevision.GenerateDescription().OptionOf() + select new ApiDto() + { + Properties = new ApiDto.ApiCreateOrUpdateProperties { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + ServiceUrl = serviceUrl, + ApiRevisionDescription = revisionDescription.ValueUnsafe() } - }); + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.SelectMany(model => model.Revisions.Select((revision, index) => + { + var apiName = GetApiName(model.Name, revision, index); + var dto = GetDto(model.Name, model.Type, model.Path, model.Version, revision); + return (apiName, dto); + })) + .Where(api => ApiName.IsNotRevisioned(api.apiName)) + .ToFrozenDictionary(); + + private static ApiName GetApiName(ApiName name, ApiRevision revision, int index) + { + var rootApiName = ApiName.GetRootName(name); + + return index == 0 ? rootApiName : ApiName.GetRevisionedName(rootApiName, revision.Number); } + + private static ApiDto GetDto(ApiName name, ApiType type, string path, Option version, ApiRevision revision) => + new ApiDto() + { + Properties = new ApiDto.ApiCreateOrUpdateProperties + { + // APIM sets the description to null when it imports for SOAP APIs. + DisplayName = name.ToString(), + Path = path, + ApiType = type switch + { + ApiType.Http => null, + ApiType.Soap => "soap", + ApiType.GraphQl => null, + ApiType.WebSocket => null, + _ => throw new NotSupportedException() + }, + Type = type switch + { + ApiType.Http => "http", + ApiType.Soap => "soap", + ApiType.GraphQl => "graphql", + ApiType.WebSocket => "websocket", + _ => throw new NotSupportedException() + }, + Protocols = type switch + { + ApiType.Http => ["http", "https"], + ApiType.Soap => ["http", "https"], + ApiType.GraphQl => ["http", "https"], + ApiType.WebSocket => ["ws", "wss"], + _ => throw new NotSupportedException() + }, + ServiceUrl = revision.ServiceUri.ValueUnsafe()?.ToString(), + ApiRevisionDescription = revision.Description.ValueUnsafe(), + ApiRevision = $"{revision.Number.ToInt()}", + ApiVersion = version.Map(version => version.Version).ValueUnsafe(), + ApiVersionSetId = version.Map(version => $"/apiVersionSets/{version.VersionSetName}").ValueUnsafe() + } + }; } diff --git a/tools/code/integration.tests/App.cs b/tools/code/integration.tests/App.cs new file mode 100644 index 00000000..06c382b5 --- /dev/null +++ b/tools/code/integration.tests/App.cs @@ -0,0 +1,30 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace integration.tests; + +internal delegate ValueTask RunApplication(CancellationToken cancellationToken); + +file sealed class RunApplicationHandler(ActivitySource activitySource, RunTests runTests) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(RunApplication)); + + await runTests(cancellationToken); + } +} + +internal static class AppServices +{ + public static void ConfigureRunApplication(IServiceCollection services) + { + TestServices.ConfigureRunTests(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/Backend.cs b/tools/code/integration.tests/Backend.cs index 92373b8c..3c71eddc 100644 --- a/tools/code/integration.tests/Backend.cs +++ b/tools/code/integration.tests/Backend.cs @@ -5,41 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Backend +internal delegate ValueTask DeleteAllBackends(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutBackendModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedBackends(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimBackends(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileBackends(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteBackendModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedBackends(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllBackendsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(BackendModel original) => - from url in Generator.AbsoluteUri - from description in BackendModel.GenerateDescription().OptionOf() - select original with - { - Url = url, - Description = description - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllBackends)); - public static Gen GenerateOverride(BackendDto original) => - from url in Generator.AbsoluteUri - from description in BackendModel.GenerateDescription().OptionOf() - select new BackendDto + logger.LogInformation("Deleting all backends in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await BackendsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutBackendModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutBackendModels)); + + logger.LogInformation("Putting backend models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new BackendDto.BackendContract - { - Url = url.ToString(), - Description = description.ValueUnsafe() - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(BackendModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = BackendUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static BackendDto GetDto(BackendModel model) => new() @@ -51,29 +77,111 @@ private static BackendDto GetDto(BackendModel model) => Protocol = model.Protocol } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedBackendsHandler(ILogger logger, GetApimBackends getApimResources, GetFileBackends getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedBackends)); + + logger.LogInformation("Validating extracted backends in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(BackendDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + Url = dto.Properties.Url ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty, + Protocol = dto.Properties.Protocol ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(BackendModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimBackendsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = BackendUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimBackends)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting backends from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = BackendsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await BackendsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileBackendsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileBackends)); + + logger.LogInformation("Getting backends from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => BackendInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, BackendInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileBackends)); + + logger.LogInformation("Getting backends from {ServiceDirectory}...", serviceDirectory); + + return await BackendModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => +file sealed class WriteBackendModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteBackendModels)); + + logger.LogInformation("Writing backend models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(BackendModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -83,33 +191,36 @@ private static async ValueTask WriteInformationFile(BackendModel model, Manageme await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static BackendDto GetDto(BackendModel model) => + new() + { + Properties = new BackendDto.BackendContract + { + Url = model.Url.ToString(), + Description = model.Description.ValueUnsafe(), + Protocol = model.Protocol + } + }; +} + +file sealed class ValidatePublishedBackendsHandler(ILogger logger, GetFileBackends getFileResources, GetApimBackends getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedBackends)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published backends in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = BackendsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await BackendModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(BackendDto dto) => new { @@ -117,49 +228,99 @@ private static string NormalizeDto(BackendDto dto) => Description = dto.Properties.Description ?? string.Empty, Protocol = dto.Properties.Protocol ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class BackendServices +{ + public static void ConfigureDeleteAllBackends(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutBackendModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedBackends(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimBackends(services); + ConfigureGetFileBackends(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => BackendInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimBackends(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, BackendInformationFile file, CancellationToken cancellationToken) + private static void ConfigureGetFileBackends(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteBackendModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedBackends(IServiceCollection services) + { + ConfigureGetFileBackends(services); + ConfigureGetApimBackends(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Backend +{ + public static Gen GenerateUpdate(BackendModel original) => + from url in Generator.AbsoluteUri + from description in BackendModel.GenerateDescription().OptionOf() + select original with { - using (contents) + Url = url, + Description = description + }; + + public static Gen GenerateOverride(BackendDto original) => + from url in Generator.AbsoluteUri + from description in BackendModel.GenerateDescription().OptionOf() + select new BackendDto + { + Properties = new BackendDto.BackendContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + Url = url.ToString(), + Description = description.ValueUnsafe() } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static BackendDto GetDto(BackendModel model) => + new() + { + Properties = new BackendDto.BackendContract + { + Url = model.Url.ToString(), + Description = model.Description.ValueUnsafe(), + Protocol = model.Protocol + } + }; } diff --git a/tools/code/integration.tests/Common.cs b/tools/code/integration.tests/Common.cs new file mode 100644 index 00000000..3c9ecc50 --- /dev/null +++ b/tools/code/integration.tests/Common.cs @@ -0,0 +1,114 @@ +using Azure.Core; +using Azure.Core.Pipeline; +using Azure.Identity; +using Azure.ResourceManager; +using common; +using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.JsonWebTokens; +using System; +using System.Diagnostics; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; + +internal delegate string GetSubscriptionId(); +internal delegate string GetResourceGroupName(); +internal delegate ValueTask GetBearerToken(CancellationToken cancellationToken); + +internal static class CommonServices +{ + public static void Configure(IServiceCollection services) + { + services.AddSingleton(GetActivitySource); + services.AddSingleton(GetAzureEnvironment); + services.AddSingleton(GetTokenCredential); + services.AddSingleton(GetHttpPipeline); + services.AddSingleton(GetSubscriptionId); + services.AddSingleton(GetResourceGroupName); + services.AddSingleton(GetBearerToken); + OpenTelemetryServices.Configure(services); + } + + private static ActivitySource GetActivitySource(IServiceProvider provider) => + new("ApiOps.Integration.Tests"); + + private static AzureEnvironment GetAzureEnvironment(IServiceProvider provider) + { + var configuration = provider.GetRequiredService(); + + return configuration.TryGetValue("AZURE_CLOUD_ENVIRONMENT").ValueUnsafe() switch + { + null => AzureEnvironment.Public, + "AzureGlobalCloud" or nameof(ArmEnvironment.AzurePublicCloud) => AzureEnvironment.Public, + "AzureChinaCloud" or nameof(ArmEnvironment.AzureChina) => AzureEnvironment.China, + "AzureUSGovernment" or nameof(ArmEnvironment.AzureGovernment) => AzureEnvironment.USGovernment, + "AzureGermanCloud" or nameof(ArmEnvironment.AzureGermany) => AzureEnvironment.Germany, + _ => throw new InvalidOperationException($"AZURE_CLOUD_ENVIRONMENT is invalid. Valid values are {nameof(ArmEnvironment.AzurePublicCloud)}, {nameof(ArmEnvironment.AzureChina)}, {nameof(ArmEnvironment.AzureGovernment)}, {nameof(ArmEnvironment.AzureGermany)}") + }; + } + + private static TokenCredential GetTokenCredential(IServiceProvider provider) + { + var configuration = provider.GetRequiredService(); + var azureAuthorityHost = provider.GetRequiredService().AuthorityHost; + + return configuration.TryGetValue("AZURE_BEARER_TOKEN") + .Map(GetCredentialFromToken) + .IfNone(() => GetDefaultAzureCredential(azureAuthorityHost)); + } + + private static TokenCredential GetCredentialFromToken(string token) + { + var jsonWebToken = new JsonWebToken(token); + var expirationDate = new DateTimeOffset(jsonWebToken.ValidTo); + var accessToken = new AccessToken(token, expirationDate); + + return DelegatedTokenCredential.Create((context, cancellationToken) => accessToken); + } + + private static DefaultAzureCredential GetDefaultAzureCredential(Uri azureAuthorityHost) => + new(new DefaultAzureCredentialOptions + { + AuthorityHost = azureAuthorityHost + }); + + private static HttpPipeline GetHttpPipeline(IServiceProvider provider) + { + var clientOptions = ClientOptions.Default; + clientOptions.RetryPolicy = new CommonRetryPolicy(); + + var tokenCredential = provider.GetRequiredService(); + var azureEnvironment = provider.GetRequiredService(); + var bearerAuthenticationPolicy = new BearerTokenAuthenticationPolicy(tokenCredential, azureEnvironment.DefaultScope); + + var logger = provider.GetRequiredService().CreateLogger(nameof(HttpPipeline)); + var loggingPolicy = new ILoggerHttpPipelinePolicy(logger); + + var version = Assembly.GetExecutingAssembly()?.GetName().Version ?? new Version("-1"); + var telemetryPolicy = new TelemetryPolicy(version); + + return HttpPipelineBuilder.Build(clientOptions, bearerAuthenticationPolicy, loggingPolicy, telemetryPolicy); + } + + private static GetSubscriptionId GetSubscriptionId(IServiceProvider provider) => + () => provider.GetRequiredService().GetValue("AZURE_SUBSCRIPTION_ID"); + + private static GetResourceGroupName GetResourceGroupName(IServiceProvider provider) => + () => provider.GetRequiredService().GetValue("AZURE_RESOURCE_GROUP_NAME"); + + private static GetBearerToken GetBearerToken(IServiceProvider provider) + { + var tokenCredential = provider.GetRequiredService(); + var azureEnvironment = provider.GetRequiredService(); + var context = new TokenRequestContext([azureEnvironment.DefaultScope]); + + return async cancellationToken => + { + var token = await tokenCredential.GetTokenAsync(context, cancellationToken); + return token.Token; + }; + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/Configuration.cs b/tools/code/integration.tests/Configuration.cs index d508ad1e..dbf3a1ea 100644 --- a/tools/code/integration.tests/Configuration.cs +++ b/tools/code/integration.tests/Configuration.cs @@ -1,169 +1,169 @@ -using Azure.Core; -using Azure.Core.Pipeline; -using Azure.Identity; -using Azure.ResourceManager; -using common; -using Flurl; -using LanguageExt; -using LanguageExt.UnsafeValueAccess; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.IdentityModel.JsonWebTokens; -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace integration.tests; - -internal static class Configuration -{ - private static readonly Lazy @default = new(() => CreateDefault()); - public static IConfiguration Default => @default.Value; - - private static readonly Lazy azureEnvironment = new(() => GetAzureEnvironment(Default)); - private static AzureEnvironment AzureEnvironment => azureEnvironment.Value; - - private static readonly Lazy location = new(() => GetLocation(Default)); - public static string Location => location.Value; - - private static readonly Lazy apimProviderUri = new(() => GetApimProviderUri(Default, AzureEnvironment)); - public static Uri ApimProviderUri => apimProviderUri.Value; - - private static readonly Lazy managementServiceProviderUri = new(() => GetManagementServiceProviderUri(ApimProviderUri)); - public static Uri ManagementServiceProviderUri => managementServiceProviderUri.Value; - - private static readonly Lazy subscriptionId = new(() => GetSubscriptionId(Default)); - public static string SubscriptionId => subscriptionId.Value; - - private static readonly Lazy resourceGroupName = new(() => GetResourceGroupName(Default)); - public static string ResourceGroupName => resourceGroupName.Value; - - private static readonly Lazy tokenCredential = new(() => GetTokenCredential(AzureEnvironment.AuthorityHost, Default)); - private static TokenCredential TokenCredential => tokenCredential.Value; - - private static readonly Lazy httpPipeline = new(() => GetHttpPipeline(TokenCredential, AzureEnvironment)); - public static HttpPipeline HttpPipeline => httpPipeline.Value; - - private static readonly Lazy firstManagementServiceName = new(() => GetFirstManagementServiceName(Default)); - public static ManagementServiceName FirstServiceName => firstManagementServiceName.Value; - - private static readonly Lazy firstServiceUri = new(() => GetManagementServiceUri(FirstServiceName)); - public static ManagementServiceUri FirstServiceUri => firstServiceUri.Value; - - private static readonly Lazy secondManagementServiceName = new(() => GetSecondManagementServiceName(Default)); - public static ManagementServiceName SecondServiceName => secondManagementServiceName.Value; - - private static readonly Lazy secondServiceUri = new(() => GetManagementServiceUri(SecondServiceName)); - public static ManagementServiceUri SecondServiceUri => secondServiceUri.Value; - - private static IConfiguration CreateDefault() => - new ConfigurationBuilder().AddEnvironmentVariables() - .AddUserSecrets(typeof(Extractor).Assembly) - .Build(); - - private static AzureEnvironment GetAzureEnvironment(IConfiguration configuration) => - configuration.TryGetValue("AZURE_CLOUD_ENVIRONMENT").ValueUnsafe() switch - { - null => AzureEnvironment.Public, - "AzureGlobalCloud" or nameof(ArmEnvironment.AzurePublicCloud) => AzureEnvironment.Public, - "AzureChinaCloud" or nameof(ArmEnvironment.AzureChina) => AzureEnvironment.China, - "AzureUSGovernment" or nameof(ArmEnvironment.AzureGovernment) => AzureEnvironment.USGovernment, - "AzureGermanCloud" or nameof(ArmEnvironment.AzureGermany) => AzureEnvironment.Germany, - _ => throw new InvalidOperationException($"AZURE_CLOUD_ENVIRONMENT is invalid. Valid values are {nameof(ArmEnvironment.AzurePublicCloud)}, {nameof(ArmEnvironment.AzureChina)}, {nameof(ArmEnvironment.AzureGovernment)}, {nameof(ArmEnvironment.AzureGermany)}") - }; - - private static Uri GetApimProviderUri(IConfiguration configuration, AzureEnvironment azureEnvironment) - { - var apiVersion = configuration.TryGetValue("ARM_API_VERSION") - .IfNone(() => "2022-08-01"); - - return azureEnvironment.ManagementEndpoint - .AppendPathSegment("subscriptions") - .AppendPathSegment(GetSubscriptionId(configuration)) - .AppendPathSegment("resourceGroups") - .AppendPathSegment(GetResourceGroupName(configuration)) - .AppendPathSegment("providers/Microsoft.ApiManagement") - .SetQueryParam("api-version", apiVersion) - .ToUri(); - } - - private static Uri GetManagementServiceProviderUri(Uri apimProviderUri) => - apimProviderUri.AppendPathSegment("service").ToUri(); - - public static ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName) => - GetManagementServiceUri(serviceName, ManagementServiceProviderUri); - - private static ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName, Uri managementServiceProviderUri) - { - var uri = managementServiceProviderUri.AppendPathSegment(serviceName.ToString()) - .ToUri(); - - return ManagementServiceUri.From(uri); - } - - private static string GetLocation(IConfiguration configuration) => - configuration.TryGetValue("AZURE_LOCATION").IfNone("westus"); - - private static string GetSubscriptionId(IConfiguration configuration) => - configuration.GetValue("AZURE_SUBSCRIPTION_ID"); - - private static string GetResourceGroupName(IConfiguration configuration) => - configuration.GetValue("AZURE_RESOURCE_GROUP_NAME"); - - private static TokenCredential GetTokenCredential(Uri azureAuthorityHost, IConfiguration configuration) => - configuration.TryGetValue("AZURE_BEARER_TOKEN") - .Map(GetCredentialFromToken) - .IfNone(() => GetDefaultAzureCredential(azureAuthorityHost)); - - private static DefaultAzureCredential GetDefaultAzureCredential(Uri azureAuthorityHost) => - new(new DefaultAzureCredentialOptions - { - AuthorityHost = azureAuthorityHost, - ExcludeVisualStudioCredential = true - }); - - private static HttpPipeline GetHttpPipeline(TokenCredential tokenCredential, AzureEnvironment azureEnvironment) - { - var clientOptions = ClientOptions.Default; - clientOptions.RetryPolicy = new RetryPolicy(); - var bearerAuthenticationPolicy = new BearerTokenAuthenticationPolicy(tokenCredential, azureEnvironment.DefaultScope); - -#pragma warning disable CA2000 // Dispose objects before losing scope - var logger = LoggerFactory.Create(builder => - { - builder.AddDebug().AddConsole(); - builder.SetMinimumLevel(LogLevel.Trace); - }).CreateLogger(); -#pragma warning restore CA2000 // Dispose objects before losing scope - var loggingPolicy = new ILoggerHttpPipelinePolicy(logger); - - return HttpPipelineBuilder.Build(clientOptions, bearerAuthenticationPolicy, loggingPolicy); - } - - private static TokenCredential GetCredentialFromToken(string token) - { - var jsonWebToken = new JsonWebToken(token); - var expirationDate = new DateTimeOffset(jsonWebToken.ValidTo); - var accessToken = new AccessToken(token, expirationDate); - - return DelegatedTokenCredential.Create((context, cancellationToken) => accessToken); - } - - public static async ValueTask GetBearerToken(CancellationToken cancellationToken) => - await GetBearerToken(TokenCredential, AzureEnvironment, cancellationToken); - - private static async ValueTask GetBearerToken(TokenCredential tokenCredential, AzureEnvironment azureEnvironment, CancellationToken cancellationToken) - { - var context = new TokenRequestContext([azureEnvironment.DefaultScope]); - - var token = await tokenCredential.GetTokenAsync(context, cancellationToken); - - return token.Token; - } - - private static ManagementServiceName GetFirstManagementServiceName(IConfiguration configuration) => - ManagementServiceName.From(configuration.GetValue("FIRST_API_MANAGEMENT_SERVICE_NAME")); - - private static ManagementServiceName GetSecondManagementServiceName(IConfiguration configuration) => - ManagementServiceName.From(configuration.GetValue("SECOND_API_MANAGEMENT_SERVICE_NAME")); -} \ No newline at end of file +//using Azure.Core; +//using Azure.Core.Pipeline; +//using Azure.Identity; +//using Azure.ResourceManager; +//using common; +//using Flurl; +//using LanguageExt; +//using LanguageExt.UnsafeValueAccess; +//using Microsoft.Extensions.Configuration; +//using Microsoft.Extensions.Logging; +//using Microsoft.IdentityModel.JsonWebTokens; +//using System; +//using System.Threading; +//using System.Threading.Tasks; + +//namespace integration.tests; + +//internal static class Configuration +//{ +// private static readonly Lazy @default = new(() => CreateDefault()); +// public static IConfiguration Default => @default.Value; + +// private static readonly Lazy azureEnvironment = new(() => GetAzureEnvironment(Default)); +// private static AzureEnvironment AzureEnvironment => azureEnvironment.Value; + +// private static readonly Lazy location = new(() => GetLocation(Default)); +// public static string Location => location.Value; + +// private static readonly Lazy apimProviderUri = new(() => GetApimProviderUri(Default, AzureEnvironment)); +// public static Uri ApimProviderUri => apimProviderUri.Value; + +// private static readonly Lazy managementServiceProviderUri = new(() => GetManagementServiceProviderUri(ApimProviderUri)); +// public static Uri ManagementServiceProviderUri => managementServiceProviderUri.Value; + +// private static readonly Lazy subscriptionId = new(() => GetSubscriptionId(Default)); +// public static string SubscriptionId => subscriptionId.Value; + +// private static readonly Lazy resourceGroupName = new(() => GetResourceGroupName(Default)); +// public static string ResourceGroupName => resourceGroupName.Value; + +// private static readonly Lazy tokenCredential = new(() => GetTokenCredential(AzureEnvironment.AuthorityHost, Default)); +// private static TokenCredential TokenCredential => tokenCredential.Value; + +// private static readonly Lazy httpPipeline = new(() => GetHttpPipeline(TokenCredential, AzureEnvironment)); +// public static HttpPipeline HttpPipeline => httpPipeline.Value; + +// private static readonly Lazy firstManagementServiceName = new(() => GetFirstManagementServiceName(Default)); +// public static ManagementServiceName FirstServiceName => firstManagementServiceName.Value; + +// private static readonly Lazy firstServiceUri = new(() => GetManagementServiceUri(FirstServiceName)); +// public static ManagementServiceUri FirstServiceUri => firstServiceUri.Value; + +// private static readonly Lazy secondManagementServiceName = new(() => GetSecondManagementServiceName(Default)); +// public static ManagementServiceName SecondServiceName => secondManagementServiceName.Value; + +// private static readonly Lazy secondServiceUri = new(() => GetManagementServiceUri(SecondServiceName)); +// public static ManagementServiceUri SecondServiceUri => secondServiceUri.Value; + +// private static IConfiguration CreateDefault() => +// new ConfigurationBuilder().AddEnvironmentVariables() +// .AddUserSecrets(typeof(Extractor).Assembly) +// .Build(); + +// private static AzureEnvironment GetAzureEnvironment(IConfiguration configuration) => +// configuration.TryGetValue("AZURE_CLOUD_ENVIRONMENT").ValueUnsafe() switch +// { +// null => AzureEnvironment.Public, +// "AzureGlobalCloud" or nameof(ArmEnvironment.AzurePublicCloud) => AzureEnvironment.Public, +// "AzureChinaCloud" or nameof(ArmEnvironment.AzureChina) => AzureEnvironment.China, +// "AzureUSGovernment" or nameof(ArmEnvironment.AzureGovernment) => AzureEnvironment.USGovernment, +// "AzureGermanCloud" or nameof(ArmEnvironment.AzureGermany) => AzureEnvironment.Germany, +// _ => throw new InvalidOperationException($"AZURE_CLOUD_ENVIRONMENT is invalid. Valid values are {nameof(ArmEnvironment.AzurePublicCloud)}, {nameof(ArmEnvironment.AzureChina)}, {nameof(ArmEnvironment.AzureGovernment)}, {nameof(ArmEnvironment.AzureGermany)}") +// }; + +// private static Uri GetApimProviderUri(IConfiguration configuration, AzureEnvironment azureEnvironment) +// { +// var apiVersion = configuration.TryGetValue("ARM_API_VERSION") +// .IfNone(() => "2022-08-01"); + +// return azureEnvironment.ManagementEndpoint +// .AppendPathSegment("subscriptions") +// .AppendPathSegment(GetSubscriptionId(configuration)) +// .AppendPathSegment("resourceGroups") +// .AppendPathSegment(GetResourceGroupName(configuration)) +// .AppendPathSegment("providers/Microsoft.ApiManagement") +// .SetQueryParam("api-version", apiVersion) +// .ToUri(); +// } + +// private static Uri GetManagementServiceProviderUri(Uri apimProviderUri) => +// apimProviderUri.AppendPathSegment("service").ToUri(); + +// public static ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName) => +// GetManagementServiceUri(serviceName, ManagementServiceProviderUri); + +// private static ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName, Uri managementServiceProviderUri) +// { +// var uri = managementServiceProviderUri.AppendPathSegment(serviceName.ToString()) +// .ToUri(); + +// return ManagementServiceUri.From(uri); +// } + +// private static string GetLocation(IConfiguration configuration) => +// configuration.TryGetValue("AZURE_LOCATION").IfNone("westus"); + +// private static string GetSubscriptionId(IConfiguration configuration) => +// configuration.GetValue("AZURE_SUBSCRIPTION_ID"); + +// private static string GetResourceGroupName(IConfiguration configuration) => +// configuration.GetValue("AZURE_RESOURCE_GROUP_NAME"); + +// private static TokenCredential GetTokenCredential(Uri azureAuthorityHost, IConfiguration configuration) => +// configuration.TryGetValue("AZURE_BEARER_TOKEN") +// .Map(GetCredentialFromToken) +// .IfNone(() => GetDefaultAzureCredential(azureAuthorityHost)); + +// private static DefaultAzureCredential GetDefaultAzureCredential(Uri azureAuthorityHost) => +// new(new DefaultAzureCredentialOptions +// { +// AuthorityHost = azureAuthorityHost, +// ExcludeVisualStudioCredential = true +// }); + +// private static HttpPipeline GetHttpPipeline(TokenCredential tokenCredential, AzureEnvironment azureEnvironment) +// { +// var clientOptions = ClientOptions.Default; +// clientOptions.RetryPolicy = new RetryPolicy(); +// var bearerAuthenticationPolicy = new BearerTokenAuthenticationPolicy(tokenCredential, azureEnvironment.DefaultScope); + +//#pragma warning disable CA2000 // Dispose objects before losing scope +// var logger = LoggerFactory.Create(builder => +// { +// builder.AddDebug().AddConsole(); +// builder.SetMinimumLevel(LogLevel.Trace); +// }).CreateLogger(); +//#pragma warning restore CA2000 // Dispose objects before losing scope +// var loggingPolicy = new ILoggerHttpPipelinePolicy(logger); + +// return HttpPipelineBuilder.Build(clientOptions, bearerAuthenticationPolicy, loggingPolicy); +// } + +// private static TokenCredential GetCredentialFromToken(string token) +// { +// var jsonWebToken = new JsonWebToken(token); +// var expirationDate = new DateTimeOffset(jsonWebToken.ValidTo); +// var accessToken = new AccessToken(token, expirationDate); + +// return DelegatedTokenCredential.Create((context, cancellationToken) => accessToken); +// } + +// public static async ValueTask GetBearerToken(CancellationToken cancellationToken) => +// await GetBearerToken(TokenCredential, AzureEnvironment, cancellationToken); + +// private static async ValueTask GetBearerToken(TokenCredential tokenCredential, AzureEnvironment azureEnvironment, CancellationToken cancellationToken) +// { +// var context = new TokenRequestContext([azureEnvironment.DefaultScope]); + +// var token = await tokenCredential.GetTokenAsync(context, cancellationToken); + +// return token.Token; +// } + +// private static ManagementServiceName GetFirstManagementServiceName(IConfiguration configuration) => +// ManagementServiceName.From(configuration.GetValue("FIRST_API_MANAGEMENT_SERVICE_NAME")); + +// private static ManagementServiceName GetSecondManagementServiceName(IConfiguration configuration) => +// ManagementServiceName.From(configuration.GetValue("SECOND_API_MANAGEMENT_SERVICE_NAME")); +//} \ No newline at end of file diff --git a/tools/code/integration.tests/Diagnostic.cs b/tools/code/integration.tests/Diagnostic.cs index 7a157e80..fa563fc9 100644 --- a/tools/code/integration.tests/Diagnostic.cs +++ b/tools/code/integration.tests/Diagnostic.cs @@ -5,45 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Diagnostic +internal delegate ValueTask DeleteAllDiagnostics(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutDiagnosticModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedDiagnostics(Option> diagnosticNamesOption, Option> loggerNamesOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimDiagnostics(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileDiagnostics(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteDiagnosticModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedDiagnostics(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllDiagnosticsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(DiagnosticModel original) => - from alwaysLog in Gen.Const("allErrors").OptionOf() - from sampling in DiagnosticSampling.Generate().OptionOf() - select original with - { - AlwaysLog = alwaysLog, - Sampling = sampling - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllDiagnostics)); - public static Gen GenerateOverride(DiagnosticDto original) => - from alwaysLog in Gen.Const("allErrors").OptionOf() - from sampling in DiagnosticSampling.Generate().OptionOf() - select new DiagnosticDto + logger.LogInformation("Deleting all diagnostics in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await DiagnosticsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutDiagnosticModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutDiagnosticModels)); + + logger.LogInformation("Putting diagnostic models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new DiagnosticDto.DiagnosticContract - { - AlwaysLog = alwaysLog.ValueUnsafe(), - Sampling = sampling.Map(sampling => new DiagnosticDto.SamplingSettings - { - SamplingType = sampling.Type, - Percentage = sampling.Percentage - }).ValueUnsafe() - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(DiagnosticModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = DiagnosticUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static DiagnosticDto GetDto(DiagnosticModel model) => new() @@ -59,29 +81,118 @@ private static DiagnosticDto GetDto(DiagnosticModel model) => }).ValueUnsafe() } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedDiagnosticsHandler(ILogger logger, GetApimDiagnostics getApimResources, GetFileDiagnostics getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> diagnosticNamesOption, Option> loggerNamesOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedDiagnostics)); + + logger.LogInformation("Validating extracted diagnostics in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, diagnosticNamesOption)) + .WhereValue(dto => DiagnosticModule.TryGetLoggerName(dto) + .Map(name => ExtractorOptions.ShouldExtract(name, loggerNamesOption)) + .IfNone(true)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(DiagnosticDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + LoggerId = string.Join('/', dto.Properties.LoggerId?.Split('/')?.TakeLast(2)?.ToArray() ?? []), + AlwaysLog = dto.Properties.AlwaysLog ?? string.Empty, + Sampling = new + { + Type = dto.Properties.Sampling?.SamplingType ?? string.Empty, + Percentage = dto.Properties.Sampling?.Percentage ?? 0 + } + }.ToString()!; +} - private static async ValueTask Put(DiagnosticModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimDiagnosticsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = DiagnosticUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimDiagnostics)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting diagnostics from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = DiagnosticsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await DiagnosticsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileDiagnosticsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileDiagnostics)); + + logger.LogInformation("Getting diagnostics from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => DiagnosticInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, DiagnosticInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileDiagnostics)); + + logger.LogInformation("Getting diagnostics from {ServiceDirectory}...", serviceDirectory); + + return await DiagnosticModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => +file sealed class WriteDiagnosticModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteDiagnosticModels)); + + logger.LogInformation("Writing diagnostic models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(DiagnosticModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -91,33 +202,40 @@ private static async ValueTask WriteInformationFile(DiagnosticModel model, Manag await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static DiagnosticDto GetDto(DiagnosticModel model) => + new() + { + Properties = new DiagnosticDto.DiagnosticContract + { + LoggerId = $"/loggers/{model.LoggerName}", + AlwaysLog = model.AlwaysLog.ValueUnsafe(), + Sampling = model.Sampling.Map(sampling => new DiagnosticDto.SamplingSettings + { + SamplingType = sampling.Type, + Percentage = sampling.Percentage + }).ValueUnsafe() + } + }; +} + +file sealed class ValidatePublishedDiagnosticsHandler(ILogger logger, GetFileDiagnostics getFileResources, GetApimDiagnostics getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedDiagnostics)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published diagnostics in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = DiagnosticsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await DiagnosticModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(DiagnosticDto dto) => new { @@ -129,49 +247,107 @@ private static string NormalizeDto(DiagnosticDto dto) => Percentage = dto.Properties.Sampling?.Percentage ?? 0 } }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class DiagnosticServices +{ + public static void ConfigureDeleteAllDiagnostics(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutDiagnosticModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedDiagnostics(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimDiagnostics(services); + ConfigureGetFileDiagnostics(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => DiagnosticInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimDiagnostics(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, DiagnosticInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileDiagnostics(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteDiagnosticModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedDiagnostics(IServiceCollection services) + { + ConfigureGetFileDiagnostics(services); + ConfigureGetApimDiagnostics(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Diagnostic +{ + public static Gen GenerateUpdate(DiagnosticModel original) => + from alwaysLog in Gen.Const("allErrors").OptionOf() + from sampling in DiagnosticSampling.Generate().OptionOf() + select original with { - using (contents) + AlwaysLog = alwaysLog, + Sampling = sampling + }; + + public static Gen GenerateOverride(DiagnosticDto original) => + from alwaysLog in Gen.Const("allErrors").OptionOf() + from sampling in DiagnosticSampling.Generate().OptionOf() + select new DiagnosticDto + { + Properties = new DiagnosticDto.DiagnosticContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + AlwaysLog = alwaysLog.ValueUnsafe(), + Sampling = sampling.Map(sampling => new DiagnosticDto.SamplingSettings + { + SamplingType = sampling.Type, + Percentage = sampling.Percentage + }).ValueUnsafe() } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static DiagnosticDto GetDto(DiagnosticModel model) => + new() + { + Properties = new DiagnosticDto.DiagnosticContract + { + LoggerId = $"/loggers/{model.LoggerName}", + AlwaysLog = model.AlwaysLog.ValueUnsafe(), + Sampling = model.Sampling.Map(sampling => new DiagnosticDto.SamplingSettings + { + SamplingType = sampling.Type, + Percentage = sampling.Percentage + }).ValueUnsafe() + } + }; } diff --git a/tools/code/integration.tests/Extractor.cs b/tools/code/integration.tests/Extractor.cs index 78a3b8de..123426ad 100644 --- a/tools/code/integration.tests/Extractor.cs +++ b/tools/code/integration.tests/Extractor.cs @@ -3,9 +3,13 @@ using CsCheck; using extractor; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Linq.Expressions; @@ -16,43 +20,9 @@ namespace integration.tests; -internal static class Extractor -{ - public static async ValueTask Run(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, string subscriptionId, string resourceGroupName, string bearerToken, CancellationToken cancellationToken) - { - var argumentDictionary = new Dictionary - { - [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), - ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, - ["AZURE_SUBSCRIPTION_ID"] = subscriptionId, - ["AZURE_RESOURCE_GROUP_NAME"] = resourceGroupName, - ["AZURE_BEARER_TOKEN"] = bearerToken, - ["Logging:LogLevel:Default"] = "Trace" - }; - - var optionsJson = options.ToJsonObject(); - if (optionsJson.Count > 0) - { - var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.extractor.yaml"); - var yamlFile = new FileInfo(yamlFilePath); - await WriteYamlToFile(optionsJson, yamlFile, cancellationToken); - argumentDictionary.Add("CONFIGURATION_YAML_PATH", yamlFile.FullName); - } +internal delegate ValueTask RunExtractor(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); - var arguments = argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); - await extractor.Program.Main(arguments); - } - - private static string GetApiManagementServiceNameParameter() => - Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); - - private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) - { - var yaml = YamlConverter.Serialize(json); - var content = BinaryData.FromString(yaml); - await file.OverwriteWithBinaryData(content, cancellationToken); - } -} +internal delegate ValueTask ValidateExtractorArtifacts(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); internal sealed record ExtractorOptions { @@ -70,6 +40,23 @@ internal sealed record ExtractorOptions public required Option DefaultApiSpecification { get; init; } public required Option> SubscriptionNamesToExport { get; init; } + public static ExtractorOptions NoFilter { get; } = new() + { + ApiNamesToExport = Option>.None, + BackendNamesToExport = Option>.None, + DefaultApiSpecification = Option.None, + DiagnosticNamesToExport = Option>.None, + GatewayNamesToExport = Option>.None, + GroupNamesToExport = Option>.None, + LoggerNamesToExport = Option>.None, + NamedValueNamesToExport = Option>.None, + PolicyFragmentNamesToExport = Option>.None, + ProductNamesToExport = Option>.None, + SubscriptionNamesToExport = Option>.None, + TagNamesToExport = Option>.None, + VersionSetNamesToExport = Option>.None + }; + public static Gen Generate(ServiceModel service) => from namedValues in GenerateOptionalNamesToExport(service.NamedValues) from tags in GenerateOptionalNamesToExport(service.Tags) @@ -192,7 +179,129 @@ public static bool ShouldExtract(T name, Option> namesToExport) // Run T.From(nameToFindString) var nameToFind = Expression.Lambda>(Expression.Call(typeof(T), "From", [], Expression.Constant(nameToFindString))).Compile()(); - + return names.Contains(nameToFind); }, () => true); } + +file sealed class RunExtractorHandler(ILogger logger, + ActivitySource activitySource, + GetSubscriptionId getSubscriptionId, + GetResourceGroupName getResourceGroupName, + GetBearerToken getBearerToken) +{ + public async ValueTask Handle(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(RunExtractor)); + + logger.LogInformation("Running extractor..."); + + var configurationFileOption = await TryGetConfigurationYamlFile(options, serviceDirectory, cancellationToken); + var arguments = await GetArguments(serviceName, serviceDirectory, configurationFileOption, cancellationToken); + await extractor.Program.Main(arguments); + } + + private static async ValueTask> TryGetConfigurationYamlFile(ExtractorOptions extractorOptions, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var optionsJson = extractorOptions.ToJsonObject(); + if (optionsJson.Count == 0) + { + return Option.None; + } + + var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.extractor.yaml"); + var yamlFile = new FileInfo(yamlFilePath); + await WriteYamlToFile(optionsJson, yamlFile, cancellationToken); + + return yamlFile; + } + + private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) + { + var yaml = YamlConverter.Serialize(json); + var content = BinaryData.FromString(yaml); + await file.OverwriteWithBinaryData(content, cancellationToken); + } + + private async ValueTask GetArguments(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, Option configurationFileOption, CancellationToken cancellationToken) + { + var argumentDictionary = new Dictionary + { + [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), + ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, + ["AZURE_SUBSCRIPTION_ID"] = getSubscriptionId(), + ["AZURE_RESOURCE_GROUP_NAME"] = getResourceGroupName(), + ["AZURE_BEARER_TOKEN"] = await getBearerToken(cancellationToken) + }; + +#pragma warning disable CA1849 // Call async methods when in an async method + configurationFileOption.Iter(file => argumentDictionary.Add("CONFIGURATION_YAML_PATH", file.FullName)); +#pragma warning restore CA1849 // Call async methods when in an async method + + return argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); + } + + private static string GetApiManagementServiceNameParameter() => + Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); +} + +file sealed class ValidateExtractorArtifactsHandler(ILogger logger, + ActivitySource activitySource, + ValidateExtractedNamedValues validateNamedValues, + ValidateExtractedTags validateTags, + ValidateExtractedVersionSets validateVersionSets, + ValidateExtractedBackends validateBackends, + ValidateExtractedLoggers validateLoggers, + ValidateExtractedDiagnostics validateDiagnostics, + ValidateExtractedPolicyFragments validatePolicyFragments, + ValidateExtractedServicePolicies validateServicePolicies, + ValidateExtractedGroups validateGroups, + ValidateExtractedProducts validateProducts, + ValidateExtractedApis validateApis) +{ + public async ValueTask Handle(ExtractorOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractorArtifacts)); + + logger.LogInformation("Validating extractor artifacts..."); + + await validateNamedValues(options.NamedValueNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateTags(options.TagNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateVersionSets(options.VersionSetNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateBackends(options.BackendNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateLoggers(options.LoggerNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateDiagnostics(options.DiagnosticNamesToExport, options.LoggerNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validatePolicyFragments(options.PolicyFragmentNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateServicePolicies(serviceName, serviceDirectory, cancellationToken); + await validateGroups(options.GroupNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateProducts(options.ProductNamesToExport, serviceName, serviceDirectory, cancellationToken); + await validateApis(options.ApiNamesToExport, options.DefaultApiSpecification, options.VersionSetNamesToExport, serviceName, serviceDirectory, cancellationToken); + } +} + +internal static class ExtractorServices +{ + public static void ConfigureRunExtractor(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidateExtractorArtifacts(IServiceCollection services) + { + NamedValueServices.ConfigureValidateExtractedNamedValues(services); + TagServices.ConfigureValidateExtractedTags(services); + VersionSetServices.ConfigureValidateExtractedVersionSets(services); + BackendServices.ConfigureValidateExtractedBackends(services); + LoggerServices.ConfigureValidateExtractedLoggers(services); + DiagnosticServices.ConfigureValidateExtractedDiagnostics(services); + PolicyFragmentServices.ConfigureValidateExtractedPolicyFragments(services); + ServicePolicyServices.ConfigureValidateExtractedServicePolicies(services); + GroupServices.ConfigureValidateExtractedGroups(services); + ProductServices.ConfigureValidateExtractedProducts(services); + ApiServices.ConfigureValidateExtractedApis(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/Fixture.cs b/tools/code/integration.tests/Fixture.cs index f7adc76c..1da9bfe0 100644 --- a/tools/code/integration.tests/Fixture.cs +++ b/tools/code/integration.tests/Fixture.cs @@ -1,4 +1,4 @@ -using common; +using common; using common.tests; using CsCheck; using LanguageExt; @@ -177,4 +177,4 @@ private sealed record ChangeParameters public Option MaxSize { get; init; } = Option.None; } -} +} \ No newline at end of file diff --git a/tools/code/integration.tests/Gateway.cs b/tools/code/integration.tests/Gateway.cs index d9796d05..0641b886 100644 --- a/tools/code/integration.tests/Gateway.cs +++ b/tools/code/integration.tests/Gateway.cs @@ -5,16 +5,45 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; +internal delegate ValueTask DeleteAllGateways(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file sealed class DeleteAllGatewaysHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllGateways)); + + logger.LogInformation("Deleting all gateways in {ServiceName}.", serviceName); + var serviceUri = getServiceUri(serviceName); + await GatewaysUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +internal static class GatewayServices +{ + public static void ConfigureDeleteAllGateways(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + internal static class Gateway { public static Gen GenerateUpdate(GatewayModel original) => diff --git a/tools/code/integration.tests/Group.cs b/tools/code/integration.tests/Group.cs index 5d5d1a91..e158119a 100644 --- a/tools/code/integration.tests/Group.cs +++ b/tools/code/integration.tests/Group.cs @@ -5,41 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Group +internal delegate ValueTask DeleteAllGroups(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutGroupModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedGroups(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimGroups(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileGroups(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteGroupModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedGroups(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllGroupsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(GroupModel original) => - from displayName in GroupModel.GenerateDisplayName() - from description in GroupModel.GenerateDescription().OptionOf() - select original with - { - DisplayName = displayName, - Description = description - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllGroups)); - public static Gen GenerateOverride(GroupDto original) => - from displayName in GroupModel.GenerateDisplayName() - from description in GroupModel.GenerateDescription().OptionOf() - select new GroupDto + logger.LogInformation("Deleting all groups in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await GroupsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutGroupModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutGroupModels)); + + logger.LogInformation("Putting group models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new GroupDto.GroupContract - { - DisplayName = displayName, - Description = description.ValueUnsafe() - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(GroupModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = GroupUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static GroupDto GetDto(GroupModel model) => new() @@ -50,29 +76,110 @@ private static GroupDto GetDto(GroupModel model) => Description = model.Description.ValueUnsafe() } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedGroupsHandler(ILogger logger, GetApimGroups getApimResources, GetFileGroups getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedGroups)); + + logger.LogInformation("Validating extracted groups in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(GroupDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(GroupModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimGroupsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = GroupUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimGroups)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting groups from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = GroupsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await GroupsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileGroupsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileGroups)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Getting groups from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => GroupInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, GroupInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileGroups)); + + logger.LogInformation("Getting groups from {ServiceDirectory}...", serviceDirectory); + + return await GroupModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteGroupModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteGroupModels)); + + logger.LogInformation("Writing group models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(GroupModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -82,86 +189,135 @@ private static async ValueTask WriteInformationFile(GroupModel model, Management await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static GroupDto GetDto(GroupModel model) => + new() + { + Properties = new GroupDto.GroupContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe() + } + }; +} + +file sealed class ValidatePublishedGroupsHandler(ILogger logger, GetFileGroups getFileResources, GetApimGroups getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedGroups)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published groups in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = GroupsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .Where(kvp => (kvp.Value.Properties.Type?.Contains("system", StringComparison.OrdinalIgnoreCase) ?? false) is false) + .MapValue(NormalizeDto); + var actual = apimResources.Where(kvp => (kvp.Value.Properties.Type?.Contains("system", StringComparison.OrdinalIgnoreCase) ?? false) is false) + .MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await GroupModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(GroupDto dto) => new { DisplayName = dto.Properties.DisplayName ?? string.Empty, Description = dto.Properties.Description ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class GroupServices +{ + public static void ConfigureDeleteAllGroups(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutGroupModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .Where(kvp => (kvp.Value.Properties.Type?.Contains("system", StringComparison.OrdinalIgnoreCase) ?? false) is false) - .MapValue(NormalizeDto); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - var actual = apimResources.Where(kvp => (kvp.Value.Properties.Type?.Contains("system", StringComparison.OrdinalIgnoreCase) ?? false) is false) - .MapValue(NormalizeDto); + public static void ConfigureValidateExtractedGroups(IServiceCollection services) + { + ConfigureGetApimGroups(services); + ConfigureGetFileGroups(services); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetApimGroups(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => GroupInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetFileGroups(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, GroupInformationFile file, CancellationToken cancellationToken) + public static void ConfigureWriteGroupModels(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureValidatePublishedGroups(IServiceCollection services) + { + ConfigureGetFileGroups(services); + ConfigureGetApimGroups(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Group +{ + public static Gen GenerateUpdate(GroupModel original) => + from displayName in GroupModel.GenerateDisplayName() + from description in GroupModel.GenerateDescription().OptionOf() + select original with { - using (contents) + DisplayName = displayName, + Description = description + }; + + public static Gen GenerateOverride(GroupDto original) => + from displayName in GroupModel.GenerateDisplayName() + from description in GroupModel.GenerateDescription().OptionOf() + select new GroupDto + { + Properties = new GroupDto.GroupContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + DisplayName = displayName, + Description = description.ValueUnsafe() } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static GroupDto GetDto(GroupModel model) => + new() + { + Properties = new GroupDto.GroupContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe() + } + }; } diff --git a/tools/code/integration.tests/Logger.cs b/tools/code/integration.tests/Logger.cs index 302d57aa..cea70b0d 100644 --- a/tools/code/integration.tests/Logger.cs +++ b/tools/code/integration.tests/Logger.cs @@ -5,10 +5,14 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text.Json.Nodes; using System.Threading; @@ -16,33 +20,53 @@ namespace integration.tests; -internal static class Logger +internal delegate ValueTask DeleteAllLoggers(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutLoggerModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedLoggers(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimLoggers(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileLoggers(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteLoggerModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedLoggers(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllLoggersHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(LoggerModel original) => - from type in LoggerType.Generate() - from description in LoggerModel.GenerateDescription().OptionOf() - from isBuffered in Gen.Bool - select original with - { - Type = type, - Description = description, - IsBuffered = isBuffered - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllLoggers)); - public static Gen GenerateOverride(LoggerDto original) => - from description in LoggerModel.GenerateDescription().OptionOf() - from isBuffered in Gen.Bool - select new LoggerDto + logger.LogInformation("Deleting all loggers in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await LoggersUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutLoggerModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutLoggerModels)); + + logger.LogInformation("Putting logger models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new LoggerDto.LoggerContract - { - Description = description.ValueUnsafe(), - IsBuffered = isBuffered - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(LoggerModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = LoggerUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static LoggerDto GetDto(LoggerModel model) => new() @@ -79,29 +103,118 @@ private static LoggerDto GetDto(LoggerModel model) => } } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedLoggersHandler(ILogger logger, GetApimLoggers getApimResources, GetFileLoggers getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedLoggers)); + + logger.LogInformation("Validating extracted loggers in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(LoggerDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + LoggerType = dto.Properties.LoggerType ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty, + IsBuffered = dto.Properties.IsBuffered ?? false, + ResourceId = dto.Properties.ResourceId ?? string.Empty, + Credentials = new + { + Name = dto.Properties.Credentials?.TryGetStringProperty("name").ValueUnsafe() ?? string.Empty, + ConnectionString = dto.Properties.Credentials?.TryGetStringProperty("connectionString").ValueUnsafe() ?? string.Empty, + InstrumentationKey = dto.Properties.Credentials?.TryGetStringProperty("instrumentationKey").ValueUnsafe() ?? string.Empty + } + }.ToString()!; +} - private static async ValueTask Put(LoggerModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimLoggersHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = LoggerUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimLoggers)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting loggers from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = LoggersUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class GetFileLoggersHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileLoggers)); + + logger.LogInformation("Getting loggers from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => LoggerInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); } - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await LoggersUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, LoggerInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileLoggers)); + + logger.LogInformation("Getting loggers from {ServiceDirectory}...", serviceDirectory); + + return await LoggerModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteLoggerModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteLoggerModels)); + + logger.LogInformation("Writing logger models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(LoggerModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -111,33 +224,61 @@ private static async ValueTask WriteInformationFile(LoggerModel model, Managemen await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static LoggerDto GetDto(LoggerModel model) => + new() + { + Properties = new LoggerDto.LoggerContract + { + LoggerType = model.Type switch + { + LoggerType.ApplicationInsights => "applicationInsights", + LoggerType.AzureMonitor => "azureMonitor", + LoggerType.EventHub => "azureEventHub", + _ => throw new ArgumentException($"Model type '{model.Type}' is not supported.", nameof(model)) + }, + Description = model.Description.ValueUnsafe(), + IsBuffered = model.IsBuffered, + ResourceId = model.Type switch + { + LoggerType.ApplicationInsights applicationInsights => applicationInsights.ResourceId, + LoggerType.EventHub eventHub => eventHub.ResourceId, + _ => null + }, + Credentials = model.Type switch + { + LoggerType.ApplicationInsights applicationInsights => new JsonObject + { + ["instrumentationKey"] = $"{{{{{applicationInsights.InstrumentationKeyNamedValue}}}}}" + }, + LoggerType.EventHub eventHub => new JsonObject + { + ["name"] = eventHub.Name, + ["connectionString"] = $"{{{{{eventHub.ConnectionStringNamedValue}}}}}" + }, + _ => null + } + } + }; +} + +file sealed class ValidatePublishedLoggersHandler(ILogger logger, GetFileLoggers getFileResources, GetApimLoggers getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedLoggers)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published loggers in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = LoggersUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await LoggerModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(LoggerDto dto) => new { @@ -152,49 +293,126 @@ private static string NormalizeDto(LoggerDto dto) => InstrumentationKey = dto.Properties.Credentials?.TryGetStringProperty("instrumentationKey").ValueUnsafe() ?? string.Empty } }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class LoggerServices +{ + public static void ConfigureDeleteAllLoggers(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutLoggerModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedLoggers(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimLoggers(services); + ConfigureGetFileLoggers(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => LoggerInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimLoggers(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, LoggerInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileLoggers(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteLoggerModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedLoggers(IServiceCollection services) + { + ConfigureGetFileLoggers(services); + ConfigureGetApimLoggers(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Logger +{ + public static Gen GenerateUpdate(LoggerModel original) => + from type in LoggerType.Generate() + from description in LoggerModel.GenerateDescription().OptionOf() + from isBuffered in Gen.Bool + select original with { - using (contents) + Type = type, + Description = description, + IsBuffered = isBuffered + }; + + public static Gen GenerateOverride(LoggerDto original) => + from description in LoggerModel.GenerateDescription().OptionOf() + from isBuffered in Gen.Bool + select new LoggerDto + { + Properties = new LoggerDto.LoggerContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + Description = description.ValueUnsafe(), + IsBuffered = isBuffered } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static LoggerDto GetDto(LoggerModel model) => + new() + { + Properties = new LoggerDto.LoggerContract + { + LoggerType = model.Type switch + { + LoggerType.ApplicationInsights => "applicationInsights", + LoggerType.AzureMonitor => "azureMonitor", + LoggerType.EventHub => "azureEventHub", + _ => throw new ArgumentException($"Model type '{model.Type}' is not supported.", nameof(model)) + }, + Description = model.Description.ValueUnsafe(), + IsBuffered = model.IsBuffered, + ResourceId = model.Type switch + { + LoggerType.ApplicationInsights applicationInsights => applicationInsights.ResourceId, + LoggerType.EventHub eventHub => eventHub.ResourceId, + _ => null + }, + Credentials = model.Type switch + { + LoggerType.ApplicationInsights applicationInsights => new JsonObject + { + ["instrumentationKey"] = $"{{{{{applicationInsights.InstrumentationKeyNamedValue}}}}}" + }, + LoggerType.EventHub eventHub => new JsonObject + { + ["name"] = eventHub.Name, + ["connectionString"] = $"{{{{{eventHub.ConnectionStringNamedValue}}}}}" + }, + _ => null + } + } + }; } diff --git a/tools/code/integration.tests/ManagementService.cs b/tools/code/integration.tests/ManagementService.cs new file mode 100644 index 00000000..22926bed --- /dev/null +++ b/tools/code/integration.tests/ManagementService.cs @@ -0,0 +1,462 @@ +using Azure.Core; +using Azure.Core.Pipeline; +using common; +using common.tests; +using Flurl; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Polly; +using publisher; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace integration.tests; + +internal delegate IAsyncEnumerable ListApimServiceNames(CancellationToken cancellationToken); + +internal delegate ValueTask DeleteApimService(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask CreateApimService(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask EmptyApimService(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutServiceModel(ServiceModel serviceModel, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ManagementServiceUri GetManagementServiceUri(ManagementServiceName serviceName); + +internal delegate ValueTask WriteServiceModelArtifacts(ServiceModel serviceModel, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask> WriteServiceModelCommits(IEnumerable serviceModels, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed record ManagementServiceProviderUri : ResourceUri +{ + private readonly Uri uri; + + public ManagementServiceProviderUri(Uri uri) + { + this.uri = uri; + } + + protected override Uri Value => uri; +} + +file sealed class ListApimServiceNamesHandler(ILogger logger, + ActivitySource activitySource, + HttpPipeline pipeline, + ManagementServiceProviderUri serviceProviderUri) +{ + public async IAsyncEnumerable Handle([EnumeratorCancellation] CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ListApimServiceNames)); + + logger.LogInformation("Listing APIM service names..."); + + var serviceNames = pipeline.ListJsonObjects(serviceProviderUri.ToUri(), cancellationToken) + .Choose(json => json.TryGetStringProperty("name").ToOption()) + .Select(ManagementServiceName.From); + + await foreach (var serviceName in serviceNames.WithCancellation(cancellationToken)) + { + yield return serviceName; + } + } +} + +file sealed class DeleteApimServiceHandler(ILogger logger, + ActivitySource activitySource, + HttpPipeline pipeline, + ManagementServiceProviderUri serviceProviderUri) +{ + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteApimService)); + + logger.LogInformation("Deleting APIM service {ServiceName}...", serviceName); + + var uri = serviceProviderUri.ToUri().AppendPathSegment(serviceName.Value).ToUri(); + + try + { + await pipeline.DeleteResource(uri, waitForCompletion: false, cancellationToken); + } + catch (HttpRequestException exception) when (exception.StatusCode == HttpStatusCode.Conflict && exception.Message.Contains("ServiceLocked", StringComparison.OrdinalIgnoreCase)) + { + } + } +} + +file sealed class CreateApimServiceHandler(ILogger logger, + ActivitySource activitySource, + HttpPipeline pipeline, + ManagementServiceProviderUri serviceProviderUri, + AzureLocation location) +{ + private static readonly ResiliencePipeline resiliencePipeline = + new ResiliencePipelineBuilder() + .AddRetry(new() + { + ShouldHandle = async arguments => + { + await ValueTask.CompletedTask; + var result = arguments.Outcome.Result; + var succeeded = "Succeeded".Equals(result, StringComparison.OrdinalIgnoreCase); + return succeeded is false; + }, + Delay = TimeSpan.FromSeconds(5), + BackoffType = DelayBackoffType.Linear, + MaxRetryAttempts = 100 + }) + .AddTimeout(TimeSpan.FromMinutes(3)) + .Build(); + + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(CreateApimService)); + + logger.LogInformation("Creating APIM service {ServiceName}...", serviceName); + + var uri = serviceProviderUri.ToUri().AppendPathSegment(serviceName.Value).ToUri(); + var body = BinaryData.FromObjectAsJson(new + { + location = location.Name, + sku = new + { + name = "StandardV2", + capacity = 1 + }, + identity = new + { + type = "SystemAssigned" + }, + properties = new + { + publisherEmail = "admin@contoso.com", + publisherName = "Contoso" + } + }); + + await pipeline.PutContent(uri, body, cancellationToken); + + // Wait until the service is successfully provisioned + await resiliencePipeline.ExecuteAsync(async cancellationToken => + { + var content = await pipeline.GetJsonObject(uri, cancellationToken); + + return content.TryGetJsonObjectProperty("properties") + .Bind(properties => properties.TryGetStringProperty("provisioningState")) + .IfLeft(string.Empty); + }, cancellationToken); + } +} + +file sealed class EmptyApimServiceHandler(ILogger logger, + ActivitySource activitySource, + DeleteAllSubscriptions deleteSubscriptions, + DeleteAllApis deleteApis, + DeleteAllGroups deleteGroups, + DeleteAllProducts deleteProducts, + DeleteAllServicePolicies deleteServicePolicies, + DeleteAllPolicyFragments deletePolicyFragments, + DeleteAllDiagnostics deleteDiagnostics, + DeleteAllLoggers deleteLoggers, + DeleteAllBackends deleteBackends, + DeleteAllVersionSets deleteVersionSets, + DeleteAllGateways deleteGateways, + DeleteAllTags deleteTags, + DeleteAllNamedValues deleteNamedValues) +{ + private static ResiliencePipeline resiliencePipeline = + new ResiliencePipelineBuilder() + .AddRetry(new() + { + BackoffType = DelayBackoffType.Constant, + UseJitter = true, + MaxRetryAttempts = 3, + ShouldHandle = new PredicateBuilder().Handle(exception => exception.StatusCode == HttpStatusCode.PreconditionFailed && exception.Message.Contains("Resource was modified since last retrieval", StringComparison.OrdinalIgnoreCase)) + }) + .Build(); + + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(EmptyApimService)); + + logger.LogInformation("Emptying APIM service {ServiceName}...", serviceName); + + await resiliencePipeline.ExecuteAsync(async cancellationToken => + { + await deleteSubscriptions(serviceName, cancellationToken); + await deleteApis(serviceName, cancellationToken); + await deleteGroups(serviceName, cancellationToken); + await deleteProducts(serviceName, cancellationToken); + await deleteServicePolicies(serviceName, cancellationToken); + await deletePolicyFragments(serviceName, cancellationToken); + await deleteDiagnostics(serviceName, cancellationToken); + await deleteLoggers(serviceName, cancellationToken); + await deleteBackends(serviceName, cancellationToken); + await deleteVersionSets(serviceName, cancellationToken); + await deleteGateways(serviceName, cancellationToken); + await deleteTags(serviceName, cancellationToken); + await deleteNamedValues(serviceName, cancellationToken); + }, cancellationToken); + } +} + +file sealed class PutServiceModelHandler(ILogger logger, + ActivitySource activitySource, + PutNamedValueModels putNamedValues, + PutTagModels putTags, + PutVersionSetModels putVersionSets, + PutBackendModels putBackends, + PutLoggerModels putLoggers, + PutDiagnosticModels putDiagnostics, + PutPolicyFragmentModels putPolicyFragments, + PutServicePolicyModels putServicePolicies, + PutGroupModels putGroups, + PutProductModels putProducts, + PutApiModels putApis) +{ + public async ValueTask Handle(ServiceModel serviceModel, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutServiceModel)); + + logger.LogInformation("Putting service model in APIM service {ServiceName}...", serviceName); + + await putNamedValues(serviceModel.NamedValues, serviceName, cancellationToken); + await putTags(serviceModel.Tags, serviceName, cancellationToken); + await putVersionSets(serviceModel.VersionSets, serviceName, cancellationToken); + await putBackends(serviceModel.Backends, serviceName, cancellationToken); + await putLoggers(serviceModel.Loggers, serviceName, cancellationToken); + await putDiagnostics(serviceModel.Diagnostics, serviceName, cancellationToken); + await putPolicyFragments(serviceModel.PolicyFragments, serviceName, cancellationToken); + await putServicePolicies(serviceModel.ServicePolicies, serviceName, cancellationToken); + await putGroups(serviceModel.Groups, serviceName, cancellationToken); + await putProducts(serviceModel.Products, serviceName, cancellationToken); + await putApis(serviceModel.Apis, serviceName, cancellationToken); + } +} + +file sealed class GetManagementServiceUriHandler(ManagementServiceProviderUri serviceProviderUri) +{ + public ManagementServiceUri Handle(ManagementServiceName serviceName) => + ManagementServiceUri.From(serviceProviderUri.ToUri() + .AppendPathSegment(serviceName.Value) + .ToUri()); +} + +file sealed class WriteServiceModelArtifactsHandler(ILogger logger, + ActivitySource activitySource, + WriteNamedValueModels writeNamedValues, + WriteTagModels writeTags, + WriteVersionSetModels writeVersionSets, + WriteBackendModels writeBackends, + WriteLoggerModels writeLoggers, + WriteDiagnosticModels writeDiagnostics, + WritePolicyFragmentModels writePolicyFragments, + WriteServicePolicyModels writeServicePolicies, + WriteGroupModels writeGroups, + WriteProductModels writeProducts, + WriteApiModels writeApis) +{ + public async ValueTask Handle(ServiceModel serviceModel, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteServiceModelArtifacts)); + + logger.LogInformation("Writing service model artifacts to {ServiceDirectory}...", serviceDirectory); + + await writeNamedValues(serviceModel.NamedValues, serviceDirectory, cancellationToken); + await writeTags(serviceModel.Tags, serviceDirectory, cancellationToken); + await writeVersionSets(serviceModel.VersionSets, serviceDirectory, cancellationToken); + await writeBackends(serviceModel.Backends, serviceDirectory, cancellationToken); + await writeLoggers(serviceModel.Loggers, serviceDirectory, cancellationToken); + await writeDiagnostics(serviceModel.Diagnostics, serviceDirectory, cancellationToken); + await writePolicyFragments(serviceModel.PolicyFragments, serviceDirectory, cancellationToken); + await writeServicePolicies(serviceModel.ServicePolicies, serviceDirectory, cancellationToken); + await writeGroups(serviceModel.Groups, serviceDirectory, cancellationToken); + await writeProducts(serviceModel.Products, serviceDirectory, cancellationToken); + await writeApis(serviceModel.Apis, serviceDirectory, cancellationToken); + } +} + +file sealed class WriteServiceModelCommitsHandler(ILogger logger, + ActivitySource activitySource, + WriteServiceModelArtifacts writeServiceModelArtifacts) +{ + public async ValueTask> Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteServiceModelCommits)); + + logger.LogInformation("Writing service model commits to {ServiceDirectory}...", serviceDirectory); + + var authorName = "apiops"; + var authorEmail = "apiops@apiops.com"; + var serviceDirectoryInfo = serviceDirectory.ToDirectoryInfo(); + Git.InitializeRepository(serviceDirectoryInfo, commitMessage: "Initial commit", authorName, authorEmail, DateTimeOffset.UtcNow); + + var commitIds = ImmutableArray.Empty; + await models.Map((index, model) => (index, model)) + .Iter(async x => + { + var (index, model) = x; + DeleteNonGitDirectories(serviceDirectory); + await writeServiceModelArtifacts(model, serviceDirectory, cancellationToken); + var commit = Git.CommitChanges(serviceDirectoryInfo, commitMessage: $"Commit {index}", authorName, authorEmail, DateTimeOffset.UtcNow); + var commitId = new CommitId(commit.Sha); + ImmutableInterlocked.Update(ref commitIds, commitIds => commitIds.Add(commitId)); + }, cancellationToken); + + return commitIds; + } + + private static void DeleteNonGitDirectories(ManagementServiceDirectory serviceDirectory) => + serviceDirectory.ToDirectoryInfo() + .ListDirectories("*") + .Where(directory => directory.Name.Equals(".git", StringComparison.OrdinalIgnoreCase) is false) + .Iter(directory => directory.ForceDelete()); +} + +internal static class ManagementServices +{ + public static void ConfigureListApimServiceNames(IServiceCollection services) + { + ConfigureManagementServiceProviderUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureManagementServiceProviderUri(IServiceCollection services) + { + services.TryAddSingleton(provider => + { + var configuration = provider.GetRequiredService(); + var apiVersion = configuration.TryGetValue("ARM_API_VERSION") + .IfNone(() => "2022-08-01"); + + var azureEnvironment = provider.GetRequiredService(); + var subscriptionId = provider.GetRequiredService().Invoke(); + var resourceGroupName = provider.GetRequiredService().Invoke(); + var uri = azureEnvironment.ManagementEndpoint + .AppendPathSegment("subscriptions") + .AppendPathSegment(subscriptionId) + .AppendPathSegment("resourceGroups") + .AppendPathSegment(resourceGroupName) + .AppendPathSegment("providers/Microsoft.ApiManagement/service") + .SetQueryParam("api-version", apiVersion) + .ToUri(); + + return new ManagementServiceProviderUri(uri); + }); + } + + public static void ConfigureDeleteApimService(IServiceCollection services) + { + ConfigureManagementServiceProviderUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureCreateApimService(IServiceCollection services) + { + ConfigureManagementServiceProviderUri(services); + ConfigureAzureLocation(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureAzureLocation(IServiceCollection services) + { + services.TryAddSingleton(typeof(AzureLocation), provider => + { + var configuration = provider.GetRequiredService(); + var locationName = configuration.TryGetValue("AZURE_LOCATION") + .IfNone("westus"); + + return new AzureLocation(locationName); + }); + } + + public static void ConfigureEmptyApimService(IServiceCollection services) + { + SubscriptionServices.ConfigureDeleteAllSubscriptions(services); + ApiServices.ConfigureDeleteAllApis(services); + GroupServices.ConfigureDeleteAllGroups(services); + ProductServices.ConfigureDeleteAllProducts(services); + ServicePolicyServices.ConfigureDeleteAllServicePolicies(services); + PolicyFragmentServices.ConfigureDeleteAllPolicyFragments(services); + DiagnosticServices.ConfigureDeleteAllDiagnostics(services); + LoggerServices.ConfigureDeleteAllLoggers(services); + BackendServices.ConfigureDeleteAllBackends(services); + VersionSetServices.ConfigureDeleteAllVersionSets(services); + GatewayServices.ConfigureDeleteAllGateways(services); + TagServices.ConfigureDeleteAllTags(services); + NamedValueServices.ConfigureDeleteAllNamedValues(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigurePutServiceModel(IServiceCollection services) + { + NamedValueServices.ConfigurePutNamedValueModels(services); + TagServices.ConfigurePutTagModels(services); + VersionSetServices.ConfigurePutVersionSetModels(services); + BackendServices.ConfigurePutBackendModels(services); + ApiServices.ConfigurePutApiModels(services); + LoggerServices.ConfigurePutLoggerModels(services); + DiagnosticServices.ConfigurePutDiagnosticModels(services); + PolicyFragmentServices.ConfigurePutPolicyFragmentModels(services); + ServicePolicyServices.ConfigurePutServicePolicyModels(services); + GroupServices.ConfigurePutGroupModels(services); + ProductServices.ConfigurePutProductModels(services); + ApiServices.ConfigurePutApiModels(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureGetManagementServiceUri(IServiceCollection services) + { + ConfigureManagementServiceProviderUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureWriteServiceModelArtifacts(IServiceCollection services) + { + NamedValueServices.ConfigureWriteNamedValueModels(services); + TagServices.ConfigureWriteTagModels(services); + VersionSetServices.ConfigureWriteVersionSetModels(services); + BackendServices.ConfigureWriteBackendModels(services); + LoggerServices.ConfigureWriteLoggerModels(services); + DiagnosticServices.ConfigureWriteDiagnosticModels(services); + PolicyFragmentServices.ConfigureWritePolicyFragmentModels(services); + ServicePolicyServices.ConfigureWriteServicePolicyModels(services); + GroupServices.ConfigureWriteGroupModels(services); + ProductServices.ConfigureWriteProductModels(services); + ApiServices.ConfigureWriteApiModels(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureWriteServiceModelCommits(IServiceCollection services) + { + ConfigureWriteServiceModelArtifacts(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/NamedValue.cs b/tools/code/integration.tests/NamedValue.cs index 69b14f62..b327e0f8 100644 --- a/tools/code/integration.tests/NamedValue.cs +++ b/tools/code/integration.tests/NamedValue.cs @@ -5,41 +5,69 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class NamedValue +internal delegate ValueTask DeleteAllNamedValues(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutNamedValueModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedNamedValues(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimNamedValues(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileNamedValues(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteNamedValueModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedNamedValues(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllNamedValuesHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(NamedValueModel original) => - from tags in NamedValueModel.GenerateTags() - select original with - { - Tags = tags - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllNamedValues)); - public static Gen GenerateOverride(NamedValueDto original) => - from value in Generator.AlphaNumericStringBetween(1, 100) - from tags in NamedValueModel.GenerateTags() - select new NamedValueDto + logger.LogInformation("Deleting all named values in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await NamedValuesUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutNamedValueModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutNamedValueModels)); + + logger.LogInformation("Putting named value models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new NamedValueDto.NamedValueContract - { - Value = value, - Tags = tags - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetPublisherDto); + private async ValueTask Put(NamedValueModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = NamedValueUri.From(model.Name, serviceUri); + var dto = GetDto(model); - private static NamedValueDto GetPublisherDto(NamedValueModel model) => + await uri.PutDto(dto, pipeline, cancellationToken); + } + + private static NamedValueDto GetDto(NamedValueModel model) => new() { Properties = new NamedValueDto.NamedValueContract @@ -64,39 +92,122 @@ private static NamedValueDto GetPublisherDto(NamedValueModel model) => } } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedNamedValuesHandler(ILogger logger, GetApimNamedValues getApimResources, GetFileNamedValues getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedNamedValues)); + + logger.LogInformation("Validating extracted named values in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(NamedValueDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Tags = string.Join(',', (dto.Properties.Tags ?? []).Order()), + Value = dto.Properties.Secret is true ? string.Empty : dto.Properties.Value ?? string.Empty, + Secret = dto.Properties.Secret + }.ToString()!; +} - private static async ValueTask Put(NamedValueModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimNamedValuesHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = NamedValueUri.From(model.Name, serviceUri); - var dto = GetPublisherDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimNamedValues)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting named values from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = NamedValuesUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await NamedValuesUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileNamedValuesHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileNamedValues)); + + logger.LogInformation("Getting named values from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => NamedValueInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, NamedValueInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileNamedValues)); + + logger.LogInformation("Getting named values from {ServiceDirectory}...", serviceDirectory); + + return await NamedValueModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteNamedValueModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteNamedValueModels)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Writing named value models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(NamedValueModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { var informationFile = NamedValueInformationFile.From(model.Name, serviceDirectory); - var dto = GetExtractorDto(model); + var dto = GetDto(model); await informationFile.WriteDto(dto, cancellationToken); } - private static NamedValueDto GetExtractorDto(NamedValueModel model) => + private static NamedValueDto GetDto(NamedValueModel model) => new() { Properties = new NamedValueDto.NamedValueContract @@ -116,34 +227,29 @@ private static NamedValueDto GetExtractorDto(NamedValueModel model) => Value = model.Type is NamedValueType.Default @default ? @default.Value : null } }; +} - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class ValidatePublishedNamedValuesHandler(ILogger logger, GetFileNamedValues getFileResources, GetApimNamedValues getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedNamedValues)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published named values in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = NamedValuesUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .WhereValue(dto => (dto.Properties.Secret is true + && dto.Properties.Value is null + && dto.Properties.KeyVault?.SecretIdentifier is null) is false) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await NamedValueModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(NamedValueDto dto) => new { @@ -152,52 +258,112 @@ private static string NormalizeDto(NamedValueDto dto) => Value = dto.Properties.Secret is true ? string.Empty : dto.Properties.Value ?? string.Empty, Secret = dto.Properties.Secret }.ToString()!; +} + +internal static class NamedValueServices +{ + public static void ConfigureDeleteAllNamedValues(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutNamedValueModels(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedNamedValues(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ConfigureGetApimNamedValues(services); + ConfigureGetFileNamedValues(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .WhereValue(dto => (dto.Properties.Secret is true - && dto.Properties.Value is null - && dto.Properties.KeyVault?.SecretIdentifier is null) is false) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetApimNamedValues(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => NamedValueInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetFileNamedValues(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, NamedValueInformationFile file, CancellationToken cancellationToken) + public static void ConfigureWriteNamedValueModels(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureValidatePublishedNamedValues(IServiceCollection services) + { + ConfigureGetFileNamedValues(services); + ConfigureGetApimNamedValues(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class NamedValue +{ + public static Gen GenerateUpdate(NamedValueModel original) => + from tags in NamedValueModel.GenerateTags() + select original with { - using (contents) + Tags = tags + }; + + public static Gen GenerateOverride(NamedValueDto original) => + from value in Generator.AlphaNumericStringBetween(1, 100) + from tags in NamedValueModel.GenerateTags() + select new NamedValueDto + { + Properties = new NamedValueDto.NamedValueContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + Value = value, + Tags = tags } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static NamedValueDto GetDto(NamedValueModel model) => + new() + { + Properties = new NamedValueDto.NamedValueContract + { + DisplayName = model.Name.ToString(), + Tags = model.Tags, + KeyVault = model.Type switch + { + NamedValueType.KeyVault keyVault => new NamedValueDto.KeyVaultContract + { + SecretIdentifier = keyVault.SecretIdentifier, + IdentityClientId = keyVault.IdentityClientId.ValueUnsafe() + }, + _ => null + }, + Secret = model.Type is NamedValueType.Secret or NamedValueType.KeyVault, + Value = model.Type switch + { + NamedValueType.Secret secret => secret.Value, + NamedValueType.Default @default => @default.Value, + _ => null + } + } + }; } diff --git a/tools/code/integration.tests/PolicyFragment.cs b/tools/code/integration.tests/PolicyFragment.cs index 5b511a95..12a0213c 100644 --- a/tools/code/integration.tests/PolicyFragment.cs +++ b/tools/code/integration.tests/PolicyFragment.cs @@ -5,42 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class PolicyFragment +internal delegate ValueTask DeleteAllPolicyFragments(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutPolicyFragmentModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedPolicyFragments(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimPolicyFragments(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFilePolicyFragments(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WritePolicyFragmentModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedPolicyFragments(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllPolicyFragmentsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(PolicyFragmentModel original) => - from description in PolicyFragmentModel.GenerateDescription().OptionOf() - from content in PolicyFragmentModel.GenerateContent() - select original with - { - Description = description, - Content = content - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllPolicyFragments)); - public static Gen GenerateOverride(PolicyFragmentDto original) => - from description in PolicyFragmentModel.GenerateDescription().OptionOf() - from content in PolicyFragmentModel.GenerateContent() - select new PolicyFragmentDto + logger.LogInformation("Deleting all policy fragments in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await PolicyFragmentsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutPolicyFragmentModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutPolicyFragmentModels)); + + logger.LogInformation("Putting policy fragment models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new PolicyFragmentDto.PolicyFragmentContract - { - Description = description.ValueUnsafe(), - Format = "rawxml", - Value = content - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(PolicyFragmentModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = PolicyFragmentUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static PolicyFragmentDto GetDto(PolicyFragmentModel model) => new() @@ -52,30 +77,110 @@ private static PolicyFragmentDto GetDto(PolicyFragmentModel model) => Value = model.Content } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedPolicyFragmentsHandler(ILogger logger, GetApimPolicyFragments getApimResources, GetFilePolicyFragments getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedPolicyFragments)); + + logger.LogInformation("Validating extracted policy fragments in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(PolicyFragmentDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + Description = dto.Properties.Description ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(PolicyFragmentModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimPolicyFragmentsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = PolicyFragmentUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimPolicyFragments)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting policy fragments from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = PolicyFragmentsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await PolicyFragmentsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFilePolicyFragmentsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFilePolicyFragments)); + + logger.LogInformation("Getting policy fragments from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => PolicyFragmentInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, PolicyFragmentInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFilePolicyFragments)); + + logger.LogInformation("Getting policy fragments from {ServiceDirectory}...", serviceDirectory); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + return await PolicyFragmentModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WritePolicyFragmentModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WritePolicyFragmentModels)); + + logger.LogInformation("Writing policy fragment models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); await WritePolicyFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(PolicyFragmentModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -85,87 +190,141 @@ private static async ValueTask WriteInformationFile(PolicyFragmentModel model, M await informationFile.WriteDto(dto, cancellationToken); } + private static PolicyFragmentDto GetDto(PolicyFragmentModel model) => + new() + { + Properties = new PolicyFragmentDto.PolicyFragmentContract + { + Description = model.Description.ValueUnsafe(), + Format = "rawxml", + Value = model.Content + } + }; + private static async ValueTask WritePolicyFile(PolicyFragmentModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { var policyFile = PolicyFragmentPolicyFile.From(model.Name, serviceDirectory); await policyFile.WritePolicy(model.Content, cancellationToken); } +} - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class ValidatePublishedPolicyFragmentsHandler(ILogger logger, GetFilePolicyFragments getFileResources, GetApimPolicyFragments getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedPolicyFragments)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published policy fragments in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = PolicyFragmentsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await PolicyFragmentModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(PolicyFragmentDto dto) => new { Description = dto.Properties.Description ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class PolicyFragmentServices +{ + public static void ConfigureDeleteAllPolicyFragments(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutPolicyFragmentModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedPolicyFragments(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimPolicyFragments(services); + ConfigureGetFilePolicyFragments(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => PolicyFragmentInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimPolicyFragments(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, PolicyFragmentInformationFile file, CancellationToken cancellationToken) + private static void ConfigureGetFilePolicyFragments(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWritePolicyFragmentModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedPolicyFragments(IServiceCollection services) + { + ConfigureGetFilePolicyFragments(services); + ConfigureGetApimPolicyFragments(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class PolicyFragment +{ + public static Gen GenerateUpdate(PolicyFragmentModel original) => + from description in PolicyFragmentModel.GenerateDescription().OptionOf() + from content in PolicyFragmentModel.GenerateContent() + select original with { - using (contents) + Description = description, + Content = content + }; + + public static Gen GenerateOverride(PolicyFragmentDto original) => + from description in PolicyFragmentModel.GenerateDescription().OptionOf() + from content in PolicyFragmentModel.GenerateContent() + select new PolicyFragmentDto + { + Properties = new PolicyFragmentDto.PolicyFragmentContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + Description = description.ValueUnsafe(), + Format = "rawxml", + Value = content } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static PolicyFragmentDto GetDto(PolicyFragmentModel model) => + new() + { + Properties = new PolicyFragmentDto.PolicyFragmentContract + { + Description = model.Description.ValueUnsafe(), + Format = "rawxml", + Value = model.Content + } + }; } diff --git a/tools/code/integration.tests/Product.cs b/tools/code/integration.tests/Product.cs index 79dc03a7..a32557ad 100644 --- a/tools/code/integration.tests/Product.cs +++ b/tools/code/integration.tests/Product.cs @@ -5,49 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Product +internal delegate ValueTask DeleteAllProducts(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutProductModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedProducts(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimProducts(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileProducts(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteProductModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedProducts(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllProductsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(ProductModel original) => - from displayName in ProductModel.GenerateDisplayName() - from description in ProductModel.GenerateDescription().OptionOf() - from terms in ProductModel.GenerateTerms().OptionOf() - from state in ProductModel.GenerateState() - select original with - { - DisplayName = displayName, - Description = description, - Terms = terms, - State = state - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllProducts)); - public static Gen GenerateOverride(ProductDto original) => - from displayName in ProductModel.GenerateDisplayName() - from description in ProductModel.GenerateDescription().OptionOf() - from terms in ProductModel.GenerateTerms().OptionOf() - from state in ProductModel.GenerateState() - select new ProductDto + logger.LogInformation("Deleting all products in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await ProductsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutProductModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutProductModels)); + + logger.LogInformation("Putting product models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new ProductDto.ProductContract - { - DisplayName = displayName, - Description = description.ValueUnsafe(), - Terms = terms.ValueUnsafe(), - State = state - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(ProductModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = ProductUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static ProductDto GetDto(ProductModel model) => new() @@ -60,39 +78,112 @@ private static ProductDto GetDto(ProductModel model) => State = model.State } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedProductsHandler(ILogger logger, GetApimProducts getApimResources, GetFileProducts getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedProducts)); + + logger.LogInformation("Validating extracted products in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(ProductDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty, + Terms = dto.Properties.Terms ?? string.Empty, + State = dto.Properties.State ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(ProductModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimProductsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = ProductUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimProducts)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting products from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = ProductsUri.From(serviceUri); - await ProductPolicy.Put(model.Policies, model.Name, serviceUri, pipeline, cancellationToken); - await ProductGroup.Put(model.Groups, model.Name, serviceUri, pipeline, cancellationToken); - await ProductTag.Put(model.Tags, model.Name, serviceUri, pipeline, cancellationToken); - await ProductApi.Put(model.Apis, model.Name, serviceUri, pipeline, cancellationToken); + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await ProductsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileProductsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileProducts)); + + logger.LogInformation("Getting products from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => ProductInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ProductInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileProducts)); + + logger.LogInformation("Getting products from {ServiceDirectory}...", serviceDirectory); + + return await ProductModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteProductModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteProductModels)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Writing product models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); - - await ProductPolicy.WriteArtifacts(model.Policies, model.Name, serviceDirectory, cancellationToken); - await ProductGroup.WriteArtifacts(model.Groups, model.Name, serviceDirectory, cancellationToken); - await ProductTag.WriteArtifacts(model.Tags, model.Name, serviceDirectory, cancellationToken); - await ProductApi.WriteArtifacts(model.Apis, model.Name, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(ProductModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -102,41 +193,37 @@ private static async ValueTask WriteInformationFile(ProductModel model, Manageme await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + private static ProductDto GetDto(ProductModel model) => + new() + { + Properties = new ProductDto.ProductContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe(), + Terms = model.Terms.ValueUnsafe(), + State = model.State + } + }; +} - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); +file sealed class ValidatePublishedProductsHandler(ILogger logger, GetFileProducts getFileResources, GetApimProducts getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidatePublishedProducts)); - actual.Should().BeEquivalentTo(expected); + logger.LogInformation("Validating published products in {ServiceDirectory}...", serviceDirectory); - await expected.Keys.IterParallel(async productName => - { - await ProductPolicy.ValidateExtractedArtifacts(serviceDirectory, productName, serviceUri, pipeline, cancellationToken); - await ProductGroup.ValidateExtractedArtifacts(serviceDirectory, productName, serviceUri, pipeline, cancellationToken); - await ProductTag.ValidateExtractedArtifacts(serviceDirectory, productName, serviceUri, pipeline, cancellationToken); - await ProductApi.ValidateExtractedArtifacts(serviceDirectory, productName, serviceUri, pipeline, cancellationToken); - }, cancellationToken); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = ProductsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await ProductModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(ProductDto dto) => new { @@ -145,65 +232,108 @@ private static string NormalizeDto(ProductDto dto) => Terms = dto.Properties.Terms ?? string.Empty, State = dto.Properties.State ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class ProductServices +{ + public static void ConfigureDeleteAllProducts(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - await fileResources.Keys.IterParallel(async productName => - { - await ProductPolicy.ValidatePublisherChanges(productName, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductGroup.ValidatePublisherChanges(productName, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductTag.ValidatePublisherChanges(productName, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductApi.ValidatePublisherChanges(productName, serviceDirectory, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutProductModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedProducts(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimProducts(services); + ConfigureGetFileProducts(services); - await fileResources.Keys.IterParallel(async productName => - { - await ProductPolicy.ValidatePublisherCommitChanges(productName, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductGroup.ValidatePublisherCommitChanges(productName, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductTag.ValidatePublisherCommitChanges(productName, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ProductApi.ValidatePublisherCommitChanges(productName, commitId, serviceDirectory, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => ProductInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimProducts(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ProductInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileProducts(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteProductModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedProducts(IServiceCollection services) + { + ConfigureGetFileProducts(services); + ConfigureGetApimProducts(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Product +{ + public static Gen GenerateUpdate(ProductModel original) => + from displayName in ProductModel.GenerateDisplayName() + from description in ProductModel.GenerateDescription().OptionOf() + from terms in ProductModel.GenerateTerms().OptionOf() + from state in ProductModel.GenerateState() + select original with { - using (contents) + DisplayName = displayName, + Description = description, + Terms = terms, + State = state + }; + + public static Gen GenerateOverride(ProductDto original) => + from displayName in ProductModel.GenerateDisplayName() + from description in ProductModel.GenerateDescription().OptionOf() + from terms in ProductModel.GenerateTerms().OptionOf() + from state in ProductModel.GenerateState() + select new ProductDto + { + Properties = new ProductDto.ProductContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + DisplayName = displayName, + Description = description.ValueUnsafe(), + Terms = terms.ValueUnsafe(), + State = state } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static ProductDto GetDto(ProductModel model) => + new() + { + Properties = new ProductDto.ProductContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe(), + Terms = model.Terms.ValueUnsafe(), + State = model.State + } + }; } diff --git a/tools/code/integration.tests/Program.cs b/tools/code/integration.tests/Program.cs new file mode 100644 index 00000000..d2a4f0fc --- /dev/null +++ b/tools/code/integration.tests/Program.cs @@ -0,0 +1,65 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Threading.Tasks; + +namespace integration.tests; + +internal static class Program +{ + public static async Task Main(string[] args) + { + var builder = GetHostBuilder(args); + using var host = builder.Build(); + await Run(host); + } + + private static HostApplicationBuilder GetHostBuilder(string[] arguments) + { + var builder = Host.CreateApplicationBuilder(arguments); + Configure(builder); + return builder; + } + + private static void Configure(HostApplicationBuilder builder) + { + Configure(builder.Configuration); + Configure(builder.Services); + } + + private static void Configure(IConfigurationBuilder builder) + { + builder.AddUserSecrets(typeof(Program).Assembly); ; + } + + private static void Configure(IServiceCollection services) + { + CommonServices.Configure(services); + AppServices.ConfigureRunApplication(services); + } + + private static async Task Run(IHost host) + { + var applicationLifetime = host.Services.GetRequiredService(); + var cancellationToken = applicationLifetime.ApplicationStopping; + + try + { + await host.StartAsync(cancellationToken); + var runApplication = host.Services.GetRequiredService(); + await runApplication(cancellationToken); + } + catch (Exception exception) + { + var logger = host.Services.GetRequiredService().CreateLogger(nameof(Program)); + logger.LogCritical(exception, "An unhandled exception occurred."); + throw; + } + finally + { + applicationLifetime.StopApplication(); + } + } +} \ No newline at end of file diff --git a/tools/code/integration.tests/Publisher.cs b/tools/code/integration.tests/Publisher.cs index 55b013f3..372b217f 100644 --- a/tools/code/integration.tests/Publisher.cs +++ b/tools/code/integration.tests/Publisher.cs @@ -2,10 +2,14 @@ using common.tests; using CsCheck; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Linq.Expressions; @@ -16,51 +20,9 @@ namespace integration.tests; -internal static class Publisher -{ - public static async ValueTask Run(PublisherOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, string subscriptionId, string resourceGroupName, string bearerToken, Option commitId, CancellationToken cancellationToken) - { - var argumentDictionary = new Dictionary - { - [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), - ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, - ["AZURE_SUBSCRIPTION_ID"] = subscriptionId, - ["AZURE_RESOURCE_GROUP_NAME"] = resourceGroupName, - ["AZURE_BEARER_TOKEN"] = bearerToken, - ["Logging:LogLevel:Default"] = "Trace" - }; +internal delegate ValueTask RunPublisher(PublisherOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); -#pragma warning disable CA1849 // Call async methods when in an async method - commitId.Iter(id => argumentDictionary.Add("COMMIT_ID", id.Value)); -#pragma warning restore CA1849 // Call async methods when in an async method - - var yamlFile = await WriteConfigurationYaml(options, serviceDirectory, cancellationToken); - argumentDictionary.Add("CONFIGURATION_YAML_PATH", yamlFile.FullName); - - var arguments = argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); - await Program.Main(arguments); - } - - private static string GetApiManagementServiceNameParameter() => - Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); - - private static async ValueTask WriteConfigurationYaml(PublisherOptions options, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) - { - var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.publisher.yaml"); - var yamlFile = new FileInfo(yamlFilePath); - var json = options.ToJsonObject(); - await WriteYamlToFile(json, yamlFile, cancellationToken); - - return yamlFile; - } - - private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) - { - var yaml = YamlConverter.Serialize(json); - var content = BinaryData.FromString(yaml); - await file.OverwriteWithBinaryData(content, cancellationToken); - } -} +internal delegate ValueTask ValidatePublishedArtifacts(PublisherOptions options, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); internal sealed record PublisherOptions { @@ -178,3 +140,172 @@ public static FrozenDictionary Override(IDictionary logger, + ActivitySource activitySource, + GetSubscriptionId getSubscriptionId, + GetResourceGroupName getResourceGroupName, + GetBearerToken getBearerToken) +{ + public async ValueTask Handle(PublisherOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(RunPublisher)); + + logger.LogInformation("Running publisher..."); + + var configurationFileOption = await TryGetConfigurationYamlFile(options, serviceDirectory, cancellationToken); + var arguments = await GetArguments(serviceName, serviceDirectory, configurationFileOption, commitIdOption, cancellationToken); + await publisher.Program.Main(arguments); + } + + private static async ValueTask> TryGetConfigurationYamlFile(PublisherOptions options, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var optionsJson = options.ToJsonObject(); + if (optionsJson.Count == 0) + { + return Option.None; + } + + var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.publisher.yaml"); + var yamlFile = new FileInfo(yamlFilePath); + await WriteYamlToFile(optionsJson, yamlFile, cancellationToken); + + return yamlFile; + } + + private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) + { + var yaml = YamlConverter.Serialize(json); + var content = BinaryData.FromString(yaml); + await file.OverwriteWithBinaryData(content, cancellationToken); + } + + private async ValueTask GetArguments(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, Option configurationFileOption, Option commitIdOption, CancellationToken cancellationToken) + { + var argumentDictionary = new Dictionary + { + [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), + ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, + ["AZURE_SUBSCRIPTION_ID"] = getSubscriptionId(), + ["AZURE_RESOURCE_GROUP_NAME"] = getResourceGroupName(), + ["AZURE_BEARER_TOKEN"] = await getBearerToken(cancellationToken) + }; + +#pragma warning disable CA1849 // Call async methods when in an async method + configurationFileOption.Iter(file => argumentDictionary.Add("CONFIGURATION_YAML_PATH", file.FullName)); + commitIdOption.Iter(id => argumentDictionary.Add("COMMIT_ID", id.Value)); +#pragma warning restore CA1849 // Call async methods when in an async method + + return argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); + } + + private static string GetApiManagementServiceNameParameter() => + Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); +} + +file sealed class ValidatePublishedArtifactsHandler(ILogger logger, + ActivitySource activitySource, + ValidatePublishedNamedValues validateNamedValues, + ValidatePublishedTags validateTags, + ValidatePublishedVersionSets validateVersionSets, + ValidatePublishedBackends validateBackends, + ValidatePublishedLoggers validateLoggers, + ValidatePublishedDiagnostics validateDiagnostics, + ValidatePublishedPolicyFragments validatePolicyFragments, + ValidatePublishedServicePolicies validateServicePolicies, + ValidatePublishedGroups validateGroups, + ValidatePublishedProducts validateProducts, + ValidatePublishedApis validateApis) +{ + public async ValueTask Handle(PublisherOptions options, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidatePublishedArtifacts)); + + logger.LogInformation("Validating published artifacts..."); + + await validateNamedValues(options.NamedValueOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateTags(options.TagOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateVersionSets(options.VersionSetOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateBackends(options.BackendOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateLoggers(options.LoggerOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateDiagnostics(options.DiagnosticOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validatePolicyFragments(options.PolicyFragmentOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateServicePolicies(options.ServicePolicyOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateGroups(options.GroupOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateProducts(options.ProductOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + await validateApis(options.ApiOverrides, commitIdOption, serviceName, serviceDirectory, cancellationToken); + } +} + +internal static class PublisherServices +{ + public static void ConfigureRunPublisher(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedArtifacts(IServiceCollection services) + { + NamedValueServices.ConfigureValidatePublishedNamedValues(services); + TagServices.ConfigureValidatePublishedTags(services); + VersionSetServices.ConfigureValidatePublishedVersionSets(services); + BackendServices.ConfigureValidatePublishedBackends(services); + LoggerServices.ConfigureValidatePublishedLoggers(services); + DiagnosticServices.ConfigureValidatePublishedDiagnostics(services); + PolicyFragmentServices.ConfigureValidatePublishedPolicyFragments(services); + ServicePolicyServices.ConfigureValidatePublishedServicePolicies(services); + GroupServices.ConfigureValidatePublishedGroups(services); + ProductServices.ConfigureValidatePublishedProducts(services); + ApiServices.ConfigureValidatePublishedApis(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Publisher +{ + public static async ValueTask Run(PublisherOptions options, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, string subscriptionId, string resourceGroupName, string bearerToken, Option commitId, CancellationToken cancellationToken) + { + var argumentDictionary = new Dictionary + { + [$"{GetApiManagementServiceNameParameter()}"] = serviceName.ToString(), + ["API_MANAGEMENT_SERVICE_OUTPUT_FOLDER_PATH"] = serviceDirectory.ToDirectoryInfo().FullName, + ["AZURE_SUBSCRIPTION_ID"] = subscriptionId, + ["AZURE_RESOURCE_GROUP_NAME"] = resourceGroupName, + ["AZURE_BEARER_TOKEN"] = bearerToken, + ["Logging:LogLevel:Default"] = "Trace" + }; + +#pragma warning disable CA1849 // Call async methods when in an async method + commitId.Iter(id => argumentDictionary.Add("COMMIT_ID", id.Value)); +#pragma warning restore CA1849 // Call async methods when in an async method + + var yamlFile = await WriteConfigurationYaml(options, serviceDirectory, cancellationToken); + argumentDictionary.Add("CONFIGURATION_YAML_PATH", yamlFile.FullName); + + var arguments = argumentDictionary.Aggregate(Array.Empty(), (arguments, kvp) => [.. arguments, $"--{kvp.Key}", kvp.Value]); + await Program.Main(arguments); + } + + private static string GetApiManagementServiceNameParameter() => + Gen.OneOfConst("API_MANAGEMENT_SERVICE_NAME", "apimServiceName").Single(); + + private static async ValueTask WriteConfigurationYaml(PublisherOptions options, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var yamlFilePath = Path.Combine(serviceDirectory.ToDirectoryInfo().FullName, "configuration.publisher.yaml"); + var yamlFile = new FileInfo(yamlFilePath); + var json = options.ToJsonObject(); + await WriteYamlToFile(json, yamlFile, cancellationToken); + + return yamlFile; + } + + private static async ValueTask WriteYamlToFile(JsonNode json, FileInfo file, CancellationToken cancellationToken) + { + var yaml = YamlConverter.Serialize(json); + var content = BinaryData.FromString(yaml); + await file.OverwriteWithBinaryData(content, cancellationToken); + } +} diff --git a/tools/code/integration.tests/Service.cs b/tools/code/integration.tests/Service.cs deleted file mode 100644 index 72a036e1..00000000 --- a/tools/code/integration.tests/Service.cs +++ /dev/null @@ -1,266 +0,0 @@ -using Azure.Core.Pipeline; -using common; -using common.tests; -using Flurl; -using LanguageExt; -using Polly; -using publisher; -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace integration.tests; - -internal static class ServiceModule -{ - public static async ValueTask DeleteManagementServices(Uri serviceProviderUri, string serviceNamesToDeletePrefix, HttpPipeline pipeline, CancellationToken cancellationToken) - { - try - { - await pipeline.ListJsonObjects(serviceProviderUri, cancellationToken) - .Choose(json => json.TryGetStringProperty("name").ToOption()) - .Where(name => name.StartsWith(serviceNamesToDeletePrefix, StringComparison.OrdinalIgnoreCase)) - .Select(name => ManagementServiceUri.From(serviceProviderUri.AppendPathSegment(name).ToUri())) - .IterParallel(async uri => await DeleteManagementService(uri, pipeline, cancellationToken), cancellationToken); - } - catch (HttpRequestException) - { - return; - } - } - - public static async ValueTask DeleteManagementService(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await pipeline.DeleteResource(serviceUri.ToUri(), waitForCompletion: false, cancellationToken); - - public static async ValueTask CreateManagementService(ManagementServiceUri serviceUri, string location, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var body = BinaryData.FromObjectAsJson(new - { - location = location, - sku = new - { - name = "StandardV2", - capacity = 1 - }, - identity = new - { - type = "SystemAssigned" - }, - properties = new - { - publisherEmail = "admin@contoso.com", - publisherName = "Contoso" - } - }); - - await pipeline.PutContent(serviceUri.ToUri(), body, cancellationToken); - - // Wait until the service is successfully provisioned - var resiliencePipeline = GetCreationStatusResiliencePipeline(); - await resiliencePipeline.ExecuteAsync(async cancellationToken => - { - var content = await pipeline.GetJsonObject(serviceUri.ToUri(), cancellationToken); - - return content.TryGetJsonObjectProperty("properties") - .Bind(properties => properties.TryGetStringProperty("provisioningState")) - .IfLeft(string.Empty); - }, cancellationToken); - } - - private static ResiliencePipeline GetCreationStatusResiliencePipeline() => - new ResiliencePipelineBuilder() - .AddRetry(new() - { - ShouldHandle = async arguments => - { - await ValueTask.CompletedTask; - var result = arguments.Outcome.Result; - var succeeded = "Succeeded".Equals(result, StringComparison.OrdinalIgnoreCase); - return succeeded is false; - }, - Delay = TimeSpan.FromSeconds(5), - BackoffType = DelayBackoffType.Linear, - MaxRetryAttempts = 100 - }) - .AddTimeout(TimeSpan.FromMinutes(3)) - .Build(); - - public static async ValueTask Put(ServiceModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, bool putSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.Put(model.NamedValues, serviceUri, pipeline, cancellationToken); - await Tag.Put(model.Tags, serviceUri, pipeline, cancellationToken); - await VersionSet.Put(model.VersionSets, serviceUri, pipeline, cancellationToken); - await Backend.Put(model.Backends, serviceUri, pipeline, cancellationToken); - await Logger.Put(model.Loggers, serviceUri, pipeline, cancellationToken); - await Diagnostic.Put(model.Diagnostics, serviceUri, pipeline, cancellationToken); - await PolicyFragment.Put(model.PolicyFragments, serviceUri, pipeline, cancellationToken); - await Group.Put(model.Groups, serviceUri, pipeline, cancellationToken); - await Api.Put(model.Apis, serviceUri, pipeline, cancellationToken); - await ServicePolicy.Put(model.ServicePolicies, serviceUri, pipeline, cancellationToken); - await Product.Put(model.Products, serviceUri, pipeline, cancellationToken); - - if (putSpecialSkuResources) - { - await Gateway.Put(model.Gateways, serviceUri, pipeline, cancellationToken); - } - - await Subscription.Put(model.Subscriptions, serviceUri, pipeline, cancellationToken); - } - - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await ResiliencePipelines.DeletePolicy.ExecuteAsync(async cancellationToken => - { - await Subscription.DeleteAll(serviceUri, pipeline, cancellationToken); - await Api.DeleteAll(serviceUri, pipeline, cancellationToken); - await Group.DeleteAll(serviceUri, pipeline, cancellationToken); - await Product.DeleteAll(serviceUri, pipeline, cancellationToken); - await ServicePolicy.DeleteAll(serviceUri, pipeline, cancellationToken); - await PolicyFragment.DeleteAll(serviceUri, pipeline, cancellationToken); - await Diagnostic.DeleteAll(serviceUri, pipeline, cancellationToken); - await Logger.DeleteAll(serviceUri, pipeline, cancellationToken); - await Backend.DeleteAll(serviceUri, pipeline, cancellationToken); - await VersionSet.DeleteAll(serviceUri, pipeline, cancellationToken); - await Gateway.DeleteAll(serviceUri, pipeline, cancellationToken); - await Tag.DeleteAll(serviceUri, pipeline, cancellationToken); - await NamedValue.DeleteAll(serviceUri, pipeline, cancellationToken); - }, cancellationToken); - - public static void DeleteServiceDirectory(ManagementServiceDirectory serviceDirectory) - { - var directoryInfo = serviceDirectory.ToDirectoryInfo(); - - if (directoryInfo.Exists()) - { - directoryInfo.ForceDelete(); - } - } - - public static async ValueTask ValidateExtractedArtifacts(ExtractorOptions extractorOptions, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, bool validateSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.ValidateExtractedArtifacts(extractorOptions.NamedValueNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Tag.ValidateExtractedArtifacts(extractorOptions.TagNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await VersionSet.ValidateExtractedArtifacts(extractorOptions.VersionSetNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Backend.ValidateExtractedArtifacts(extractorOptions.BackendNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Logger.ValidateExtractedArtifacts(extractorOptions.LoggerNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Diagnostic.ValidateExtractedArtifacts(extractorOptions.DiagnosticNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await PolicyFragment.ValidateExtractedArtifacts(extractorOptions.PolicyFragmentNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await ServicePolicy.ValidateExtractedArtifacts(serviceDirectory, serviceUri, pipeline, cancellationToken); - await Product.ValidateExtractedArtifacts(extractorOptions.ProductNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Group.ValidateExtractedArtifacts(extractorOptions.GroupNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Api.ValidateExtractedArtifacts(extractorOptions.ApiNamesToExport, extractorOptions.DefaultApiSpecification, serviceDirectory, serviceUri, pipeline, cancellationToken); - await Subscription.ValidateExtractedArtifacts(extractorOptions.SubscriptionNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - - if (validateSpecialSkuResources) - { - await Gateway.ValidateExtractedArtifacts(extractorOptions.GatewayNamesToExport, serviceDirectory, serviceUri, pipeline, cancellationToken); - } - } - - public static async ValueTask WriteArtifacts(ServiceModel model, ManagementServiceDirectory serviceDirectory, bool writeSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.WriteArtifacts(model.NamedValues, serviceDirectory, cancellationToken); - await Tag.WriteArtifacts(model.Tags, serviceDirectory, cancellationToken); - await Gateway.WriteArtifacts(model.Gateways, serviceDirectory, cancellationToken); - await VersionSet.WriteArtifacts(model.VersionSets, serviceDirectory, cancellationToken); - await Backend.WriteArtifacts(model.Backends, serviceDirectory, cancellationToken); - await Logger.WriteArtifacts(model.Loggers, serviceDirectory, cancellationToken); - await Diagnostic.WriteArtifacts(model.Diagnostics, serviceDirectory, cancellationToken); - await PolicyFragment.WriteArtifacts(model.PolicyFragments, serviceDirectory, cancellationToken); - await ServicePolicy.WriteArtifacts(model.ServicePolicies, serviceDirectory, cancellationToken); - await Product.WriteArtifacts(model.Products, serviceDirectory, cancellationToken); - await Group.WriteArtifacts(model.Groups, serviceDirectory, cancellationToken); - await Api.WriteArtifacts(model.Apis, serviceDirectory, cancellationToken); - await Subscription.WriteArtifacts(model.Subscriptions, serviceDirectory, cancellationToken); - } - - public static async ValueTask ValidatePublisherChanges(PublisherOptions publisherOptions, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, bool validateSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.ValidatePublisherChanges(serviceDirectory, publisherOptions.NamedValueOverrides, serviceUri, pipeline, cancellationToken); - await Tag.ValidatePublisherChanges(serviceDirectory, publisherOptions.TagOverrides, serviceUri, pipeline, cancellationToken); - await VersionSet.ValidatePublisherChanges(serviceDirectory, publisherOptions.VersionSetOverrides, serviceUri, pipeline, cancellationToken); - await Backend.ValidatePublisherChanges(serviceDirectory, publisherOptions.BackendOverrides, serviceUri, pipeline, cancellationToken); - await Logger.ValidatePublisherChanges(serviceDirectory, publisherOptions.LoggerOverrides, serviceUri, pipeline, cancellationToken); - await Diagnostic.ValidatePublisherChanges(serviceDirectory, publisherOptions.DiagnosticOverrides, serviceUri, pipeline, cancellationToken); - await PolicyFragment.ValidatePublisherChanges(serviceDirectory, publisherOptions.PolicyFragmentOverrides, serviceUri, pipeline, cancellationToken); - await ServicePolicy.ValidatePublisherChanges(serviceDirectory, publisherOptions.ServicePolicyOverrides, serviceUri, pipeline, cancellationToken); - await Product.ValidatePublisherChanges(serviceDirectory, publisherOptions.ProductOverrides, serviceUri, pipeline, cancellationToken); - await Group.ValidatePublisherChanges(serviceDirectory, publisherOptions.GroupOverrides, serviceUri, pipeline, cancellationToken); - await Api.ValidatePublisherChanges(serviceDirectory, publisherOptions.ApiOverrides, serviceUri, pipeline, cancellationToken); - await Subscription.ValidatePublisherChanges(serviceDirectory, publisherOptions.SubscriptionOverrides, serviceUri, pipeline, cancellationToken); - - if (validateSpecialSkuResources) - { - await Gateway.ValidatePublisherChanges(serviceDirectory, publisherOptions.GatewayOverrides, serviceUri, pipeline, cancellationToken); - } - } - - public static async ValueTask> WriteCommitArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, bool writeSpecialSkuResources, CancellationToken cancellationToken) - { - var authorName = "apiops"; - var authorEmail = "apiops@apiops.com"; - var serviceDirectoryInfo = serviceDirectory.ToDirectoryInfo(); - Git.InitializeRepository(serviceDirectoryInfo, commitMessage: "Initial commit", authorName, authorEmail, DateTimeOffset.UtcNow); - - var commitIds = ImmutableArray.Empty; - await models.Map((index, model) => (index, model)) - .Iter(async x => - { - var (index, model) = x; - DeleteNonGitDirectories(serviceDirectory); - await WriteArtifacts(model, serviceDirectory, writeSpecialSkuResources, cancellationToken); - var commit = Git.CommitChanges(serviceDirectoryInfo, commitMessage: $"Commit {index}", authorName, authorEmail, DateTimeOffset.UtcNow); - var commitId = new CommitId(commit.Sha); - ImmutableInterlocked.Update(ref commitIds, commitIds => commitIds.Add(commitId)); - }, cancellationToken); - - return commitIds; - } - - private static void DeleteNonGitDirectories(ManagementServiceDirectory serviceDirectory) => - serviceDirectory.ToDirectoryInfo() - .ListDirectories("*") - .Where(directory => directory.Name.Equals(".git", StringComparison.OrdinalIgnoreCase) is false) - .Iter(directory => directory.Delete(recursive: true)); - - public static async ValueTask ValidatePublisherCommitChanges(PublisherOptions publisherOptions, CommitId commitId, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, bool validateSpecialSkuResources, CancellationToken cancellationToken) - { - await NamedValue.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.NamedValueOverrides, serviceUri, pipeline, cancellationToken); - await Tag.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.TagOverrides, serviceUri, pipeline, cancellationToken); - await VersionSet.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.VersionSetOverrides, serviceUri, pipeline, cancellationToken); - await Backend.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.BackendOverrides, serviceUri, pipeline, cancellationToken); - await Logger.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.LoggerOverrides, serviceUri, pipeline, cancellationToken); - await Diagnostic.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.DiagnosticOverrides, serviceUri, pipeline, cancellationToken); - await PolicyFragment.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.PolicyFragmentOverrides, serviceUri, pipeline, cancellationToken); - await ServicePolicy.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.ServicePolicyOverrides, serviceUri, pipeline, cancellationToken); - await Product.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.ProductOverrides, serviceUri, pipeline, cancellationToken); - await Group.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.GroupOverrides, serviceUri, pipeline, cancellationToken); - await Api.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.ApiOverrides, serviceUri, pipeline, cancellationToken); - await Subscription.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.SubscriptionOverrides, serviceUri, pipeline, cancellationToken); - - if (validateSpecialSkuResources) - { - await Gateway.ValidatePublisherCommitChanges(commitId, serviceDirectory, publisherOptions.GatewayOverrides, serviceUri, pipeline, cancellationToken); - } - } -} - -file static class ResiliencePipelines -{ - private static readonly Lazy deletePolicy = new(() => - new ResiliencePipelineBuilder() - .AddRetry(new() - { - BackoffType = DelayBackoffType.Constant, - UseJitter = true, - MaxRetryAttempts = 3, - ShouldHandle = new PredicateBuilder().Handle(exception => exception.StatusCode == HttpStatusCode.PreconditionFailed && exception.Message.Contains("Resource was modified since last retrieval", StringComparison.OrdinalIgnoreCase)) - }) - .Build()); - - public static ResiliencePipeline DeletePolicy => deletePolicy.Value; -} \ No newline at end of file diff --git a/tools/code/integration.tests/ServicePolicy.cs b/tools/code/integration.tests/ServicePolicy.cs index 6a3c1e4f..978e47c6 100644 --- a/tools/code/integration.tests/ServicePolicy.cs +++ b/tools/code/integration.tests/ServicePolicy.cs @@ -4,37 +4,67 @@ using CsCheck; using FluentAssertions; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class ServicePolicy +internal delegate ValueTask DeleteAllServicePolicies(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutServicePolicyModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedServicePolicies(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimServicePolicies(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileServicePolicies(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteServicePolicyModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedServicePolicies(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllServicePoliciesHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(ServicePolicyModel original) => - from content in ServicePolicyModel.GenerateContent() - select original with - { - Content = content - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllServicePolicies)); - public static Gen GenerateOverride(ServicePolicyDto original) => - from content in ServicePolicyModel.GenerateContent() - select new ServicePolicyDto + logger.LogInformation("Deleting all service policies in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await ServicePoliciesUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutServicePolicyModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutServicePolicyModels)); + + logger.LogInformation("Putting version set models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new ServicePolicyDto.ServicePolicyContract - { - Value = content - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(ServicePolicyModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = ServicePolicyUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static ServicePolicyDto GetDto(ServicePolicyModel model) => new() @@ -45,29 +75,123 @@ private static ServicePolicyDto GetDto(ServicePolicyModel model) => Value = model.Content } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedServicePoliciesHandler(ILogger logger, GetApimServicePolicies getApimResources, GetFileServicePolicies getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedServicePolicies)); + + logger.LogInformation("Validating extracted service policies in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(ServicePolicyDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + Value = new string((dto.Properties.Value ?? string.Empty) + .ReplaceLineEndings(string.Empty) + .Where(c => char.IsWhiteSpace(c) is false) + .ToArray()) + }.ToString()!; +} - private static async ValueTask Put(ServicePolicyModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimServicePoliciesHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = ServicePolicyUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimServicePolicies)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting service policies from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = ServicePoliciesUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await ServicePoliciesUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileServicePoliciesHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileServicePolicies)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Getting service policies from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => ServicePolicyFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ServicePolicyFile file, CancellationToken cancellationToken) + { + var name = file.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = new ServicePolicyDto + { + Properties = new ServicePolicyDto.ServicePolicyContract + { + Value = data.ToString() + } + }; + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileServicePolicies)); + + logger.LogInformation("Getting service policies from {ServiceDirectory}...", serviceDirectory); + + return await ServicePolicyModule.ListPolicyFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Name, + new ServicePolicyDto + { + Properties = new ServicePolicyDto.ServicePolicyContract + { + Value = await file.ReadPolicy(cancellationToken) + } + })) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteServicePolicyModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteServicePolicyModels)); + + logger.LogInformation("Writing version set models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WritePolicyFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WritePolicyFile(ServicePolicyModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -75,38 +199,35 @@ private static async ValueTask WritePolicyFile(ServicePolicyModel model, Managem await policyFile.WritePolicy(model.Content, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static ServicePolicyDto GetDto(ServicePolicyModel model) => + new() + { + Properties = new ServicePolicyDto.ServicePolicyContract + { + Format = "rawxml", + Value = model.Content + } + }; +} + +file sealed class ValidatePublishedServicePoliciesHandler(ILogger logger, GetFileServicePolicies getFileResources, GetApimServicePolicies getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedServicePolicies)); - var expected = apimResources.MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published service policies in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = ServicePoliciesUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await ServicePolicyModule.ListPolicyFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Name, - new ServicePolicyDto - { - Properties = new ServicePolicyDto.ServicePolicyContract - { - Value = await file.ReadPolicy(cancellationToken) - } - })) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(ServicePolicyDto dto) => new { @@ -115,55 +236,94 @@ private static string NormalizeDto(ServicePolicyDto dto) => .Where(c => char.IsWhiteSpace(c) is false) .ToArray()) }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class ServicePolicyServices +{ + public static void ConfigureDeleteAllServicePolicies(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutServicePolicyModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedServicePolicies(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimServicePolicies(services); + ConfigureGetFileServicePolicies(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => ServicePolicyFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimServicePolicies(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, ServicePolicyFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileServicePolicies(IServiceCollection services) { - var name = file.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteServicePolicyModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedServicePolicies(IServiceCollection services) + { + ConfigureGetFileServicePolicies(services); + ConfigureGetApimServicePolicies(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class ServicePolicy +{ + public static Gen GenerateUpdate(ServicePolicyModel original) => + from content in ServicePolicyModel.GenerateContent() + select original with { - using (contents) + Content = content + }; + + public static Gen GenerateOverride(ServicePolicyDto original) => + from content in ServicePolicyModel.GenerateContent() + select new ServicePolicyDto + { + Properties = new ServicePolicyDto.ServicePolicyContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = new ServicePolicyDto - { - Properties = new ServicePolicyDto.ServicePolicyContract - { - Value = data.ToString() - } - }; - return (name, dto); + Value = content } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static ServicePolicyDto GetDto(ServicePolicyModel model) => + new() + { + Properties = new ServicePolicyDto.ServicePolicyContract + { + Format = "rawxml", + Value = model.Content + } + }; } diff --git a/tools/code/integration.tests/Subscription.cs b/tools/code/integration.tests/Subscription.cs index 5a4f22d5..937ae56f 100644 --- a/tools/code/integration.tests/Subscription.cs +++ b/tools/code/integration.tests/Subscription.cs @@ -4,17 +4,45 @@ using CsCheck; using FluentAssertions; using LanguageExt; -using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; +internal delegate ValueTask DeleteAllSubscriptions(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file sealed class DeleteAllSubscriptionsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllSubscriptions)); + + logger.LogInformation("Deleting all subscriptions in {ServiceName}.", serviceName); + var serviceUri = getServiceUri(serviceName); + await SubscriptionsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +internal static class SubscriptionServices +{ + public static void ConfigureDeleteAllSubscriptions(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + internal static class Subscription { public static Gen GenerateUpdate(SubscriptionModel original) => diff --git a/tools/code/integration.tests/Tag.cs b/tools/code/integration.tests/Tag.cs index 12720a92..69e23551 100644 --- a/tools/code/integration.tests/Tag.cs +++ b/tools/code/integration.tests/Tag.cs @@ -4,37 +4,67 @@ using CsCheck; using FluentAssertions; using LanguageExt; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class Tag +internal delegate ValueTask DeleteAllTags(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutTagModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedTags(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimTags(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileTags(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteTagModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedTags(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllTagsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(TagModel original) => - from displayName in TagModel.GenerateDisplayName() - select original with - { - DisplayName = displayName - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllTags)); - public static Gen GenerateOverride(TagDto original) => - from displayName in TagModel.GenerateDisplayName() - select new TagDto + logger.LogInformation("Deleting all tags in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await TagsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} + +file sealed class PutTagModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutTagModels)); + + logger.LogInformation("Putting tag models in {ServiceName}...", serviceName); + await models.IterParallel(async model => { - Properties = new TagDto.TagContract - { - DisplayName = displayName - } - }; + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(TagModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = TagUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static TagDto GetDto(TagModel model) => new() @@ -44,29 +74,109 @@ private static TagDto GetDto(TagModel model) => DisplayName = model.DisplayName } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedTagsHandler(ILogger logger, GetApimTags getApimResources, GetFileTags getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedTags)); + + logger.LogInformation("Validating extracted tags in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(TagDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(TagModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimTagsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = TagUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimTags)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting tags from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = TagsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await TagsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileTagsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileTags)); + + logger.LogInformation("Getting tags from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => TagInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, TagInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileTags)); + + logger.LogInformation("Getting tags from {ServiceDirectory}...", serviceDirectory); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + return await TagModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteTagModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteTagModels)); + + logger.LogInformation("Writing tag models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(TagModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -76,81 +186,126 @@ private static async ValueTask WriteInformationFile(TagModel model, ManagementSe await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static TagDto GetDto(TagModel model) => + new() + { + Properties = new TagDto.TagContract + { + DisplayName = model.DisplayName + } + }; +} + +file sealed class ValidatePublishedTagsHandler(ILogger logger, GetFileTags getFileResources, GetApimTags getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedTags)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published tags in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = TagsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await TagModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(TagDto dto) => new { DisplayName = dto.Properties.DisplayName ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class TagServices +{ + public static void ConfigureDeleteAllTags(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutTagModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidateExtractedTags(IServiceCollection services) + { + ConfigureGetApimTags(services); + ConfigureGetFileTags(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static void ConfigureGetApimTags(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => TagInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetFileTags(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, TagInformationFile file, CancellationToken cancellationToken) + public static void ConfigureWriteTagModels(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureValidatePublishedTags(IServiceCollection services) + { + ConfigureGetFileTags(services); + ConfigureGetApimTags(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class Tag +{ + public static Gen GenerateUpdate(TagModel original) => + from displayName in TagModel.GenerateDisplayName() + select original with { - using (contents) + DisplayName = displayName + }; + + public static Gen GenerateOverride(TagDto original) => + from displayName in TagModel.GenerateDisplayName() + select new TagDto + { + Properties = new TagDto.TagContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + DisplayName = displayName } - }); - } + }; + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static TagDto GetDto(TagModel model) => + new() + { + Properties = new TagDto.TagContract + { + DisplayName = model.DisplayName + } + }; } diff --git a/tools/code/integration.tests/Test.cs b/tools/code/integration.tests/Test.cs new file mode 100644 index 00000000..af5031f9 --- /dev/null +++ b/tools/code/integration.tests/Test.cs @@ -0,0 +1,506 @@ +using common; +using common.tests; +using CsCheck; +using LanguageExt; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using publisher; +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace integration.tests; + +internal delegate ValueTask RunTests(CancellationToken cancellationToken); + +file delegate ValueTask TestExtractor(CancellationToken cancellationToken); + +file delegate ValueTask TestExtractThenPublish(CancellationToken cancellationToken); + +file delegate ValueTask TestPublisher(CancellationToken cancellationToken); + +file delegate ValueTask CleanUpTests(CancellationToken cancellationToken); + +file sealed class RunTestsHandler(ILogger logger, + ActivitySource activitySource, + TestExtractor testExtractor, + TestExtractThenPublish testExtractThenPublish, + TestPublisher testPublisher, + CleanUpTests cleanUpTests) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(RunTests)); + + logger.LogInformation("Running tests..."); + await testExtractor(cancellationToken); + await testExtractThenPublish(cancellationToken); + await testPublisher(cancellationToken); + await cleanUpTests(cancellationToken); + } +} + +file sealed class TestExtractorHandler(ILogger logger, + IConfiguration configuration, + ActivitySource activitySource, + CreateApimService createService, + EmptyApimService emptyService, + PutServiceModel putServiceModel, + RunExtractor runExtractor, + ValidateExtractorArtifacts validateExtractor, + DeleteApimService deleteApimService) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(TestExtractor)); + + logger.LogInformation("Testing extractor..."); + + var generator = Fixture.Generate(configuration); + await generator.SampleAsync(async fixture => await Run(fixture, cancellationToken), iter: 1); + } + + private async ValueTask Run(Fixture fixture, CancellationToken cancellationToken) + { + await createService(fixture.ServiceName, cancellationToken); + await emptyService(fixture.ServiceName, cancellationToken); + await putServiceModel(fixture.ServiceModel, fixture.ServiceName, cancellationToken); + await runExtractor(fixture.ExtractorOptions, fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + await validateExtractor(fixture.ExtractorOptions, fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + await CleanUp(fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + } + + private async ValueTask CleanUp(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + if (serviceName.ToString().StartsWith(Common.TestServiceNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + await deleteApimService(serviceName, cancellationToken); + } + + Common.DeleteServiceDirectory(serviceDirectory); + } + + private sealed record Fixture + { + public required ExtractorOptions ExtractorOptions { get; init; } + public required ServiceModel ServiceModel { get; init; } + public required ManagementServiceName ServiceName { get; init; } + public required ManagementServiceDirectory ServiceDirectory { get; init; } + + public static Gen Generate(IConfiguration configuration) => + from serviceModel in ServiceModel.Generate() + from extractorOptions in ExtractorOptions.Generate(serviceModel) + from serviceName in Common.useExistingInstance + ? Gen.Const(ManagementServiceName.From(configuration.GetValue("FIRST_API_MANAGEMENT_SERVICE_NAME"))) + : Common.GenerateManagementServiceName(Common.TestServiceNamePrefix) + from serviceDirectory in Common.GenerateManagementServiceDirectory() + select new Fixture + { + ExtractorOptions = extractorOptions, + ServiceModel = serviceModel, + ServiceName = serviceName, + ServiceDirectory = serviceDirectory + }; + } +} + +file sealed class TestExtractThenPublishHandler(ILogger logger, + IConfiguration configuration, + ActivitySource activitySource, + CreateApimService createService, + PutServiceModel putServiceModel, + RunExtractor runExtractor, + DeleteApimService deleteApimService, + RunPublisher runPublisher, + ValidatePublishedArtifacts validatePublisher) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(TestExtractThenPublish)); + + logger.LogInformation("Testing extracting, then publishing to a fresh instance..."); + + var generator = Fixture.Generate(configuration); + await generator.SampleAsync(async fixture => await Run(fixture, cancellationToken), iter: 1); + } + + private async ValueTask Run(Fixture fixture, CancellationToken cancellationToken) + { + await CreateExtractorArtifacts(fixture.ServiceModel, fixture.SourceServiceName, fixture.ServiceDirectory, cancellationToken); + await PublishToDestination(fixture.PublisherOptions, fixture.DestinationServiceName, fixture.ServiceDirectory, cancellationToken); + await validatePublisher(fixture.PublisherOptions, Option.None, fixture.DestinationServiceName, fixture.ServiceDirectory, cancellationToken); + await CleanUp(fixture.SourceServiceName, fixture.DestinationServiceName, fixture.ServiceDirectory, cancellationToken); + } + + private async ValueTask CreateExtractorArtifacts(ServiceModel serviceModel, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(CreateExtractorArtifacts)); + + logger.LogInformation("Creating extractor artifacts..."); + + await createService(serviceName, cancellationToken); + await putServiceModel(serviceModel, serviceName, cancellationToken); + await runExtractor(ExtractorOptions.NoFilter, serviceName, serviceDirectory, cancellationToken); + await deleteApimService(serviceName, cancellationToken); + } + + private async ValueTask PublishToDestination(PublisherOptions publisherOptions, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PublishToDestination)); + + logger.LogInformation("Publishing artifacts to destination..."); + await createService(serviceName, cancellationToken); + await runPublisher(publisherOptions, serviceName, serviceDirectory, Option.None, cancellationToken); + } + + private async ValueTask CleanUp(ManagementServiceName sourceServiceName, ManagementServiceName destinationServiceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + await new[] { sourceServiceName, destinationServiceName } + .Where(name => name.ToString().StartsWith(Common.TestServiceNamePrefix, StringComparison.OrdinalIgnoreCase)) + .IterParallel(deleteApimService.Invoke, cancellationToken); + + Common.DeleteServiceDirectory(serviceDirectory); + } + + private sealed record Fixture + { + public required PublisherOptions PublisherOptions { get; init; } + public required ServiceModel ServiceModel { get; init; } + public required ManagementServiceName SourceServiceName { get; init; } + public required ManagementServiceName DestinationServiceName { get; init; } + public required ManagementServiceDirectory ServiceDirectory { get; init; } + + public static Gen Generate(IConfiguration configuration) => + from serviceModel in ServiceModel.Generate() + from publisherOptions in PublisherOptions.Generate(serviceModel) + from sourceServiceName in GenerateManagementServiceName(configuration, "FIRST_API_MANAGEMENT_SERVICE_NAME") + from destinationServiceName in GenerateManagementServiceName(configuration, "SECOND_API_MANAGEMENT_SERVICE_NAME") + from serviceDirectory in Common.GenerateManagementServiceDirectory() + select new Fixture + { + PublisherOptions = publisherOptions, + ServiceModel = serviceModel, + SourceServiceName = sourceServiceName, + DestinationServiceName = destinationServiceName, + ServiceDirectory = serviceDirectory + }; + + private static Gen GenerateManagementServiceName(IConfiguration configuration, string configurationKey) => + Common.useExistingInstance + ? Gen.Const(ManagementServiceName.From(configuration.GetValue(configurationKey))) + : Common.GenerateManagementServiceName(Common.TestServiceNamePrefix); + } +} + +file sealed class TestPublisherHandler(ILogger logger, + IConfiguration configuration, + ActivitySource activitySource, + WriteServiceModelArtifacts writeArtifacts, + CreateApimService createService, + EmptyApimService emptyService, + RunPublisher runPublisher, + ValidatePublishedArtifacts validatePublisher, + WriteServiceModelCommits writeCommits, + DeleteApimService deleteApimService) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(TestPublisher)); + + logger.LogInformation("Testing publisher..."); + + var generator = Fixture.Generate(configuration); + await generator.SampleAsync(async fixture => await Run(fixture, cancellationToken), iter: 1); + } + + private async ValueTask Run(Fixture fixture, CancellationToken cancellationToken) + { + await PublishAllChangesAndValidate(fixture.PublisherOptions, fixture.ServiceModel, fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + await PublishCommitsAndValidate(fixture.PublisherOptions, fixture.CommitModels, fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + await CleanUp(fixture.ServiceName, fixture.ServiceDirectory, cancellationToken); + } + + private async ValueTask PublishAllChangesAndValidate(PublisherOptions publisherOptions, ServiceModel serviceModel, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + await writeArtifacts(serviceModel, serviceDirectory, cancellationToken); + await createService(serviceName, cancellationToken); + await emptyService(serviceName, cancellationToken); + await runPublisher(publisherOptions, serviceName, serviceDirectory, Option.None, cancellationToken); + await validatePublisher(publisherOptions, Option.None, serviceName, serviceDirectory, cancellationToken); + } + + private async ValueTask PublishCommitsAndValidate(PublisherOptions publisherOptions, IEnumerable serviceModels, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + var commits = await writeCommits(serviceModels, serviceDirectory, cancellationToken); + await runPublisher(publisherOptions, serviceName, serviceDirectory, commits.HeadOrNone(), cancellationToken); + await validatePublisher(publisherOptions, commits.HeadOrNone(), serviceName, serviceDirectory, cancellationToken); + } + + private async ValueTask CleanUp(ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + if (serviceName.ToString().StartsWith(Common.TestServiceNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + await deleteApimService(serviceName, cancellationToken); + } + + Common.DeleteServiceDirectory(serviceDirectory); + } + + private sealed record Fixture + { + public required PublisherOptions PublisherOptions { get; init; } + public required ServiceModel ServiceModel { get; init; } + public required ImmutableArray CommitModels { get; init; } + public required ManagementServiceName ServiceName { get; init; } + public required ManagementServiceDirectory ServiceDirectory { get; init; } + + public static Gen Generate(IConfiguration configuration) => + from serviceModel in ServiceModel.Generate() + from commitModels in GenerateCommitModels(serviceModel) + from publisherOptions in PublisherOptions.Generate(serviceModel) + from serviceName in GenerateManagementServiceName(configuration, "FIRST_API_MANAGEMENT_SERVICE_NAME") + from serviceDirectory in Common.GenerateManagementServiceDirectory() + select new Fixture + { + PublisherOptions = publisherOptions, + ServiceModel = serviceModel, + CommitModels = commitModels, + ServiceName = serviceName, + ServiceDirectory = serviceDirectory + }; + + private static Gen GenerateManagementServiceName(IConfiguration configuration, string configurationKey) => + Common.useExistingInstance + ? Gen.Const(ManagementServiceName.From(configuration.GetValue(configurationKey))) + : Common.GenerateManagementServiceName(Common.TestServiceNamePrefix); + + private static Gen> GenerateCommitModels(ServiceModel initialModel) => + from list in Gen.Int.List[1, 10] + from aggregate in list.Aggregate(Gen.Const(ImmutableArray.Empty), + (gen, _) => from commits in gen + let lastCommit = commits.LastOrDefault(initialModel) + from newCommit in GenerateUpdatedServiceModel(lastCommit, ChangeParameters.All) + select commits.Add(newCommit)) + select aggregate; + + private static Gen GenerateUpdatedServiceModel(ServiceModel initialModel, ChangeParameters changeParameters) => + from namedValues in GenerateNewSet(initialModel.NamedValues, NamedValueModel.GenerateSet(), NamedValue.GenerateUpdate, changeParameters) + from tags in GenerateNewSet(initialModel.Tags, TagModel.GenerateSet(), Tag.GenerateUpdate, changeParameters) + from versionSets in GenerateNewSet(initialModel.VersionSets, VersionSetModel.GenerateSet(), VersionSet.GenerateUpdate, changeParameters) + from backends in GenerateNewSet(initialModel.Backends, BackendModel.GenerateSet(), Backend.GenerateUpdate, changeParameters) + from loggers in GenerateNewSet(initialModel.Loggers, LoggerModel.GenerateSet(), Logger.GenerateUpdate, changeParameters) + from diagnostics in from diagnosticSet in GenerateNewSet(initialModel.Diagnostics, DiagnosticModel.GenerateSet(), Diagnostic.GenerateUpdate, changeParameters) + from updatedDiagnostics in ServiceModel.UpdateDiagnostics(diagnosticSet, loggers) + select updatedDiagnostics + from policyFragments in GenerateNewSet(initialModel.PolicyFragments, PolicyFragmentModel.GenerateSet(), PolicyFragment.GenerateUpdate, changeParameters) + from servicePolicies in GenerateNewSet(initialModel.ServicePolicies, ServicePolicyModel.GenerateSet(), ServicePolicy.GenerateUpdate, changeParameters) + from groups in GenerateNewSet(initialModel.Groups, GroupModel.GenerateSet(), Group.GenerateUpdate, changeParameters) + from apis in from apiSet in GenerateNewSet(initialModel.Apis, ApiModel.GenerateSet(), Api.GenerateUpdate, changeParameters) + from updatedApis in ServiceModel.UpdateApis(apiSet, versionSets, tags) + select updatedApis + from products in from productSet in GenerateNewSet(initialModel.Products, ProductModel.GenerateSet(), Product.GenerateUpdate, changeParameters) + from updatedProductGroups in ServiceModel.UpdateProducts(productSet, groups, tags, apis) + select updatedProductGroups + from gateways in from gatewaySet in GenerateNewSet(initialModel.Gateways, GatewayModel.GenerateSet(), Gateway.GenerateUpdate, changeParameters) + from updatedGatewayApis in ServiceModel.UpdateGateways(gatewaySet, apis) + select updatedGatewayApis + from subscriptions in from subscriptionSet in GenerateNewSet(initialModel.Subscriptions, SubscriptionModel.GenerateSet(), Subscription.GenerateUpdate, changeParameters) + from updatedSubscriptions in ServiceModel.UpdateSubscriptions(subscriptionSet, products, apis) + select updatedSubscriptions + select initialModel with + { + NamedValues = namedValues, + Tags = tags, + Gateways = gateways, + VersionSets = versionSets, + Backends = backends, + Loggers = loggers, + Diagnostics = diagnostics, + PolicyFragments = policyFragments, + ServicePolicies = servicePolicies, + Products = products, + Groups = groups, + Apis = apis, + Subscriptions = subscriptions + }; + + public static Gen> GenerateNewSet(FrozenSet original, Gen> newGen, Func> updateGen) => + GenerateNewSet(original, newGen, updateGen, ChangeParameters.All); + + private static Gen> GenerateNewSet(FrozenSet original, Gen> newGen, Func> updateGen, ChangeParameters changeParameters) + { + var generator = from originalItems in Gen.Const(original) + from itemsRemoved in changeParameters.Remove ? RemoveItems(originalItems) : Gen.Const(originalItems) + from itemsAdded in changeParameters.Add ? AddItems(itemsRemoved, newGen) : Gen.Const(itemsRemoved) + from itemsModified in changeParameters.Modify ? ModifyItems(itemsAdded, updateGen) : Gen.Const(itemsAdded) + select itemsModified; + + return changeParameters.MaxSize.Map(maxSize => generator.SelectMany(set => set.Count <= maxSize + ? generator + : from smallerSet in Gen.Shuffle(set.ToArray(), maxSize) + + select smallerSet.ToFrozenSet(set.Comparer))) + .IfNone(generator); + } + + private static Gen> RemoveItems(FrozenSet set) => + from itemsToRemove in Generator.SubFrozenSetOf(set) + select set.Except(itemsToRemove, set.Comparer).ToFrozenSet(set.Comparer); + + private static Gen> AddItems(FrozenSet set, Gen> gen) => + from itemsToAdd in gen + select set.Append(itemsToAdd).ToFrozenSet(set.Comparer); + + private static Gen> ModifyItems(FrozenSet set, Func> updateGen) => + from itemsToModify in Generator.SubFrozenSetOf(set) + from modifiedItems in itemsToModify.Select(updateGen).SequenceToImmutableArray() + select set.Except(itemsToModify).Append(modifiedItems).ToFrozenSet(set.Comparer); + + private sealed record ChangeParameters + { + public required bool Add { get; init; } + public required bool Modify { get; init; } + public required bool Remove { get; init; } + + public static ChangeParameters None { get; } = new() + { + Add = false, + Modify = false, + Remove = false + }; + + public static ChangeParameters All { get; } = new() + { + Add = true, + Modify = true, + Remove = true + }; + + public Option MaxSize { get; init; } = Option.None; + } + } +} + +file sealed class CleanUpTestsHandler(ILogger logger, + ActivitySource activitySource, + ListApimServiceNames listApimServiceNames, + DeleteApimService deleteApimService) +{ + public async ValueTask Handle(CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(CleanUpTests)); + + logger.LogInformation("Cleaning up test..."); + + await listApimServiceNames(cancellationToken) + .Where(name => name.ToString().StartsWith(Common.TestServiceNamePrefix, StringComparison.OrdinalIgnoreCase)) + .IterParallel(deleteApimService.Invoke, cancellationToken); + } +} + +internal static class TestServices +{ + public static void ConfigureRunTests(IServiceCollection services) + { + ConfigureTestExtractor(services); + ConfigureTestExtractThenPublish(services); + ConfigureTestPublisher(services); + ConfigureCleanUpTests(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureTestExtractor(IServiceCollection services) + { + ManagementServices.ConfigureCreateApimService(services); + ManagementServices.ConfigureEmptyApimService(services); + ManagementServices.ConfigurePutServiceModel(services); + ExtractorServices.ConfigureRunExtractor(services); + ExtractorServices.ConfigureValidateExtractorArtifacts(services); + ManagementServices.ConfigureDeleteApimService(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureTestExtractThenPublish(IServiceCollection services) + { + ManagementServices.ConfigureCreateApimService(services); + ManagementServices.ConfigurePutServiceModel(services); + ExtractorServices.ConfigureRunExtractor(services); + ManagementServices.ConfigureDeleteApimService(services); + PublisherServices.ConfigureRunPublisher(services); + PublisherServices.ConfigureValidatePublishedArtifacts(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureTestPublisher(IServiceCollection services) + { + ManagementServices.ConfigureWriteServiceModelArtifacts(services); + ManagementServices.ConfigureCreateApimService(services); + ManagementServices.ConfigureEmptyApimService(services); + PublisherServices.ConfigureRunPublisher(services); + PublisherServices.ConfigureValidatePublishedArtifacts(services); + ManagementServices.ConfigureWriteServiceModelCommits(services); + ManagementServices.ConfigureDeleteApimService(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureCleanUpTests(IServiceCollection services) + { + ManagementServices.ConfigureListApimServiceNames(services); + ManagementServices.ConfigureDeleteApimService(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +file sealed class Common +{ +#pragma warning disable CA1802 // Use literals where appropriate +#pragma warning disable CA1805 // Do not initialize unnecessarily + public static string TestServiceNamePrefix { get; } = "apiopsinttest-"; + public static readonly bool useExistingInstance = false; +#pragma warning restore CA1805 // Do not initialize unnecessarily +#pragma warning restore CA1802 // Use literals where appropriate + + public static void DeleteServiceDirectory(ManagementServiceDirectory serviceDirectory) => serviceDirectory.ToDirectoryInfo().ForceDelete(); + + // We want the name to change between tests, even if the seed is the same. + // This avoids soft-delete issues with APIM + public static Gen GenerateManagementServiceName(string prefix) => + from lorem in Generator.Lorem + let characters = lorem.Paragraphs(3) + .Where(char.IsLetterOrDigit) + .Select(char.ToLowerInvariant) + .ToArray() + from suffixCharacters in Gen.Shuffle(characters, 8) + let name = $"{prefix}{new string(suffixCharacters)}" + select ManagementServiceName.From(name); + + public static Gen GenerateManagementServiceDirectory() => + from lorem in Generator.Lorem + let characters = lorem.Paragraphs(3) + .Where(char.IsLetterOrDigit) + .Select(char.ToLowerInvariant) + .ToArray() + from suffixCharacters in Gen.Shuffle(characters, 8) + let name = $"apiops-{new string(suffixCharacters)}" + let path = Path.Combine(Path.GetTempPath(), name) + let directoryInfo = new DirectoryInfo(path) + select ManagementServiceDirectory.From(directoryInfo); +} \ No newline at end of file diff --git a/tools/code/integration.tests/Tests.cs b/tools/code/integration.tests/Tests.cs deleted file mode 100644 index 018a282c..00000000 --- a/tools/code/integration.tests/Tests.cs +++ /dev/null @@ -1,274 +0,0 @@ -using common; -using common.tests; -using CsCheck; -using DotNext.Collections.Generic; -using FluentAssertions; -using LanguageExt; -using NUnit.Framework; -using publisher; -using System; -using System.Collections.Frozen; -using System.Collections.Immutable; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace integration.tests; - - -[TestFixture] -public sealed class Tests -{ - private const string serviceNamePrefix = "apiopsinttest-"; - -#pragma warning disable CA1802 // Use literals where appropriate -#pragma warning disable CA1805 // Do not initialize unnecessarily - private static readonly bool useExistingInstance = false; -#pragma warning restore CA1805 // Do not initialize unnecessarily -#pragma warning restore CA1802 // Use literals where appropriate - - [Test] - public async Task Runs_as_expected() - { - AssertionOptions.FormattingOptions.MaxLines = 10000; - AssertionOptions.AssertEquivalencyUsing(options => options.ComparingRecordsByValue()); - - var cancellationToken = CancellationToken.None; - await OneTimeSetup(cancellationToken); - - await WriteProgress("Generating fixture..."); - var generator = from fixture in Fixture.Generate(serviceNamePrefix) - // Use configuration service name for special SKUs - select useExistingInstance - ? fixture with - { - FirstServiceName = Configuration.FirstServiceName, - SecondServiceName = Configuration.SecondServiceName - } - : fixture with - { - ServiceModel = fixture.ServiceModel with - { - Gateways = FrozenSet.Empty - }, - PublishAllChangesModel = fixture.PublishAllChangesModel with - { - Gateways = FrozenSet.Empty - }, - CommitModels = fixture.CommitModels.Select(model => model with - { - Gateways = FrozenSet.Empty - }).ToImmutableArray() - }; - - await generator.SampleAsync(async fixture => - { - // 1. Set up the management service - await CreateManagementServices(fixture, cancellationToken); - DeleteServiceDirectory(fixture); - - // 2. Run the extractor and validate its artifacts - await PutServiceModel(fixture, cancellationToken); - await RunExtractor(fixture, cancellationToken); - await ValidateExtractorArtifacts(fixture, cancellationToken); - - // 3. Publish the extracted changes to the empty second service - await PublishExtractedArtifactsToSecondService(fixture, cancellationToken); - await ValidatePublishedExtractedArtifacts(fixture, cancellationToken); - await CleanUpExtractorResources(fixture, cancellationToken); - - // 3. Make changes to the extracted artifacts, publish the changes, then validate - await WriteFirstChange(fixture, cancellationToken); - await PublishFirstChange(fixture, cancellationToken); - await ValidatePublishedFirstChange(fixture, cancellationToken); - - // 4. Write commits, publish changes in a specific commit, then validate - var commits = await WriteCommitArtifacts(fixture, cancellationToken); - await TestCommits(fixture, commits, cancellationToken); - - await CleanUp(fixture, cancellationToken); - }, iter: 1, seed: "0000KOIPe036", threads: useExistingInstance ? 1 : -1); - } - - private static async ValueTask WriteProgress(string message) => - await TestContext.Progress.WriteLineAsync($"{DateTime.Now:O}: {message}"); - - private static async ValueTask OneTimeSetup(CancellationToken cancellationToken) - { - var serviceProviderUri = Configuration.ManagementServiceProviderUri; - var pipeline = Configuration.HttpPipeline; - - await WriteProgress("Deleting management services..."); - await ServiceModule.DeleteManagementServices(serviceProviderUri, serviceNamePrefix, pipeline, cancellationToken); - } - - private static async ValueTask CreateManagementServices(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Creating management services..."); - - var firstServiceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - var secondServiceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); - - if (useExistingInstance) - { - await Task.WhenAll(ServiceModule.DeleteAll(firstServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask(), - ServiceModule.DeleteAll(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask()); - } - else - { - await Task.WhenAll(ServiceModule.CreateManagementService(firstServiceUri, Configuration.Location, Configuration.HttpPipeline, cancellationToken).AsTask(), - ServiceModule.CreateManagementService(secondServiceUri, Configuration.Location, Configuration.HttpPipeline, cancellationToken).AsTask()); - } - } - - private static void DeleteServiceDirectory(Fixture fixture) - { - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - } - - private static async ValueTask PutServiceModel(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Putting service model..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - await ServiceModule.Put(fixture.ServiceModel, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask RunExtractor(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Running extractor..."); - - var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Extractor.Run(fixture.ExtractorOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, cancellationToken); - } - - private static async ValueTask ValidateExtractorArtifacts(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Validating extractor artifacts..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - await ServiceModule.ValidateExtractedArtifacts(fixture.ExtractorOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask PublishExtractedArtifactsToSecondService(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Extracting all artifacts from first instance..."); - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - - var bearerToken = await Configuration.GetBearerToken(cancellationToken); - var extractorOptions = fixture.ExtractorOptions with - { - ApiNamesToExport = Option>.None, - ProductNamesToExport = Option>.None, - GroupNamesToExport = Option>.None, - SubscriptionNamesToExport = Option>.None, - BackendNamesToExport = Option>.None, - LoggerNamesToExport = Option>.None, - DiagnosticNamesToExport = Option>.None, - PolicyFragmentNamesToExport = Option>.None, - GatewayNamesToExport = Option>.None, - TagNamesToExport = Option>.None, - VersionSetNamesToExport = Option>.None, - NamedValueNamesToExport = Option>.None, - }; - await Extractor.Run(extractorOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, cancellationToken); - - await WriteProgress("Publishing extracted artifacts to second instance..."); - await Publisher.Run(fixture.PublisherOptions, fixture.SecondServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId: Option.None, cancellationToken); - } - - private static async ValueTask ValidatePublishedExtractedArtifacts(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Validating published extracted artifacts..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); - await ServiceModule.ValidatePublisherChanges(fixture.PublisherOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask CleanUpExtractorResources(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Cleaning up extractor resources..."); - - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - - var firstServiceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - var secondServiceUri = Configuration.GetManagementServiceUri(fixture.SecondServiceName); - await Task.WhenAll(ServiceModule.DeleteAll(firstServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask(), - useExistingInstance - ? ServiceModule.DeleteAll(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask() - : ServiceModule.DeleteManagementService(secondServiceUri, Configuration.HttpPipeline, cancellationToken).AsTask()); - } - - private static async ValueTask WriteFirstChange(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Writing first change..."); - - var firstChange = fixture.PublishAllChangesModel; - await ServiceModule.WriteArtifacts(firstChange, fixture.ServiceDirectory, useExistingInstance, cancellationToken); - } - - private static async ValueTask PublishFirstChange(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Publishing first change..."); - - var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Publisher.Run(fixture.PublisherOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId: Option.None, cancellationToken); - } - - private static async ValueTask ValidatePublishedFirstChange(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Validating published first change..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - await ServiceModule.ValidatePublisherChanges(fixture.PublisherOptions, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask> WriteCommitArtifacts(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Writing commit artifacts..."); - - return await ServiceModule.WriteCommitArtifacts(fixture.CommitModels, fixture.ServiceDirectory, useExistingInstance, cancellationToken); - } - - private static async ValueTask TestCommits(Fixture fixture, ImmutableArray commits, CancellationToken cancellationToken) => - await commits.Take(1) - .ForEachAsync(async (commit, cancellationToken) => - { - await PublishCommit(fixture, commit, cancellationToken); - await ValidatePublishedCommit(fixture, commit, cancellationToken); - }, cancellationToken); - - private static async ValueTask PublishCommit(Fixture fixture, CommitId commitId, CancellationToken cancellationToken) - { - await WriteProgress($"Publishing commit {commitId}..."); - - var bearerToken = await Configuration.GetBearerToken(cancellationToken); - await Publisher.Run(fixture.PublisherOptions, fixture.FirstServiceName, fixture.ServiceDirectory, Configuration.SubscriptionId, Configuration.ResourceGroupName, bearerToken, commitId, cancellationToken); - } - - private static async ValueTask ValidatePublishedCommit(Fixture fixture, CommitId commitId, CancellationToken cancellationToken) - { - await WriteProgress($"Validating published commit {commitId}..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - await ServiceModule.ValidatePublisherCommitChanges(fixture.PublisherOptions, commitId, fixture.ServiceDirectory, serviceUri, Configuration.HttpPipeline, useExistingInstance, cancellationToken); - } - - private static async ValueTask CleanUp(Fixture fixture, CancellationToken cancellationToken) - { - await WriteProgress("Cleaning up..."); - - var serviceUri = Configuration.GetManagementServiceUri(fixture.FirstServiceName); - - if (useExistingInstance) - { - await ServiceModule.DeleteAll(serviceUri, Configuration.HttpPipeline, cancellationToken); - } - else - { - await ServiceModule.DeleteManagementService(serviceUri, Configuration.HttpPipeline, cancellationToken); - } - - ServiceModule.DeleteServiceDirectory(fixture.ServiceDirectory); - } -} \ No newline at end of file diff --git a/tools/code/integration.tests/VersionSet.cs b/tools/code/integration.tests/VersionSet.cs index 2fc298b9..e61e70c2 100644 --- a/tools/code/integration.tests/VersionSet.cs +++ b/tools/code/integration.tests/VersionSet.cs @@ -5,59 +5,67 @@ using FluentAssertions; using LanguageExt; using LanguageExt.UnsafeValueAccess; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; using publisher; using System; using System.Collections.Frozen; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace integration.tests; -internal static class VersionSet +internal delegate ValueTask DeleteAllVersionSets(ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask PutVersionSetModels(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken); + +internal delegate ValueTask ValidateExtractedVersionSets(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file delegate ValueTask> GetApimVersionSets(ManagementServiceName serviceName, CancellationToken cancellationToken); + +file delegate ValueTask> GetFileVersionSets(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken); + +internal delegate ValueTask WriteVersionSetModels(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +internal delegate ValueTask ValidatePublishedVersionSets(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken); + +file sealed class DeleteAllVersionSetsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) { - public static Gen GenerateUpdate(VersionSetModel original) => - from displayName in VersionSetModel.GenerateDisplayName() - from scheme in VersioningScheme.Generate() - from description in VersionSetModel.GenerateDescription().OptionOf() - select original with - { - DisplayName = displayName, - Scheme = scheme, - Description = description - }; + public async ValueTask Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(DeleteAllVersionSets)); - public static Gen GenerateOverride(VersionSetDto original) => - from displayName in VersionSetModel.GenerateDisplayName() - from header in GenerateHeaderOverride(original) - from query in GenerateQueryOverride(original) - from description in VersionSetModel.GenerateDescription().OptionOf() - select new VersionSetDto - { - Properties = new VersionSetDto.VersionSetContract - { - DisplayName = displayName, - Description = description.ValueUnsafe(), - VersionHeaderName = header, - VersionQueryName = query - } - }; + logger.LogInformation("Deleting all version sets in {ServiceName}...", serviceName); + var serviceUri = getServiceUri(serviceName); + await VersionSetsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); + } +} - private static Gen GenerateHeaderOverride(VersionSetDto original) => - Gen.OneOf(Gen.Const(original.Properties.VersionHeaderName), - string.IsNullOrWhiteSpace(original.Properties.VersionHeaderName) - ? Gen.Const(() => null as string)! - : VersioningScheme.Header.GenerateHeaderName()); +file sealed class PutVersionSetModelsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(PutVersionSetModels)); - private static Gen GenerateQueryOverride(VersionSetDto original) => - Gen.OneOf(Gen.Const(original.Properties.VersionQueryName), - string.IsNullOrWhiteSpace(original.Properties.VersionQueryName) - ? Gen.Const(() => null as string)! - : VersioningScheme.Query.GenerateQueryName()); + logger.LogInformation("Putting version set models in {ServiceName}...", serviceName); + await models.IterParallel(async model => + { + await Put(model, serviceName, cancellationToken); + }, cancellationToken); + } - public static FrozenDictionary GetDtoDictionary(IEnumerable models) => - models.ToFrozenDictionary(model => model.Name, GetDto); + private async ValueTask Put(VersionSetModel model, ManagementServiceName serviceName, CancellationToken cancellationToken) + { + var serviceUri = getServiceUri(serviceName); + var uri = VersionSetUri.From(model.Name, serviceUri); + var dto = GetDto(model); + + await uri.PutDto(dto, pipeline, cancellationToken); + } private static VersionSetDto GetDto(VersionSetModel model) => new() @@ -77,29 +85,113 @@ private static VersionSetDto GetDto(VersionSetModel model) => } } }; +} - public static async ValueTask Put(IEnumerable models, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await models.IterParallel(async model => +file sealed class ValidateExtractedVersionSetsHandler(ILogger logger, GetApimVersionSets getApimResources, GetFileVersionSets getFileResources, ActivitySource activitySource) +{ + public async ValueTask Handle(Option> namesFilterOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(ValidateExtractedVersionSets)); + + logger.LogInformation("Validating extracted version sets in {ServiceName}...", serviceName); + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, Prelude.None, cancellationToken); + + var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesFilterOption)) + .MapValue(NormalizeDto); + var actual = fileResources.MapValue(NormalizeDto); + + actual.Should().BeEquivalentTo(expected); + } + + private static string NormalizeDto(VersionSetDto dto) => + new { - await Put(model, serviceUri, pipeline, cancellationToken); - }, cancellationToken); + DisplayName = dto.Properties.DisplayName ?? string.Empty, + Description = dto.Properties.Description ?? string.Empty, + VersionHeaderName = dto.Properties.VersionHeaderName ?? string.Empty, + VersionQueryName = dto.Properties.VersionQueryName ?? string.Empty, + VersioningScheme = dto.Properties.VersioningScheme ?? string.Empty + }.ToString()!; +} - private static async ValueTask Put(VersionSetModel model, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +file sealed class GetApimVersionSetsHandler(ILogger logger, GetManagementServiceUri getServiceUri, HttpPipeline pipeline, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceName serviceName, CancellationToken cancellationToken) { - var uri = VersionSetUri.From(model.Name, serviceUri); - var dto = GetDto(model); + using var _ = activitySource.StartActivity(nameof(GetApimVersionSets)); - await uri.PutDto(dto, pipeline, cancellationToken); + logger.LogInformation("Getting version sets from {ServiceName}...", serviceName); + + var serviceUri = getServiceUri(serviceName); + var uri = VersionSetsUri.From(serviceUri); + + return await uri.List(pipeline, cancellationToken) + .ToFrozenDictionary(cancellationToken); } +} - public static async ValueTask DeleteAll(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) => - await VersionSetsUri.From(serviceUri).DeleteAll(pipeline, cancellationToken); +file sealed class GetFileVersionSetsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask> Handle(ManagementServiceDirectory serviceDirectory, Option commitIdOption, CancellationToken cancellationToken) => + await commitIdOption.Map(commitId => GetWithCommit(serviceDirectory, commitId, cancellationToken)) + .IfNone(() => GetWithoutCommit(serviceDirectory, cancellationToken)); + + private async ValueTask> GetWithCommit(ManagementServiceDirectory serviceDirectory, CommitId commitId, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileVersionSets)); + + logger.LogInformation("Getting version sets from {ServiceDirectory} as of commit {CommitId}...", serviceDirectory, commitId); + + return await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) + .ToAsyncEnumerable() + .Choose(file => VersionSetInformationFile.TryParse(file, serviceDirectory)) + .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) + .ToFrozenDictionary(cancellationToken); + } + + private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, VersionSetInformationFile file, CancellationToken cancellationToken) + { + var name = file.Parent.Name; + var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + + return await contentsOption.MapTask(async contents => + { + using (contents) + { + var data = await BinaryData.FromStreamAsync(contents, cancellationToken); + var dto = data.ToObjectFromJson(); + return (name, dto); + } + }); + } + + private async ValueTask> GetWithoutCommit(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(GetFileVersionSets)); + + logger.LogInformation("Getting version sets from {ServiceDirectory}...", serviceDirectory); + + return await VersionSetModule.ListInformationFiles(serviceDirectory) + .ToAsyncEnumerable() + .SelectAwait(async file => (file.Parent.Name, + await file.ReadDto(cancellationToken))) + .ToFrozenDictionary(cancellationToken); + } +} + +file sealed class WriteVersionSetModelsHandler(ILogger logger, ActivitySource activitySource) +{ + public async ValueTask Handle(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) + { + using var _ = activitySource.StartActivity(nameof(WriteVersionSetModels)); - public static async ValueTask WriteArtifacts(IEnumerable models, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => + logger.LogInformation("Writing version set models to {ServiceDirectory}...", serviceDirectory); await models.IterParallel(async model => { await WriteInformationFile(model, serviceDirectory, cancellationToken); }, cancellationToken); + } private static async ValueTask WriteInformationFile(VersionSetModel model, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { @@ -109,33 +201,44 @@ private static async ValueTask WriteInformationFile(VersionSetModel model, Manag await informationFile.WriteDto(dto, cancellationToken); } - public static async ValueTask ValidateExtractedArtifacts(Option> namesToExtract, ManagementServiceDirectory serviceDirectory, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + private static VersionSetDto GetDto(VersionSetModel model) => + new() + { + Properties = new VersionSetDto.VersionSetContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe(), + VersionHeaderName = model.Scheme is VersioningScheme.Header header ? header.HeaderName : null, + VersionQueryName = model.Scheme is VersioningScheme.Query query ? query.QueryName : null, + VersioningScheme = model.Scheme switch + { + VersioningScheme.Header => "Header", + VersioningScheme.Query => "Query", + VersioningScheme.Segment => "Segment", + _ => null + } + } + }; +} + +file sealed class ValidatePublishedVersionSetsHandler(ILogger logger, GetFileVersionSets getFileResources, GetApimVersionSets getApimResources, ActivitySource activitySource) +{ + public async ValueTask Handle(IDictionary overrides, Option commitIdOption, ManagementServiceName serviceName, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); + using var _ = activitySource.StartActivity(nameof(ValidatePublishedVersionSets)); - var expected = apimResources.WhereKey(name => ExtractorOptions.ShouldExtract(name, namesToExtract)) - .MapValue(NormalizeDto); - var actual = fileResources.MapValue(NormalizeDto); + logger.LogInformation("Validating published version sets in {ServiceDirectory}...", serviceDirectory); - actual.Should().BeEquivalentTo(expected); - } + var apimResources = await getApimResources(serviceName, cancellationToken); + var fileResources = await getFileResources(serviceDirectory, commitIdOption, cancellationToken); - private static async ValueTask> GetApimResources(ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) - { - var uri = VersionSetsUri.From(serviceUri); + var expected = PublisherOptions.Override(fileResources, overrides) + .MapValue(NormalizeDto); + var actual = apimResources.MapValue(NormalizeDto); - return await uri.List(pipeline, cancellationToken) - .ToFrozenDictionary(cancellationToken); + actual.Should().BeEquivalentTo(expected); } - private static async ValueTask> GetFileResources(ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await VersionSetModule.ListInformationFiles(serviceDirectory) - .ToAsyncEnumerable() - .SelectAwait(async file => (file.Parent.Name, - await file.ReadDto(cancellationToken))) - .ToFrozenDictionary(cancellationToken); - private static string NormalizeDto(VersionSetDto dto) => new { @@ -145,49 +248,125 @@ private static string NormalizeDto(VersionSetDto dto) => VersionQueryName = dto.Properties.VersionQueryName ?? string.Empty, VersioningScheme = dto.Properties.VersioningScheme ?? string.Empty }.ToString()!; +} - public static async ValueTask ValidatePublisherChanges(ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) +internal static class VersionSetServices +{ + public static void ConfigureDeleteAllVersionSets(IServiceCollection services) { - var fileResources = await GetFileResources(serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask ValidatePublisherChanges(IDictionary fileResources, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigurePutVersionSetModels(IServiceCollection services) { - var apimResources = await GetApimResources(serviceUri, pipeline, cancellationToken); + ManagementServices.ConfigureGetManagementServiceUri(services); - var expected = PublisherOptions.Override(fileResources, overrides) - .MapValue(NormalizeDto); - var actual = apimResources.MapValue(NormalizeDto); - actual.Should().BeEquivalentTo(expected); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - public static async ValueTask ValidatePublisherCommitChanges(CommitId commitId, ManagementServiceDirectory serviceDirectory, IDictionary overrides, ManagementServiceUri serviceUri, HttpPipeline pipeline, CancellationToken cancellationToken) + public static void ConfigureValidateExtractedVersionSets(IServiceCollection services) { - var fileResources = await GetFileResources(commitId, serviceDirectory, cancellationToken); - await ValidatePublisherChanges(fileResources, overrides, serviceUri, pipeline, cancellationToken); + ConfigureGetApimVersionSets(services); + ConfigureGetFileVersionSets(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static async ValueTask> GetFileResources(CommitId commitId, ManagementServiceDirectory serviceDirectory, CancellationToken cancellationToken) => - await Git.GetExistingFilesInCommit(serviceDirectory.ToDirectoryInfo(), commitId) - .ToAsyncEnumerable() - .Choose(file => VersionSetInformationFile.TryParse(file, serviceDirectory)) - .Choose(async file => await TryGetCommitResource(commitId, serviceDirectory, file, cancellationToken)) - .ToFrozenDictionary(cancellationToken); + private static void ConfigureGetApimVersionSets(IServiceCollection services) + { + ManagementServices.ConfigureGetManagementServiceUri(services); - private static async ValueTask> TryGetCommitResource(CommitId commitId, ManagementServiceDirectory serviceDirectory, VersionSetInformationFile file, CancellationToken cancellationToken) + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + private static void ConfigureGetFileVersionSets(IServiceCollection services) { - var name = file.Parent.Name; - var contentsOption = Git.TryGetFileContentsInCommit(serviceDirectory.ToDirectoryInfo(), file.ToFileInfo(), commitId); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } - return await contentsOption.MapTask(async contents => + public static void ConfigureWriteVersionSetModels(IServiceCollection services) + { + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + + public static void ConfigureValidatePublishedVersionSets(IServiceCollection services) + { + ConfigureGetFileVersionSets(services); + ConfigureGetApimVersionSets(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } +} + +internal static class VersionSet +{ + public static Gen GenerateUpdate(VersionSetModel original) => + from displayName in VersionSetModel.GenerateDisplayName() + from scheme in VersioningScheme.Generate() + from description in VersionSetModel.GenerateDescription().OptionOf() + select original with { - using (contents) + DisplayName = displayName, + Scheme = scheme, + Description = description + }; + + public static Gen GenerateOverride(VersionSetDto original) => + from displayName in VersionSetModel.GenerateDisplayName() + from header in GenerateHeaderOverride(original) + from query in GenerateQueryOverride(original) + from description in VersionSetModel.GenerateDescription().OptionOf() + select new VersionSetDto + { + Properties = new VersionSetDto.VersionSetContract { - var data = await BinaryData.FromStreamAsync(contents, cancellationToken); - var dto = data.ToObjectFromJson(); - return (name, dto); + DisplayName = displayName, + Description = description.ValueUnsafe(), + VersionHeaderName = header, + VersionQueryName = query } - }); - } + }; + + private static Gen GenerateHeaderOverride(VersionSetDto original) => + Gen.OneOf(Gen.Const(original.Properties.VersionHeaderName), + string.IsNullOrWhiteSpace(original.Properties.VersionHeaderName) + ? Gen.Const(() => null as string)! + : VersioningScheme.Header.GenerateHeaderName()); + + private static Gen GenerateQueryOverride(VersionSetDto original) => + Gen.OneOf(Gen.Const(original.Properties.VersionQueryName), + string.IsNullOrWhiteSpace(original.Properties.VersionQueryName) + ? Gen.Const(() => null as string)! + : VersioningScheme.Query.GenerateQueryName()); + + public static FrozenDictionary GetDtoDictionary(IEnumerable models) => + models.ToFrozenDictionary(model => model.Name, GetDto); + + private static VersionSetDto GetDto(VersionSetModel model) => + new() + { + Properties = new VersionSetDto.VersionSetContract + { + DisplayName = model.DisplayName, + Description = model.Description.ValueUnsafe(), + VersionHeaderName = model.Scheme is VersioningScheme.Header header ? header.HeaderName : null, + VersionQueryName = model.Scheme is VersioningScheme.Query query ? query.QueryName : null, + VersioningScheme = model.Scheme switch + { + VersioningScheme.Header => "Header", + VersioningScheme.Query => "Query", + VersioningScheme.Segment => "Segment", + _ => null + } + } + }; } diff --git a/tools/code/integration.tests/integration.tests.csproj b/tools/code/integration.tests/integration.tests.csproj index 5da383f3..6fce3f61 100644 --- a/tools/code/integration.tests/integration.tests.csproj +++ b/tools/code/integration.tests/integration.tests.csproj @@ -1,37 +1,25 @@  - - net8.0 - enable - true - CA2007,CA1707,CA1716,CA1724,CA1848 - latest-all - false - true - 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 - + + Exe + net8.0 + false + true + CA2007,CA1848,CA1812 + 6ce9dc18-ea0d-4d82-8f1a-e63877f35b88 + 8-all + enable + + + + + + - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - + + + + + diff --git a/tools/code/publisher/Common.cs b/tools/code/publisher/Common.cs index 1b440f94..2b5d2b9b 100644 --- a/tools/code/publisher/Common.cs +++ b/tools/code/publisher/Common.cs @@ -13,6 +13,7 @@ using Microsoft.IdentityModel.JsonWebTokens; using System; using System.Collections.Frozen; +using System.Diagnostics; using System.IO; using System.Reflection; using System.Threading; @@ -150,6 +151,7 @@ internal static class CommonServices { public static void Configure(IServiceCollection services) { + services.AddSingleton(GetActivitySource); services.TryAddSingleton(GetAzureEnvironment); services.TryAddSingleton(GetTokenCredential); services.TryAddSingleton(GetConfigurationJson); @@ -165,8 +167,12 @@ public static void Configure(IServiceCollection services) ConfigureGetArtifactsInPreviousCommit(services); services.ConfigureApimHttpClient(); + OpenTelemetryServices.Configure(services); } + private static ActivitySource GetActivitySource(IServiceProvider provider) => + new("ApiOps.Publisher"); + private static AzureEnvironment GetAzureEnvironment(IServiceProvider provider) { var configuration = provider.GetRequiredService(); diff --git a/tools/code/publisher/Git.cs b/tools/code/publisher/Git.cs index 3bfca618..270475ca 100644 --- a/tools/code/publisher/Git.cs +++ b/tools/code/publisher/Git.cs @@ -36,7 +36,8 @@ public static FrozenSet GetChangedFilesInCommit(DirectoryInfo reposito private static TreeChanges GetChanges(DirectoryInfo repositoryDirectory, CommitId commitId) { - using var repository = new Repository(repositoryDirectory.FullName); + var gitRootDirectory = repositoryDirectory.Parent!.FullName; + using var repository = new Repository(gitRootDirectory); var commit = GetCommit(repository, commitId); @@ -53,7 +54,8 @@ private static Commit GetCommit(Repository repository, CommitId commitId) => public static Option TryGetPreviousCommitId(DirectoryInfo repositoryDirectory, CommitId commitId) { - using var repository = new Repository(repositoryDirectory.FullName); + var gitRootDirectory = repositoryDirectory.Parent!.FullName; + using var repository = new Repository(gitRootDirectory); var commit = GetCommit(repository, commitId); @@ -66,7 +68,8 @@ public static Option TryGetPreviousCommitId(DirectoryInfo repositoryDi public static Option TryGetFileContentsInCommit(DirectoryInfo repositoryDirectory, FileInfo file, CommitId commitId) { - using var repository = new Repository(repositoryDirectory.FullName); + var gitRootDirectory = repositoryDirectory.Parent!.FullName; + using var repository = new Repository(gitRootDirectory); var relativePath = Path.GetRelativePath(repositoryDirectory.FullName, file.FullName); var relativePathString = Path.DirectorySeparatorChar == '\\' ? relativePath.Replace('\\', '/') @@ -81,7 +84,8 @@ public static Option TryGetFileContentsInCommit(DirectoryInfo repository public static FrozenSet GetExistingFilesInCommit(DirectoryInfo repositoryDirectory, CommitId commitId) { - using var repository = new Repository(repositoryDirectory.FullName); + var gitRootDirectory = repositoryDirectory.Parent!.FullName; + using var repository = new Repository(gitRootDirectory); var commit = GetCommit(repository, commitId); diff --git a/tools/code/publisher/publisher.csproj b/tools/code/publisher/publisher.csproj index bd29b26c..e58f3faf 100644 --- a/tools/code/publisher/publisher.csproj +++ b/tools/code/publisher/publisher.csproj @@ -2,8 +2,9 @@ net8.0 + false true - latest-all + 8-all CA1708,CA1724,CA1812,CA1848,CA2007,CA1034,CA1062 Exe enable @@ -13,7 +14,7 @@ - + From f37269360dfecc4f1387bb948987809e207077c6 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Fri, 24 May 2024 14:06:08 -0400 Subject: [PATCH 23/29] Update Git pathing --- tools/code/publisher/App.cs | 2 -- tools/code/publisher/Git.cs | 42 ++++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/tools/code/publisher/App.cs b/tools/code/publisher/App.cs index f7a70357..8bb27ede 100644 --- a/tools/code/publisher/App.cs +++ b/tools/code/publisher/App.cs @@ -23,12 +23,10 @@ file sealed class RunPublisherHandler(ProcessNamedValuesToPut processNamedValues public async ValueTask Handle(CancellationToken cancellationToken) { - logger.LogInformation("Putting named values..."); await processNamedValuesToPut(cancellationToken); await ProcessPublisherFiles(cancellationToken); - logger.LogInformation("Deleting named values..."); await processDeletedNamedValues(cancellationToken); logger.LogInformation("Publisher completed."); diff --git a/tools/code/publisher/Git.cs b/tools/code/publisher/Git.cs index 270475ca..a6a6dc80 100644 --- a/tools/code/publisher/Git.cs +++ b/tools/code/publisher/Git.cs @@ -34,10 +34,10 @@ public static FrozenSet GetChangedFilesInCommit(DirectoryInfo reposito .Select(path => new FileInfo(Path.Combine(repositoryDirectory.FullName, path))) .ToFrozenSet(x => x.FullName); - private static TreeChanges GetChanges(DirectoryInfo repositoryDirectory, CommitId commitId) + private static TreeChanges GetChanges(DirectoryInfo directory, CommitId commitId) { - var gitRootDirectory = repositoryDirectory.Parent!.FullName; - using var repository = new Repository(gitRootDirectory); + var repositoryDirectory = GetRepositoryDirectory(directory); + using var repository = new Repository(repositoryDirectory.FullName); var commit = GetCommit(repository, commitId); @@ -47,15 +47,32 @@ private static TreeChanges GetChanges(DirectoryInfo repositoryDirectory, CommitI .Compare(parentCommit?.Tree, commit.Tree); } + private static DirectoryInfo GetRepositoryDirectory(DirectoryInfo directory) + { + var repositoryDirectory = directory.EnumerateDirectories(".git", SearchOption.TopDirectoryOnly) + .FirstOrDefault(); + + if (repositoryDirectory is not null) + { + return directory; + } + + var parentDirectory = directory.Parent; + + return parentDirectory is null + ? throw new InvalidOperationException("Could not find a Git repository.") + : GetRepositoryDirectory(parentDirectory); + } + private static Commit GetCommit(Repository repository, CommitId commitId) => repository.Commits .Find(commit => commit.Id.Sha == commitId.Value) .IfNone(() => throw new InvalidOperationException($"Could not find commit with ID {commitId.Value}.")); - public static Option TryGetPreviousCommitId(DirectoryInfo repositoryDirectory, CommitId commitId) + public static Option TryGetPreviousCommitId(DirectoryInfo directory, CommitId commitId) { - var gitRootDirectory = repositoryDirectory.Parent!.FullName; - using var repository = new Repository(gitRootDirectory); + var repositoryDirectory = GetRepositoryDirectory(directory); + using var repository = new Repository(repositoryDirectory.FullName); var commit = GetCommit(repository, commitId); @@ -66,10 +83,11 @@ public static Option TryGetPreviousCommitId(DirectoryInfo repositoryDi }; } - public static Option TryGetFileContentsInCommit(DirectoryInfo repositoryDirectory, FileInfo file, CommitId commitId) + public static Option TryGetFileContentsInCommit(DirectoryInfo directory, FileInfo file, CommitId commitId) { - var gitRootDirectory = repositoryDirectory.Parent!.FullName; - using var repository = new Repository(gitRootDirectory); + var repositoryDirectory = GetRepositoryDirectory(directory); + using var repository = new Repository(repositoryDirectory.FullName); + var relativePath = Path.GetRelativePath(repositoryDirectory.FullName, file.FullName); var relativePathString = Path.DirectorySeparatorChar == '\\' ? relativePath.Replace('\\', '/') @@ -82,10 +100,10 @@ public static Option TryGetFileContentsInCommit(DirectoryInfo repository : blob.GetContentStream(); } - public static FrozenSet GetExistingFilesInCommit(DirectoryInfo repositoryDirectory, CommitId commitId) + public static FrozenSet GetExistingFilesInCommit(DirectoryInfo directory, CommitId commitId) { - var gitRootDirectory = repositoryDirectory.Parent!.FullName; - using var repository = new Repository(gitRootDirectory); + var repositoryDirectory = GetRepositoryDirectory(directory); + using var repository = new Repository(repositoryDirectory.FullName); var commit = GetCommit(repository, commitId); From 1cc5da55820d35c8f26379df9b066b4d16c6535b Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Thu, 30 May 2024 22:23:16 -0400 Subject: [PATCH 24/29] Fix Git file parsing issue --- tools/code/common/common.csproj | 2 +- .../integration.tests/ManagementService.cs | 37 +++++++++++++++---- tools/code/integration.tests/Test.cs | 2 +- tools/code/publisher/Git.cs | 28 +++++++------- tools/code/publisher/publisher.csproj | 2 +- 5 files changed, 48 insertions(+), 23 deletions(-) diff --git a/tools/code/common/common.csproj b/tools/code/common/common.csproj index eb08ee52..b96019ae 100644 --- a/tools/code/common/common.csproj +++ b/tools/code/common/common.csproj @@ -19,7 +19,7 @@ - + diff --git a/tools/code/integration.tests/ManagementService.cs b/tools/code/integration.tests/ManagementService.cs index 22926bed..f9e9a0c8 100644 --- a/tools/code/integration.tests/ManagementService.cs +++ b/tools/code/integration.tests/ManagementService.cs @@ -101,13 +101,36 @@ file sealed class CreateApimServiceHandler(ILogger logger, ManagementServiceProviderUri serviceProviderUri, AzureLocation location) { - private static readonly ResiliencePipeline resiliencePipeline = + private static readonly ResiliencePipeline httpResiliencePipeline = + new ResiliencePipelineBuilder() + .AddRetry(new() + { + ShouldHandle = async arguments => + { + await ValueTask.CompletedTask; + + return arguments.Outcome.Exception?.Message?.Contains("is transitioning at this time", StringComparison.OrdinalIgnoreCase) ?? false; + }, + Delay = TimeSpan.FromSeconds(5), + BackoffType = DelayBackoffType.Linear, + MaxRetryAttempts = 100 + }) + .AddTimeout(TimeSpan.FromMinutes(3)) + .Build(); + + private static readonly ResiliencePipeline statusResiliencePipeline = new ResiliencePipelineBuilder() .AddRetry(new() { ShouldHandle = async arguments => { await ValueTask.CompletedTask; + + if (arguments.Outcome.Exception?.Message?.Contains("is transitioning at this time", StringComparison.OrdinalIgnoreCase) ?? false) + { + return true; + } + var result = arguments.Outcome.Result; var succeeded = "Succeeded".Equals(result, StringComparison.OrdinalIgnoreCase); return succeeded is false; @@ -145,10 +168,10 @@ public async ValueTask Handle(ManagementServiceName serviceName, CancellationTok } }); - await pipeline.PutContent(uri, body, cancellationToken); + await httpResiliencePipeline.ExecuteAsync(async cancellationToken => await pipeline.PutContent(uri, body, cancellationToken), cancellationToken); // Wait until the service is successfully provisioned - await resiliencePipeline.ExecuteAsync(async cancellationToken => + await statusResiliencePipeline.ExecuteAsync(async cancellationToken => { var content = await pipeline.GetJsonObject(uri, cancellationToken); @@ -299,8 +322,8 @@ public async ValueTask> Handle(IEnumerable.Empty; await models.Map((index, model) => (index, model)) @@ -309,7 +332,7 @@ await models.Map((index, model) => (index, model)) var (index, model) = x; DeleteNonGitDirectories(serviceDirectory); await writeServiceModelArtifacts(model, serviceDirectory, cancellationToken); - var commit = Git.CommitChanges(serviceDirectoryInfo, commitMessage: $"Commit {index}", authorName, authorEmail, DateTimeOffset.UtcNow); + var commit = Git.CommitChanges(repositoryDirectory, commitMessage: $"Commit {index}", authorName, authorEmail, DateTimeOffset.UtcNow); var commitId = new CommitId(commit.Sha); ImmutableInterlocked.Update(ref commitIds, commitIds => commitIds.Add(commitId)); }, cancellationToken); @@ -383,7 +406,7 @@ private static void ConfigureAzureLocation(IServiceCollection services) var locationName = configuration.TryGetValue("AZURE_LOCATION") .IfNone("westus"); - return new AzureLocation(locationName); + return new AzureLocation("westus"); }); } diff --git a/tools/code/integration.tests/Test.cs b/tools/code/integration.tests/Test.cs index af5031f9..fe4906d9 100644 --- a/tools/code/integration.tests/Test.cs +++ b/tools/code/integration.tests/Test.cs @@ -500,7 +500,7 @@ from lorem in Generator.Lorem .ToArray() from suffixCharacters in Gen.Shuffle(characters, 8) let name = $"apiops-{new string(suffixCharacters)}" - let path = Path.Combine(Path.GetTempPath(), name) + let path = Path.Combine(Path.GetTempPath(), name, "artifacts") let directoryInfo = new DirectoryInfo(path) select ManagementServiceDirectory.From(directoryInfo); } \ No newline at end of file diff --git a/tools/code/publisher/Git.cs b/tools/code/publisher/Git.cs index a6a6dc80..c36d6e93 100644 --- a/tools/code/publisher/Git.cs +++ b/tools/code/publisher/Git.cs @@ -22,23 +22,25 @@ public CommitId(string value) public static class Git { - public static FrozenSet GetChangedFilesInCommit(DirectoryInfo repositoryDirectory, CommitId commitId) => - GetChanges(repositoryDirectory, commitId) - .SelectMany(change => (change.Path, change.OldPath) switch - { - (null, not null) => [change.OldPath], - (not null, null) => [change.Path], - (null, null) => [], - (var path, var oldPath) => new[] { path, oldPath }.Distinct() - }) - .Select(path => new FileInfo(Path.Combine(repositoryDirectory.FullName, path))) - .ToFrozenSet(x => x.FullName); - - private static TreeChanges GetChanges(DirectoryInfo directory, CommitId commitId) + public static FrozenSet GetChangedFilesInCommit(DirectoryInfo directory, CommitId commitId) { var repositoryDirectory = GetRepositoryDirectory(directory); using var repository = new Repository(repositoryDirectory.FullName); + return GetChanges(repository, commitId) + .SelectMany(change => (change.Path, change.OldPath) switch + { + (null, not null) => [change.OldPath], + (not null, null) => [change.Path], + (null, null) => [], + (var path, var oldPath) => new[] { path, oldPath }.Distinct() + }) + .Select(path => new FileInfo(Path.Combine(repositoryDirectory.FullName, path))) + .ToFrozenSet(x => x.FullName); + } + + private static TreeChanges GetChanges(Repository repository, CommitId commitId) + { var commit = GetCommit(repository, commitId); var parentCommit = commit.Parents.FirstOrDefault(); diff --git a/tools/code/publisher/publisher.csproj b/tools/code/publisher/publisher.csproj index e58f3faf..407e7dbf 100644 --- a/tools/code/publisher/publisher.csproj +++ b/tools/code/publisher/publisher.csproj @@ -14,7 +14,7 @@ - + From 736ce2ac7613e3411b2e8860ed4cc2cb3c2d6f9d Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Thu, 30 May 2024 22:23:40 -0400 Subject: [PATCH 25/29] Remove isCurrent from publisher DTO --- tools/code/common.tests/Api.cs | 6 +++--- tools/code/publisher/Api.cs | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/code/common.tests/Api.cs b/tools/code/common.tests/Api.cs index 3848d800..bd66b205 100644 --- a/tools/code/common.tests/Api.cs +++ b/tools/code/common.tests/Api.cs @@ -15,9 +15,9 @@ public sealed record Soap : ApiType; public static Gen Generate() => Gen.OneOfConst(new GraphQl(), - new Http(), - new WebSocket(), - new Soap()); + new Http()); + //new WebSocket(), + //new Soap()); } public record ApiRevision diff --git a/tools/code/publisher/Api.cs b/tools/code/publisher/Api.cs index 38dd6070..10961469 100644 --- a/tools/code/publisher/Api.cs +++ b/tools/code/publisher/Api.cs @@ -463,7 +463,10 @@ public async ValueTask Handle(ApiName name, ApiDto dto, CancellationToken cancel var revisionNumber = Common.GetRevisionNumber(dto); var uri = GetRevisionedUri(name, revisionNumber); - await uri.PutDto(dto, pipeline, cancellationToken); + + // APIM sometimes fails revisions if isCurrent is set to true. + var dtoWithoutIsCurrent = dto with { Properties = dto.Properties with { IsCurrent = null } }; + await uri.PutDto(dtoWithoutIsCurrent, pipeline, cancellationToken); } private ApiUri GetRevisionedUri(ApiName name, ApiRevisionNumber revisionNumber) From aae76b251991ce60b1823010d8aa69addd95335a Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Tue, 18 Jun 2024 12:28:43 -0400 Subject: [PATCH 26/29] DI --- tools/code/common/Enumerable.cs | 8 +- tools/code/common/common.csproj | 10 +- tools/code/extractor/Api.cs | 374 ++++++++++--------- tools/code/extractor/ApiOperation.cs | 55 +-- tools/code/extractor/ApiOperationPolicy.cs | 102 ++--- tools/code/extractor/ApiPolicy.cs | 102 ++--- tools/code/extractor/ApiTag.cs | 104 +++--- tools/code/extractor/App.cs | 84 +++-- tools/code/extractor/Backend.cs | 126 ++++--- tools/code/extractor/Common.cs | 36 ++ tools/code/extractor/Diagnostic.cs | 136 ++++--- tools/code/extractor/ShouldExtractFactory.cs | 45 ++- tools/code/extractor/Tag.cs | 4 +- tools/code/publisher/publisher.csproj | 2 +- 14 files changed, 665 insertions(+), 523 deletions(-) diff --git a/tools/code/common/Enumerable.cs b/tools/code/common/Enumerable.cs index 7955cc46..926746e6 100644 --- a/tools/code/common/Enumerable.cs +++ b/tools/code/common/Enumerable.cs @@ -231,7 +231,13 @@ public static ImmutableDictionary MapKey(th public static ImmutableDictionary MapValue(this IEnumerable> dictionary, Func f) where TKey : notnull => dictionary.ToImmutableDictionary(kvp => kvp.Key, kvp => f(kvp.Value)); + public static ImmutableDictionary ChooseKey(this IEnumerable> dictionary, Func> f) where TKey2 : notnull => + dictionary.Choose(kvp => from key2 in f(kvp.Key) + select KeyValuePair.Create(key2, kvp.Value)) + .ToImmutableDictionary(); + public static ImmutableDictionary ChooseValue(this IEnumerable> dictionary, Func> f) where TKey : notnull => - dictionary.Choose(kvp => f(kvp.Value).Map(value2 => KeyValuePair.Create(kvp.Key, value2))) + dictionary.Choose(kvp => from value2 in f(kvp.Value) + select KeyValuePair.Create(kvp.Key, value2)) .ToImmutableDictionary(); } \ No newline at end of file diff --git a/tools/code/common/common.csproj b/tools/code/common/common.csproj index b96019ae..b43d9bcf 100644 --- a/tools/code/common/common.csproj +++ b/tools/code/common/common.csproj @@ -10,20 +10,20 @@ - - + + - + - - + + diff --git a/tools/code/extractor/Api.cs b/tools/code/extractor/Api.cs index 0e5dbd56..65abdc31 100644 --- a/tools/code/extractor/Api.cs +++ b/tools/code/extractor/Api.cs @@ -15,245 +15,261 @@ namespace extractor; internal delegate ValueTask ExtractApis(CancellationToken cancellationToken); -file delegate IAsyncEnumerable<(ApiName Name, ApiDto Dto, Option<(ApiSpecification Specification, BinaryData Contents)> SpecificationOption)> ListApis(CancellationToken cancellationToken); +internal delegate IAsyncEnumerable<(ApiName Name, ApiDto Dto, Option<(ApiSpecification Specification, BinaryData Contents)> SpecificationOption)> ListApis(CancellationToken cancellationToken); internal delegate bool ShouldExtractApiName(ApiName name); -file delegate bool ShouldExtractApiDto(ApiDto dto); +internal delegate bool ShouldExtractApiDto(ApiDto dto); -file delegate ValueTask WriteApiArtifacts(ApiName name, ApiDto dto, Option<(ApiSpecification Specification, BinaryData Contents)> specificationOption, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiArtifacts(ApiName name, ApiDto dto, Option<(ApiSpecification Specification, BinaryData Contents)> specificationOption, CancellationToken cancellationToken); -file delegate ValueTask WriteApiInformationFile(ApiName name, ApiDto dto, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiInformationFile(ApiName name, ApiDto dto, CancellationToken cancellationToken); -file delegate ValueTask WriteApiSpecificationFile(ApiName name, ApiSpecification specification, BinaryData contents, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiSpecificationFile(ApiName name, ApiSpecification specification, BinaryData contents, CancellationToken cancellationToken); -file sealed class ExtractApisHandler(ListApis list, - ShouldExtractApiName shouldExtractName, - ShouldExtractApiDto shouldExtractDto, - WriteApiArtifacts writeArtifacts, - ExtractApiPolicies extractApiPolicies, - ExtractApiTags extractApiTags, - ExtractApiOperations extractApiOperations) +internal static class ApiServices { - public async ValueTask Handle(CancellationToken cancellationToken) => - await list(cancellationToken) - .Where(api => shouldExtractName(api.Name)) - .Where(api => shouldExtractDto(api.Dto)) - // Group APIs by version set (https://github.com/Azure/apiops/issues/316). - // We'll process each group in parallel, but each API within a group sequentially. - .GroupBy(api => api.Dto.Properties.ApiVersionSetId ?? Guid.NewGuid().ToString()) - .IterParallel(async group => await group.Iter(async api => await ExtractApi(api.Name, api.Dto, api.SpecificationOption, cancellationToken), - cancellationToken), - cancellationToken); - - private async ValueTask ExtractApi(ApiName name, ApiDto dto, Option<(ApiSpecification Specification, BinaryData Contents)> specificationOption, CancellationToken cancellationToken) + public static void ConfigureExtractApis(IServiceCollection services) { - await writeArtifacts(name, dto, specificationOption, cancellationToken); - await extractApiPolicies(name, cancellationToken); - await extractApiTags(name, cancellationToken); - await extractApiOperations(name, cancellationToken); + ConfigureListApis(services); + ConfigureShouldExtractApiName(services); + ConfigureShouldExtractApiDto(services); + ConfigureWriteApiArtifacts(services); + ApiPolicyServices.ConfigureExtractApiPolicies(services); + ApiTagServices.ConfigureExtractApiTags(services); + ApiOperationServices.ConfigureExtractApiOperations(services); + + services.TryAddSingleton(ExtractApis); } -} -file sealed class ListApisHandler(ManagementServiceUri serviceUri, HttpPipeline pipeline, IConfiguration configuration) -{ - private readonly ApiSpecification defaultApiSpecification = GetDefaultApiSpecification(configuration); - - public IAsyncEnumerable<(ApiName, ApiDto, Option<(ApiSpecification, BinaryData)>)> Handle(CancellationToken cancellationToken) => - ApisUri.From(serviceUri) - .List(pipeline, cancellationToken) - .SelectAwait(async api => - { - var (name, dto) = api; - var specificationContentsOption = await TryGetSpecificationContents(name, dto, cancellationToken); - return (name, dto, specificationContentsOption); - }); - - private async ValueTask> TryGetSpecificationContents(ApiName name, ApiDto dto, CancellationToken cancellationToken) + private static ExtractApis ExtractApis(IServiceProvider provider) { - var specificationOption = TryGetSpecification(dto); - - return await specificationOption.BindTask(async specification => + var list = provider.GetRequiredService(); + var shouldExtractName = provider.GetRequiredService(); + var shouldExtractDto = provider.GetRequiredService(); + var writeArtifacts = provider.GetRequiredService(); + var extractApiPolicies = provider.GetRequiredService(); + var extractApiTags = provider.GetRequiredService(); + var extractApiOperations = provider.GetRequiredService(); + + return async cancellationToken => + await list(cancellationToken) + .Where(api => shouldExtractName(api.Name)) + .Where(api => shouldExtractDto(api.Dto)) + // Group APIs by version set (https://github.com/Azure/apiops/issues/316). + // We'll process each group in parallel, but each API within a group sequentially. + .GroupBy(api => api.Dto.Properties.ApiVersionSetId ?? string.Empty) + .IterParallel(async group => await group.Iter(async api => await extractApi(api.Name, api.Dto, api.SpecificationOption, cancellationToken), + cancellationToken), + cancellationToken); + + async ValueTask extractApi(ApiName name, ApiDto dto, Option<(ApiSpecification Specification, BinaryData Contents)> specificationOption, CancellationToken cancellationToken) { - var uri = ApiUri.From(name, serviceUri); - var contentsOption = await uri.TryGetSpecificationContents(specification, pipeline, cancellationToken); - - return from contents in contentsOption - select (specification, contents); - }); + await writeArtifacts(name, dto, specificationOption, cancellationToken); + await extractApiPolicies(name, cancellationToken); + await extractApiTags(name, cancellationToken); + await extractApiOperations(name, cancellationToken); + } } - private static ApiSpecification GetDefaultApiSpecification(IConfiguration configuration) + private static void ConfigureListApis(IServiceCollection services) { - var formatOption = configuration.TryGetValue("API_SPECIFICATION_FORMAT") - | configuration.TryGetValue("apiSpecificationFormat"); + CommonServices.ConfigureManagementServiceUri(services); + CommonServices.ConfigureHttpPipeline(services); - return formatOption.Map(format => format switch + services.TryAddSingleton(ListApis); + } + + private static ListApis ListApis(IServiceProvider provider) + { + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + var configuration = provider.GetRequiredService(); + + var defaultApiSpecification = getDefaultApiSpecification(configuration); + + return cancellationToken => + ApisUri.From(serviceUri) + .List(pipeline, cancellationToken) + .SelectAwait(async api => + { + var (name, dto) = api; + var specificationContentsOption = await tryGetSpecificationContents(name, dto, cancellationToken); + return (name, dto, specificationContentsOption); + }); + + static ApiSpecification getDefaultApiSpecification(IConfiguration configuration) { - var value when "Wadl".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.Wadl() as ApiSpecification, - var value when "JSON".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi - { - Format = new OpenApiFormat.Json(), - Version = new OpenApiVersion.V3() - }, - var value when "YAML".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi - { - Format = new OpenApiFormat.Yaml(), - Version = new OpenApiVersion.V3() - }, - var value when "OpenApiV2Json".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi - { - Format = new OpenApiFormat.Json(), - Version = new OpenApiVersion.V2() - }, - var value when "OpenApiV2Yaml".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi - { - Format = new OpenApiFormat.Yaml(), - Version = new OpenApiVersion.V2() - }, - var value when "OpenApiV3Json".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi + var formatOption = configuration.TryGetValue("API_SPECIFICATION_FORMAT") + | configuration.TryGetValue("apiSpecificationFormat"); + + return formatOption.Map(format => format switch { - Format = new OpenApiFormat.Json(), - Version = new OpenApiVersion.V3() - }, - var value when "OpenApiV3Yaml".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi + var value when "Wadl".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.Wadl() as ApiSpecification, + var value when "JSON".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi + { + Format = new OpenApiFormat.Json(), + Version = new OpenApiVersion.V3() + }, + var value when "YAML".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi + { + Format = new OpenApiFormat.Yaml(), + Version = new OpenApiVersion.V3() + }, + var value when "OpenApiV2Json".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi + { + Format = new OpenApiFormat.Json(), + Version = new OpenApiVersion.V2() + }, + var value when "OpenApiV2Yaml".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi + { + Format = new OpenApiFormat.Yaml(), + Version = new OpenApiVersion.V2() + }, + var value when "OpenApiV3Json".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi + { + Format = new OpenApiFormat.Json(), + Version = new OpenApiVersion.V3() + }, + var value when "OpenApiV3Yaml".Equals(value, StringComparison.OrdinalIgnoreCase) => new ApiSpecification.OpenApi + { + Format = new OpenApiFormat.Yaml(), + Version = new OpenApiVersion.V3() + }, + var value => throw new NotSupportedException($"API specification format '{value}' defined in configuration is not supported.") + }).IfNone(() => new ApiSpecification.OpenApi { Format = new OpenApiFormat.Yaml(), Version = new OpenApiVersion.V3() - }, - var value => throw new NotSupportedException($"API specification format '{value}' defined in configuration is not supported.") - }).IfNone(() => new ApiSpecification.OpenApi - { - Format = new OpenApiFormat.Yaml(), - Version = new OpenApiVersion.V3() - }); - } + }); + } - private Option TryGetSpecification(ApiDto dto) => - (dto.Properties.Type ?? dto.Properties.ApiType) switch + async ValueTask> tryGetSpecificationContents(ApiName name, ApiDto dto, CancellationToken cancellationToken) { - "graphql" => new ApiSpecification.GraphQl(), - "soap" => new ApiSpecification.Wsdl(), - "http" => defaultApiSpecification, - null => defaultApiSpecification, - _ => Option.None - }; -} + var specificationOption = tryGetSpecification(dto); -file sealed class ShouldExtractApiNameHandler(ShouldExtractFactory shouldExtractFactory) -{ - public bool Handle(ApiName name) - { - var shouldExtract = shouldExtractFactory.Create(); - return shouldExtract(name); - } -} + return await specificationOption.BindTask(async specification => + { + var uri = ApiUri.From(name, serviceUri); + var contentsOption = await uri.TryGetSpecificationContents(specification, pipeline, cancellationToken); -file sealed class ShouldExtractApiDtoHandler(ShouldExtractVersionSet shouldExtractVersionSet) -{ - public bool Handle(ApiDto dto) => - // Don't extract if its version set should not be extracted - ApiModule.TryGetVersionSetName(dto) - .Map(shouldExtractVersionSet.Invoke) - .IfNone(true); -} + return from contents in contentsOption + select (specification, contents); + }); + } -file sealed class WriteApiArtifactsHandler(WriteApiInformationFile writeInformationFile, - WriteApiSpecificationFile writeSpecificationFile) -{ - public async ValueTask Handle(ApiName name, ApiDto dto, Option<(ApiSpecification, BinaryData)> specificationContentsOption, CancellationToken cancellationToken) + Option tryGetSpecification(ApiDto dto) => + (dto.Properties.Type ?? dto.Properties.ApiType) switch + { + "graphql" => new ApiSpecification.GraphQl(), + "soap" => new ApiSpecification.Wsdl(), + "http" => defaultApiSpecification, + null => defaultApiSpecification, + _ => Option.None + }; + } + + public static void ConfigureShouldExtractApiName(IServiceCollection services) { - await writeInformationFile(name, dto, cancellationToken); + CommonServices.ConfigureShouldExtractFactory(services); - await specificationContentsOption.IterTask(async x => - { - var (specification, contents) = x; - await writeSpecificationFile(name, specification, contents, cancellationToken); - }); + services.TryAddSingleton(ShouldExtractApiName); } -} - -file sealed class WriteApiInformationFileHandler(ILoggerFactory loggerFactory, ManagementServiceDirectory serviceDirectory) -{ - private readonly ILogger logger = Common.GetLogger(loggerFactory); - public async ValueTask Handle(ApiName name, ApiDto dto, CancellationToken cancellationToken) + private static ShouldExtractApiName ShouldExtractApiName(IServiceProvider provider) { - var informationFile = ApiInformationFile.From(name, serviceDirectory); + var shouldExtractFactory = provider.GetRequiredService(); - logger.LogInformation("Writing API information file {ApiInformationFile}...", informationFile); - await informationFile.WriteDto(dto, cancellationToken); - } -} + var shouldExtract = shouldExtractFactory.Create(); -file sealed class WriteApiSpecificationFileHandler(ILoggerFactory loggerFactory, ManagementServiceDirectory serviceDirectory) -{ - private readonly ILogger logger = Common.GetLogger(loggerFactory); + return name => shouldExtract(name); + } - public async ValueTask Handle(ApiName name, ApiSpecification specification, BinaryData contents, CancellationToken cancellationToken) + public static void ConfigureShouldExtractApiDto(IServiceCollection services) { - var specificationFile = ApiSpecificationFile.From(specification, name, serviceDirectory); - - logger.LogInformation("Writing API specification file {ApiSpecificationFile}...", specificationFile); - await specificationFile.WriteSpecification(contents, cancellationToken); + services.TryAddSingleton(ShouldExtractApiDto); } -} -internal static class ApiServices -{ - public static void ConfigureExtractApis(IServiceCollection services) + private static ShouldExtractApiDto ShouldExtractApiDto(IServiceProvider provider) { - ConfigureListApis(services); - ConfigureShouldExtractApiName(services); - ConfigureShouldExtractApiDto(services); - ConfigureWriteApiArtifacts(services); - ApiPolicyServices.ConfigureExtractApiPolicies(services); - ApiTagServices.ConfigureExtractApiTags(services); - ApiOperationServices.ConfigureExtractApiOperations(services); + var shouldExtractVersionSet = provider.GetRequiredService(); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + return dto => + // Don't extract if its version set should not be extracted + ApiModule.TryGetVersionSetName(dto) + .Map(shouldExtractVersionSet.Invoke) + .IfNone(true); } - private static void ConfigureListApis(IServiceCollection services) + private static void ConfigureWriteApiArtifacts(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + ConfigureWriteApiInformationFile(services); + ConfigureWriteApiSpecificationFile(services); + + services.TryAddSingleton(WriteApiArtifacts); } - public static void ConfigureShouldExtractApiName(IServiceCollection services) + private static WriteApiArtifacts WriteApiArtifacts(IServiceProvider provider) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + var writeInformationFile = provider.GetRequiredService(); + var writeSpecificationFile = provider.GetRequiredService(); + + return async (name, dto, specificationContentsOption, cancellationToken) => + { + await writeInformationFile(name, dto, cancellationToken); + + await specificationContentsOption.IterTask(async x => + { + var (specification, contents) = x; + await writeSpecificationFile(name, specification, contents, cancellationToken); + }); + }; } - public static void ConfigureShouldExtractApiDto(IServiceCollection services) + private static void ConfigureWriteApiInformationFile(IServiceCollection services) { - VersionSetServices.ConfigureShouldExtractVersionSet(services); + CommonServices.ConfigureManagementServiceDirectory(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(WriteApiInformationFile); } - private static void ConfigureWriteApiArtifacts(IServiceCollection services) + private static WriteApiInformationFile WriteApiInformationFile(IServiceProvider provider) { - ConfigureWriteApiInformationFile(services); - ConfigureWriteApiSpecificationFile(services); + var serviceDirectory = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = Common.GetLogger(loggerFactory); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + return async (name, dto, cancellationToken) => + { + var informationFile = ApiInformationFile.From(name, provider.GetRequiredService()); + + logger.LogInformation("Writing API information file {ApiInformationFile}...", informationFile); + await informationFile.WriteDto(dto, cancellationToken); + }; } - private static void ConfigureWriteApiInformationFile(IServiceCollection services) + private static void ConfigureWriteApiSpecificationFile(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureManagementServiceDirectory(services); + + services.TryAddSingleton(WriteApiSpecificationFile); } - private static void ConfigureWriteApiSpecificationFile(IServiceCollection services) + private static WriteApiSpecificationFile WriteApiSpecificationFile(IServiceProvider provider) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + var serviceDirectory = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = Common.GetLogger(loggerFactory); + + return async (name, specification, contents, cancellationToken) => + { + var specificationFile = ApiSpecificationFile.From(specification, name, serviceDirectory); + + logger.LogInformation("Writing API specification file {ApiSpecificationFile}...", specificationFile); + await specificationFile.WriteSpecification(contents, cancellationToken); + }; } } diff --git a/tools/code/extractor/ApiOperation.cs b/tools/code/extractor/ApiOperation.cs index eb73b1a4..bc937e93 100644 --- a/tools/code/extractor/ApiOperation.cs +++ b/tools/code/extractor/ApiOperation.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -11,26 +12,7 @@ namespace extractor; internal delegate ValueTask ExtractApiOperations(ApiName apiName, CancellationToken cancellationToken); -file delegate IAsyncEnumerable ListApiOperations(ApiName apiName, CancellationToken cancellationToken); - -file sealed class ExtractApiOperationsHandler(ListApiOperations list, ExtractApiOperationPolicies extractApiOperationPolicies) -{ - public async ValueTask Handle(ApiName apiName, CancellationToken cancellationToken) => - await list(apiName, cancellationToken) - .IterParallel(async name => await ExtractApiOperation(name, apiName, cancellationToken), - cancellationToken); - - private async ValueTask ExtractApiOperation(ApiOperationName name, ApiName apiName, CancellationToken cancellationToken) - { - await extractApiOperationPolicies(name, apiName, cancellationToken); - } -} - -file sealed class ListApiOperationsHandler(ManagementServiceUri serviceUri, HttpPipeline pipeline) -{ - public IAsyncEnumerable Handle(ApiName apiName, CancellationToken cancellationToken) => - ApiOperationsUri.From(apiName, serviceUri).ListNames(pipeline, cancellationToken); -} +internal delegate IAsyncEnumerable ListApiOperations(ApiName apiName, CancellationToken cancellationToken); internal static class ApiOperationServices { @@ -39,14 +21,39 @@ public static void ConfigureExtractApiOperations(IServiceCollection services) ConfigureListApiOperations(services); ApiOperationPolicyServices.ConfigureExtractApiOperationPolicies(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(ExtractApiOperations); + } + + private static ExtractApiOperations ExtractApiOperations(IServiceProvider provider) + { + var list = provider.GetRequiredService(); + var extractPolicies = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = Common.GetLogger(loggerFactory); + + return async (apiName, cancellationToken) => + await list(apiName, cancellationToken) + .IterParallel(async name => await extractPolicies(name, apiName, cancellationToken), + cancellationToken); } private static void ConfigureListApiOperations(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureManagementServiceUri(services); + CommonServices.ConfigureHttpPipeline(services); + + services.TryAddSingleton(ListApiOperations); + } + + private static ListApiOperations ListApiOperations(IServiceProvider provider) + { + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + + return (apiName, cancellationToken) => + ApiOperationsUri.From(apiName, serviceUri) + .ListNames(pipeline, cancellationToken); } } diff --git a/tools/code/extractor/ApiOperationPolicy.cs b/tools/code/extractor/ApiOperationPolicy.cs index 6b85c418..8cf3acc2 100644 --- a/tools/code/extractor/ApiOperationPolicy.cs +++ b/tools/code/extractor/ApiOperationPolicy.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -11,77 +12,88 @@ namespace extractor; internal delegate ValueTask ExtractApiOperationPolicies(ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken); -file delegate IAsyncEnumerable<(ApiOperationPolicyName Name, ApiOperationPolicyDto Dto)> ListApiOperationPolicies(ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken); +internal delegate IAsyncEnumerable<(ApiOperationPolicyName Name, ApiOperationPolicyDto Dto)> ListApiOperationPolicies(ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken); -file delegate ValueTask WriteApiOperationPolicyArtifacts(ApiOperationPolicyName name, ApiOperationPolicyDto dto, ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiOperationPolicyArtifacts(ApiOperationPolicyName name, ApiOperationPolicyDto dto, ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken); -file delegate ValueTask WriteApiOperationPolicyFile(ApiOperationPolicyName name, ApiOperationPolicyDto dto, ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiOperationPolicyFile(ApiOperationPolicyName name, ApiOperationPolicyDto dto, ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken); -file sealed class ExtractApiOperationPoliciesHandler(ListApiOperationPolicies list, WriteApiOperationPolicyArtifacts writeArtifacts) -{ - public async ValueTask Handle(ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken) => - await list(apiOperationName, apiName, cancellationToken) - .IterParallel(async apiOperationpolicy => await writeArtifacts(apiOperationpolicy.Name, apiOperationpolicy.Dto, apiOperationName, apiName, cancellationToken), - cancellationToken); -} - -file sealed class ListApiOperationPoliciesHandler(ManagementServiceUri serviceUri, HttpPipeline pipeline) -{ - public IAsyncEnumerable<(ApiOperationPolicyName, ApiOperationPolicyDto)> Handle(ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken) => - ApiOperationPoliciesUri.From(apiOperationName, apiName, serviceUri).List(pipeline, cancellationToken); -} - -file sealed class WriteApiOperationPolicyArtifactsHandler(WriteApiOperationPolicyFile writePolicyFile) +internal static class ApiOperationPolicyServices { - public async ValueTask Handle(ApiOperationPolicyName name, ApiOperationPolicyDto dto, ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken) + public static void ConfigureExtractApiOperationPolicies(IServiceCollection services) { - await writePolicyFile(name, dto, apiOperationName, apiName, cancellationToken); - } -} + ConfigureListApiOperationPolicies(services); + ConfigureWriteApiOperationPolicyArtifacts(services); -file sealed class WriteApiOperationPolicyFileHandler(ILoggerFactory loggerFactory, ManagementServiceDirectory serviceDirectory) -{ - private readonly ILogger logger = Common.GetLogger(loggerFactory); + services.TryAddSingleton(ExtractApiOperationPolicies); + } - public async ValueTask Handle(ApiOperationPolicyName name, ApiOperationPolicyDto dto, ApiOperationName apiOperationName, ApiName apiName, CancellationToken cancellationToken) + private static ExtractApiOperationPolicies ExtractApiOperationPolicies(IServiceProvider provider) { - var policyFile = ApiOperationPolicyFile.From(name, apiOperationName, apiName, serviceDirectory); + var list = provider.GetRequiredService(); + var writeArtifacts = provider.GetRequiredService(); - logger.LogInformation("Writing API operation policy file {ApiOperationPolicyFile}...", policyFile); - var policy = dto.Properties.Value ?? string.Empty; - await policyFile.WritePolicy(policy, cancellationToken); + return async (operationName, apiName, cancellationToken) => + await list(operationName, apiName, cancellationToken) + .IterParallel(async policy => await writeArtifacts(policy.Name, policy.Dto, operationName, apiName, cancellationToken), + cancellationToken); } -} -internal static class ApiOperationPolicyServices -{ - public static void ConfigureExtractApiOperationPolicies(IServiceCollection services) + private static void ConfigureListApiOperationPolicies(IServiceCollection services) { - ConfigureListApiOperationPolicies(services); - ConfigureWriteApiOperationPolicyArtifacts(services); + CommonServices.ConfigureManagementServiceUri(services); + CommonServices.ConfigureHttpPipeline(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(ListApiOperationPolicies); } - private static void ConfigureListApiOperationPolicies(IServiceCollection services) + private static ListApiOperationPolicies ListApiOperationPolicies(IServiceProvider provider) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + + return (operationName, apiName, cancellationToken) => + ApiOperationPoliciesUri.From(operationName, apiName, serviceUri) + .List(pipeline, cancellationToken); } private static void ConfigureWriteApiOperationPolicyArtifacts(IServiceCollection services) { ConfigureWriteApiOperationPolicyFile(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(WriteApiOperationPolicyArtifacts); + } + + private static WriteApiOperationPolicyArtifacts WriteApiOperationPolicyArtifacts(IServiceProvider provider) + { + var writePolicyFile = provider.GetRequiredService(); + + return async (name, dto, operationName, apiName, cancellationToken) => + await writePolicyFile(name, dto, operationName, apiName, cancellationToken); } private static void ConfigureWriteApiOperationPolicyFile(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureManagementServiceDirectory(services); + + services.TryAddSingleton(WriteApiOperationPolicyFile); + } + + private static WriteApiOperationPolicyFile WriteApiOperationPolicyFile(IServiceProvider provider) + { + var serviceDirectory = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = Common.GetLogger(loggerFactory); + + return async (name, dto, operationName, apiName, cancellationToken) => + { + var policyFile = ApiOperationPolicyFile.From(name, operationName, apiName, serviceDirectory); + + logger.LogInformation("Writing API operation policy file {ApiOperationPolicyFile}...", policyFile); + var policy = dto.Properties.Value ?? string.Empty; + await policyFile.WritePolicy(policy, cancellationToken); + }; } } diff --git a/tools/code/extractor/ApiPolicy.cs b/tools/code/extractor/ApiPolicy.cs index f1290e3e..77b7e544 100644 --- a/tools/code/extractor/ApiPolicy.cs +++ b/tools/code/extractor/ApiPolicy.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -11,77 +12,88 @@ namespace extractor; internal delegate ValueTask ExtractApiPolicies(ApiName apiName, CancellationToken cancellationToken); -file delegate IAsyncEnumerable<(ApiPolicyName Name, ApiPolicyDto Dto)> ListApiPolicies(ApiName apiName, CancellationToken cancellationToken); +internal delegate IAsyncEnumerable<(ApiPolicyName Name, ApiPolicyDto Dto)> ListApiPolicies(ApiName apiName, CancellationToken cancellationToken); -file delegate ValueTask WriteApiPolicyArtifacts(ApiPolicyName name, ApiPolicyDto dto, ApiName apiName, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiPolicyArtifacts(ApiPolicyName name, ApiPolicyDto dto, ApiName apiName, CancellationToken cancellationToken); -file delegate ValueTask WriteApiPolicyFile(ApiPolicyName name, ApiPolicyDto dto, ApiName apiName, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiPolicyFile(ApiPolicyName name, ApiPolicyDto dto, ApiName apiName, CancellationToken cancellationToken); -file sealed class ExtractApiPoliciesHandler(ListApiPolicies list, WriteApiPolicyArtifacts writeArtifacts) -{ - public async ValueTask Handle(ApiName apiName, CancellationToken cancellationToken) => - await list(apiName, cancellationToken) - .IterParallel(async apipolicy => await writeArtifacts(apipolicy.Name, apipolicy.Dto, apiName, cancellationToken), - cancellationToken); -} - -file sealed class ListApiPoliciesHandler(ManagementServiceUri serviceUri, HttpPipeline pipeline) -{ - public IAsyncEnumerable<(ApiPolicyName, ApiPolicyDto)> Handle(ApiName apiName, CancellationToken cancellationToken) => - ApiPoliciesUri.From(apiName, serviceUri).List(pipeline, cancellationToken); -} - -file sealed class WriteApiPolicyArtifactsHandler(WriteApiPolicyFile writePolicyFile) +internal static class ApiPolicyServices { - public async ValueTask Handle(ApiPolicyName name, ApiPolicyDto dto, ApiName apiName, CancellationToken cancellationToken) + public static void ConfigureExtractApiPolicies(IServiceCollection services) { - await writePolicyFile(name, dto, apiName, cancellationToken); - } -} + ConfigureListApiPolicies(services); + ConfigureWriteApiPolicyArtifacts(services); -file sealed class WriteApiPolicyFileHandler(ILoggerFactory loggerFactory, ManagementServiceDirectory serviceDirectory) -{ - private readonly ILogger logger = Common.GetLogger(loggerFactory); + services.TryAddSingleton(ExtractApiPolicies); + } - public async ValueTask Handle(ApiPolicyName name, ApiPolicyDto dto, ApiName apiName, CancellationToken cancellationToken) + private static ExtractApiPolicies ExtractApiPolicies(IServiceProvider provider) { - var policyFile = ApiPolicyFile.From(name, apiName, serviceDirectory); + var list = provider.GetRequiredService(); + var writeArtifacts = provider.GetRequiredService(); - logger.LogInformation("Writing API policy file {ApiPolicyFile}...", policyFile); - var policy = dto.Properties.Value ?? string.Empty; - await policyFile.WritePolicy(policy, cancellationToken); + return async (apiName, cancellationToken) => + await list(apiName, cancellationToken) + .IterParallel(async policy => await writeArtifacts(policy.Name, policy.Dto, apiName, cancellationToken), + cancellationToken); } -} -internal static class ApiPolicyServices -{ - public static void ConfigureExtractApiPolicies(IServiceCollection services) + private static void ConfigureListApiPolicies(IServiceCollection services) { - ConfigureListApiPolicies(services); - ConfigureWriteApiPolicyArtifacts(services); + CommonServices.ConfigureManagementServiceUri(services); + CommonServices.ConfigureHttpPipeline(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(ListApiPolicies); } - private static void ConfigureListApiPolicies(IServiceCollection services) + private static ListApiPolicies ListApiPolicies(IServiceProvider provider) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); + + return (apiName, cancellationToken) => + ApiPoliciesUri.From(apiName, serviceUri) + .List(pipeline, cancellationToken); } private static void ConfigureWriteApiPolicyArtifacts(IServiceCollection services) { ConfigureWriteApiPolicyFile(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(WriteApiPolicyArtifacts); + } + + private static WriteApiPolicyArtifacts WriteApiPolicyArtifacts(IServiceProvider provider) + { + var writePolicyFile = provider.GetRequiredService(); + + return async (name, dto, apiName, cancellationToken) => + await writePolicyFile(name, dto, apiName, cancellationToken); } private static void ConfigureWriteApiPolicyFile(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureManagementServiceDirectory(services); + + services.TryAddSingleton(WriteApiPolicyFile); + } + + private static WriteApiPolicyFile WriteApiPolicyFile(IServiceProvider provider) + { + var serviceDirectory = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = Common.GetLogger(loggerFactory); + + return async (name, dto, apiName, cancellationToken) => + { + var policyFile = ApiPolicyFile.From(name, apiName, serviceDirectory); + + logger.LogInformation("Writing API policy file {PolicyFile}", policyFile); + var policy = dto.Properties.Value ?? string.Empty; + await policyFile.WritePolicy(policy, cancellationToken); + }; } } diff --git a/tools/code/extractor/ApiTag.cs b/tools/code/extractor/ApiTag.cs index e44b4ef2..ffa37070 100644 --- a/tools/code/extractor/ApiTag.cs +++ b/tools/code/extractor/ApiTag.cs @@ -3,7 +3,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -11,76 +13,90 @@ namespace extractor; internal delegate ValueTask ExtractApiTags(ApiName apiName, CancellationToken cancellationToken); -file delegate IAsyncEnumerable<(TagName Name, ApiTagDto Dto)> ListApiTags(ApiName apiName, CancellationToken cancellationToken); +internal delegate IAsyncEnumerable<(TagName Name, ApiTagDto Dto)> ListApiTags(ApiName apiName, CancellationToken cancellationToken); -file delegate ValueTask WriteApiTagArtifacts(TagName name, ApiTagDto dto, ApiName apiName, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiTagArtifacts(TagName name, ApiTagDto dto, ApiName apiName, CancellationToken cancellationToken); -file delegate ValueTask WriteApiTagInformationFile(TagName name, ApiTagDto dto, ApiName apiName, CancellationToken cancellationToken); +internal delegate ValueTask WriteApiTagInformationFile(TagName name, ApiTagDto dto, ApiName apiName, CancellationToken cancellationToken); -file sealed class ExtractApiTagsHandler(ListApiTags list, WriteApiTagArtifacts writeArtifacts) +internal static class ApiTagServices { - public async ValueTask Handle(ApiName apiName, CancellationToken cancellationToken) => - await list(apiName, cancellationToken) - .IterParallel(async apitag => await writeArtifacts(apitag.Name, apitag.Dto, apiName, cancellationToken), - cancellationToken); -} + public static void ConfigureExtractApiTags(IServiceCollection services) + { + ConfigureListApiTags(services); + TagServices.ConfigureShouldExtractTag(services); + ConfigureWriteApiTagArtifacts(services); -file sealed class ListApiTagsHandler(ManagementServiceUri serviceUri, HttpPipeline pipeline) -{ - public IAsyncEnumerable<(TagName, ApiTagDto)> Handle(ApiName apiName, CancellationToken cancellationToken) => - ApiTagsUri.From(apiName, serviceUri).List(pipeline, cancellationToken); -} + services.TryAddSingleton(ExtractApiTags); + } -file sealed class WriteApiTagArtifactsHandler(WriteApiTagInformationFile writeTagFile) -{ - public async ValueTask Handle(TagName name, ApiTagDto dto, ApiName apiName, CancellationToken cancellationToken) + private static ExtractApiTags ExtractApiTags(IServiceProvider provider) { - await writeTagFile(name, dto, apiName, cancellationToken); + var list = provider.GetRequiredService(); + var shouldExtractTag = provider.GetRequiredService(); + var writeArtifacts = provider.GetRequiredService(); + + return async (apiName, cancellationToken) => + await list(apiName, cancellationToken) + .Where(tag => shouldExtractTag(tag.Name)) + .IterParallel(async tag => await writeArtifacts(tag.Name, tag.Dto, apiName, cancellationToken), + cancellationToken); } -} - -file sealed class WriteApiTagInformationFileHandler(ILoggerFactory loggerFactory, ManagementServiceDirectory serviceDirectory) -{ - private readonly ILogger logger = Common.GetLogger(loggerFactory); - public async ValueTask Handle(TagName name, ApiTagDto dto, ApiName apiName, CancellationToken cancellationToken) + private static void ConfigureListApiTags(IServiceCollection services) { - var informationFile = ApiTagInformationFile.From(name, apiName, serviceDirectory); + CommonServices.ConfigureManagementServiceUri(services); + CommonServices.ConfigureHttpPipeline(services); - logger.LogInformation("Writing API tag information file {ApiTagInformationFile}...", informationFile); - await informationFile.WriteDto(dto, cancellationToken); + services.TryAddSingleton(ListApiTags); } -} -internal static class ApiTagServices -{ - public static void ConfigureExtractApiTags(IServiceCollection services) + private static ListApiTags ListApiTags(IServiceProvider provider) { - ConfigureListApiTags(services); - ConfigureWriteApiTagArtifacts(services); + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + return (apiName, cancellationToken) => + ApiTagsUri.From(apiName, serviceUri) + .List(pipeline, cancellationToken); } - private static void ConfigureListApiTags(IServiceCollection services) + private static void ConfigureWriteApiTagArtifacts(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + ConfigureWriteApiTagInformationFile(services); + + services.TryAddSingleton(WriteApiTagArtifacts); } - private static void ConfigureWriteApiTagArtifacts(IServiceCollection services) + private static WriteApiTagArtifacts WriteApiTagArtifacts(IServiceProvider provider) { - ConfigureWriteApiTagInformationFile(services); + var writeInformationFile = provider.GetRequiredService(); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + return async (name, dto, apiName, cancellationToken) => + await writeInformationFile(name, dto, apiName, cancellationToken); } private static void ConfigureWriteApiTagInformationFile(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureManagementServiceDirectory(services); + + services.TryAddSingleton(WriteApiTagInformationFile); + } + + private static WriteApiTagInformationFile WriteApiTagInformationFile(IServiceProvider provider) + { + var serviceDirectory = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = Common.GetLogger(loggerFactory); + + return async (name, dto, apiName, cancellationToken) => + { + var informationFile = ApiTagInformationFile.From(name, apiName, serviceDirectory); + + logger.LogInformation("Writing API tag information file {InformationFile}", informationFile); + await informationFile.WriteDto(dto, cancellationToken); + }; } } diff --git a/tools/code/extractor/App.cs b/tools/code/extractor/App.cs index 7a393deb..8e5b45cf 100644 --- a/tools/code/extractor/App.cs +++ b/tools/code/extractor/App.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using System; using System.Threading; using System.Threading.Tasks; @@ -8,45 +9,6 @@ namespace extractor; internal delegate ValueTask RunExtractor(CancellationToken cancellationToken); -file sealed class RunExtractorHandler(ILoggerFactory loggerFactory, - ExtractNamedValues extractNamedValues, - ExtractTags extractTags, - ExtractGateways extractGateways, - ExtractVersionSets extractVersionSets, - ExtractBackends extractBackends, - ExtractLoggers extractLoggers, - ExtractDiagnostics extractDiagnostics, - ExtractPolicyFragments extractPolicyFragments, - ExtractServicePolicies extractServicePolicies, - ExtractProducts extractProducts, - ExtractGroups extractGroups, - ExtractSubscriptions extractSubscriptions, - ExtractApis extractApis) -{ - private readonly ILogger logger = loggerFactory.CreateLogger("RunExtractor"); - - public async ValueTask Handle(CancellationToken cancellationToken) - { - logger.LogInformation("Running extractor..."); - - await extractNamedValues(cancellationToken); - await extractTags(cancellationToken); - await extractGateways(cancellationToken); - await extractVersionSets(cancellationToken); - await extractBackends(cancellationToken); - await extractLoggers(cancellationToken); - await extractDiagnostics(cancellationToken); - await extractPolicyFragments(cancellationToken); - await extractServicePolicies(cancellationToken); - await extractProducts(cancellationToken); - await extractGroups(cancellationToken); - await extractSubscriptions(cancellationToken); - await extractApis(cancellationToken); - - logger.LogInformation("Extractor completed."); - } -} - internal static class AppServices { public static void ConfigureRunExtractor(IServiceCollection services) @@ -65,7 +27,47 @@ public static void ConfigureRunExtractor(IServiceCollection services) SubscriptionServices.ConfigureExtractSubscriptions(services); ApiServices.ConfigureExtractApis(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(RunExtractor); + } + + private static RunExtractor RunExtractor(IServiceProvider provider) + { + var extractNamedValues = provider.GetRequiredService(); + var extractTags = provider.GetRequiredService(); + var extractGateways = provider.GetRequiredService(); + var extractVersionSets = provider.GetRequiredService(); + var extractBackends = provider.GetRequiredService(); + var extractLoggers = provider.GetRequiredService(); + var extractDiagnostics = provider.GetRequiredService(); + var extractPolicyFragments = provider.GetRequiredService(); + var extractServicePolicies = provider.GetRequiredService(); + var extractProducts = provider.GetRequiredService(); + var extractGroups = provider.GetRequiredService(); + var extractSubscriptions = provider.GetRequiredService(); + var extractApis = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = loggerFactory.CreateLogger("Extractor"); + + return async cancellationToken => + { + logger.LogInformation("Running extractor..."); + + await extractNamedValues(cancellationToken); + await extractTags(cancellationToken); + await extractGateways(cancellationToken); + await extractVersionSets(cancellationToken); + await extractBackends(cancellationToken); + await extractLoggers(cancellationToken); + await extractDiagnostics(cancellationToken); + await extractPolicyFragments(cancellationToken); + await extractServicePolicies(cancellationToken); + await extractProducts(cancellationToken); + await extractGroups(cancellationToken); + await extractSubscriptions(cancellationToken); + await extractApis(cancellationToken); + + logger.LogInformation("Extractor completed."); + }; } } \ No newline at end of file diff --git a/tools/code/extractor/Backend.cs b/tools/code/extractor/Backend.cs index 2152a9d2..43c376c8 100644 --- a/tools/code/extractor/Backend.cs +++ b/tools/code/extractor/Backend.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -13,95 +14,108 @@ namespace extractor; internal delegate ValueTask ExtractBackends(CancellationToken cancellationToken); -file delegate IAsyncEnumerable<(BackendName Name, BackendDto Dto)> ListBackends(CancellationToken cancellationToken); +internal delegate IAsyncEnumerable<(BackendName Name, BackendDto Dto)> ListBackends(CancellationToken cancellationToken); -file delegate bool ShouldExtractBackend(BackendName name); +internal delegate bool ShouldExtractBackend(BackendName name); -file delegate ValueTask WriteBackendArtifacts(BackendName name, BackendDto dto, CancellationToken cancellationToken); +internal delegate ValueTask WriteBackendArtifacts(BackendName name, BackendDto dto, CancellationToken cancellationToken); -file delegate ValueTask WriteBackendInformationFile(BackendName name, BackendDto dto, CancellationToken cancellationToken); +internal delegate ValueTask WriteBackendInformationFile(BackendName name, BackendDto dto, CancellationToken cancellationToken); -file sealed class ExtractBackendsHandler(ListBackends list, ShouldExtractBackend shouldExtract, WriteBackendArtifacts writeArtifacts) -{ - public async ValueTask Handle(CancellationToken cancellationToken) => - await list(cancellationToken) - .Where(backend => shouldExtract(backend.Name)) - .IterParallel(async backend => await writeArtifacts(backend.Name, backend.Dto, cancellationToken), - cancellationToken); -} - -file sealed class ListBackendsHandler(ManagementServiceUri serviceUri, HttpPipeline pipeline) -{ - public IAsyncEnumerable<(BackendName, BackendDto)> Handle(CancellationToken cancellationToken) => - BackendsUri.From(serviceUri).List(pipeline, cancellationToken); -} - -file sealed class ShouldExtractBackendHandler(ShouldExtractFactory shouldExtractFactory) +internal static class BackendServices { - public bool Handle(BackendName name) + public static void ConfigureExtractBackends(IServiceCollection services) { - var shouldExtract = shouldExtractFactory.Create(); - return shouldExtract(name); + ConfigureListBackends(services); + ConfigureShouldExtractBackend(services); + ConfigureWriteBackendArtifacts(services); + + services.TryAddSingleton(ExtractBackends); } -} -file sealed class WriteBackendArtifactsHandler(WriteBackendInformationFile writeInformationFile) -{ - public async ValueTask Handle(BackendName name, BackendDto dto, CancellationToken cancellationToken) + private static ExtractBackends ExtractBackends(IServiceProvider provider) { - await writeInformationFile(name, dto, cancellationToken); + var list = provider.GetRequiredService(); + var shouldExtract = provider.GetRequiredService(); + var writeArtifacts = provider.GetRequiredService(); + + return async cancellationToken => + await list(cancellationToken) + .Where(backend => shouldExtract(backend.Name)) + .IterParallel(async backend => await writeArtifacts(backend.Name, backend.Dto, cancellationToken), + cancellationToken); } -} -file sealed class WriteBackendInformationFileHandler(ILoggerFactory loggerFactory, ManagementServiceDirectory serviceDirectory) -{ - private readonly ILogger logger = Common.GetLogger(loggerFactory); - - public async ValueTask Handle(BackendName name, BackendDto dto, CancellationToken cancellationToken) + private static void ConfigureListBackends(IServiceCollection services) { - var informationFile = BackendInformationFile.From(name, serviceDirectory); + CommonServices.ConfigureManagementServiceUri(services); + CommonServices.ConfigureHttpPipeline(services); - logger.LogInformation("Writing backend information file {InformationFile}", informationFile); - await informationFile.WriteDto(dto, cancellationToken); + services.TryAddSingleton(ListBackends); } -} -internal static class BackendServices -{ - public static void ConfigureExtractBackends(IServiceCollection services) + private static ListBackends ListBackends(IServiceProvider provider) { - ConfigureListBackends(services); - ConfigureShouldExtractBackend(services); - ConfigureWriteBackendArtifacts(services); + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + return cancellationToken => + BackendsUri.From(serviceUri) + .List(pipeline, cancellationToken); } - private static void ConfigureListBackends(IServiceCollection services) + private static void ConfigureShouldExtractBackend(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureShouldExtractFactory(services); + + services.TryAddSingleton(ShouldExtractBackend); } - private static void ConfigureShouldExtractBackend(IServiceCollection services) + private static ShouldExtractBackend ShouldExtractBackend(IServiceProvider provider) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + var shouldExtractFactory = provider.GetRequiredService(); + + var shouldExtract = shouldExtractFactory.Create(); + + return name => shouldExtract(name); } private static void ConfigureWriteBackendArtifacts(IServiceCollection services) { ConfigureWriteBackendInformationFile(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(WriteBackendArtifacts); + } + + private static WriteBackendArtifacts WriteBackendArtifacts(IServiceProvider provider) + { + var writeInformationFile = provider.GetRequiredService(); + + return async (name, dto, cancellationToken) => + await writeInformationFile(name, dto, cancellationToken); } private static void ConfigureWriteBackendInformationFile(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureManagementServiceDirectory(services); + + services.TryAddSingleton(WriteBackendInformationFile); + } + + private static WriteBackendInformationFile WriteBackendInformationFile(IServiceProvider provider) + { + var serviceDirectory = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = Common.GetLogger(loggerFactory); + + return async (name, dto, cancellationToken) => + { + var informationFile = BackendInformationFile.From(name, serviceDirectory); + + logger.LogInformation("Writing backend information file {BackendInformationFile}...", informationFile); + await informationFile.WriteDto(dto, cancellationToken); + }; } } diff --git a/tools/code/extractor/Common.cs b/tools/code/extractor/Common.cs index 9fbc3219..d35b1a72 100644 --- a/tools/code/extractor/Common.cs +++ b/tools/code/extractor/Common.cs @@ -8,6 +8,7 @@ using LanguageExt.UnsafeValueAccess; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.JsonWebTokens; using System; @@ -76,6 +77,11 @@ private static DefaultAzureCredential GetDefaultAzureCredential(Uri azureAuthori AuthorityHost = azureAuthorityHost }); + public static void ConfigureHttpPipeline(IServiceCollection services) + { + services.TryAddSingleton(GetHttpPipeline); + } + private static HttpPipeline GetHttpPipeline(IServiceProvider provider) { var clientOptions = ClientOptions.Default; @@ -104,6 +110,11 @@ private static ManagementServiceName GetManagementServiceName(IServiceProvider p return ManagementServiceName.From(name); } + public static void ConfigureManagementServiceUri(IServiceCollection services) + { + services.TryAddSingleton(GetManagementServiceUri); + } + private static ManagementServiceUri GetManagementServiceUri(IServiceProvider provider) { var azureEnvironment = provider.GetRequiredService(); @@ -126,6 +137,11 @@ private static ManagementServiceUri GetManagementServiceUri(IServiceProvider pro return ManagementServiceUri.From(uri); } + public static void ConfigureManagementServiceDirectory(IServiceCollection services) + { + services.TryAddSingleton(GetManagementServiceDirectory); + } + private static ManagementServiceDirectory GetManagementServiceDirectory(IServiceProvider provider) { var configuration = provider.GetRequiredService(); @@ -135,6 +151,26 @@ private static ManagementServiceDirectory GetManagementServiceDirectory(IService return ManagementServiceDirectory.From(directory); } + public static void ConfigureShouldExtractFactory(IServiceCollection services) + { + ConfigureConfigurationJson(services); + + services.TryAddSingleton(GetShouldExtractFactory); + } + + private static ShouldExtractFactory GetShouldExtractFactory(IServiceProvider provider) + { + var configurationJson = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + return new ShouldExtractFactory(configurationJson, loggerFactory); + } + + private static void ConfigureConfigurationJson(IServiceCollection services) + { + services.TryAddSingleton(GetConfigurationJson); + } + private static ConfigurationJson GetConfigurationJson(IServiceProvider provider) { var configuration = provider.GetRequiredService(); diff --git a/tools/code/extractor/Diagnostic.cs b/tools/code/extractor/Diagnostic.cs index 9d82184c..5abb0eb1 100644 --- a/tools/code/extractor/Diagnostic.cs +++ b/tools/code/extractor/Diagnostic.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -13,99 +14,114 @@ namespace extractor; internal delegate ValueTask ExtractDiagnostics(CancellationToken cancellationToken); -file delegate IAsyncEnumerable<(DiagnosticName Name, DiagnosticDto Dto)> ListDiagnostics(CancellationToken cancellationToken); +internal delegate IAsyncEnumerable<(DiagnosticName Name, DiagnosticDto Dto)> ListDiagnostics(CancellationToken cancellationToken); -file delegate bool ShouldExtractDiagnostic(DiagnosticName name, DiagnosticDto dto); +internal delegate bool ShouldExtractDiagnostic(DiagnosticName name, DiagnosticDto dto); -file delegate ValueTask WriteDiagnosticArtifacts(DiagnosticName name, DiagnosticDto dto, CancellationToken cancellationToken); +internal delegate ValueTask WriteDiagnosticArtifacts(DiagnosticName name, DiagnosticDto dto, CancellationToken cancellationToken); -file delegate ValueTask WriteDiagnosticInformationFile(DiagnosticName name, DiagnosticDto dto, CancellationToken cancellationToken); +internal delegate ValueTask WriteDiagnosticInformationFile(DiagnosticName name, DiagnosticDto dto, CancellationToken cancellationToken); -file sealed class ExtractDiagnosticsHandler(ListDiagnostics list, ShouldExtractDiagnostic shouldExtract, WriteDiagnosticArtifacts writeArtifacts) -{ - public async ValueTask Handle(CancellationToken cancellationToken) => - await list(cancellationToken) - .Where(diagnostic => shouldExtract(diagnostic.Name, diagnostic.Dto)) - .IterParallel(async diagnostic => await writeArtifacts(diagnostic.Name, diagnostic.Dto, cancellationToken), - cancellationToken); -} - -file sealed class ListDiagnosticsHandler(ManagementServiceUri serviceUri, HttpPipeline pipeline) +internal static class DiagnosticServices { - public IAsyncEnumerable<(DiagnosticName, DiagnosticDto)> Handle(CancellationToken cancellationToken) => - DiagnosticsUri.From(serviceUri).List(pipeline, cancellationToken); -} + public static void ConfigureExtractDiagnostics(IServiceCollection services) + { + ConfigureListDiagnostics(services); + ConfigureShouldExtractDiagnostic(services); + ConfigureWriteDiagnosticArtifacts(services); -file sealed class ShouldExtractDiagnosticHandler(ShouldExtractFactory shouldExtractFactory, ShouldExtractLogger shouldExtractLogger) -{ - public bool Handle(DiagnosticName name, DiagnosticDto dto) => - // Check name from configuration override - shouldExtractFactory.Create().Invoke(name) - // Don't extract diagnostic if its logger should not be extracted - && DiagnosticModule.TryGetLoggerName(dto) - .Map(shouldExtractLogger.Invoke) - .IfNone(true); -} + services.TryAddSingleton(ExtractDiagnostics); + } -file sealed class WriteDiagnosticArtifactsHandler(WriteDiagnosticInformationFile writeInformationFile) -{ - public async ValueTask Handle(DiagnosticName name, DiagnosticDto dto, CancellationToken cancellationToken) + private static ExtractDiagnostics ExtractDiagnostics(IServiceProvider provider) { - await writeInformationFile(name, dto, cancellationToken); + var list = provider.GetRequiredService(); + var shouldExtract = provider.GetRequiredService(); + var writeArtifacts = provider.GetRequiredService(); + + return async cancellationToken => + await list(cancellationToken) + .Where(diagnostic => shouldExtract(diagnostic.Name, diagnostic.Dto)) + .IterParallel(async diagnostic => await writeArtifacts(diagnostic.Name, diagnostic.Dto, cancellationToken), + cancellationToken); } -} - -file sealed class WriteDiagnosticInformationFileHandler(ILoggerFactory loggerFactory, ManagementServiceDirectory serviceDirectory) -{ - private readonly ILogger logger = Common.GetLogger(loggerFactory); - public async ValueTask Handle(DiagnosticName name, DiagnosticDto dto, CancellationToken cancellationToken) + private static void ConfigureListDiagnostics(IServiceCollection services) { - var informationFile = DiagnosticInformationFile.From(name, serviceDirectory); + CommonServices.ConfigureManagementServiceUri(services); + CommonServices.ConfigureHttpPipeline(services); - logger.LogInformation("Writing diagnostic information file {InformationFile}", informationFile); - await informationFile.WriteDto(dto, cancellationToken); + services.TryAddSingleton(ListDiagnostics); } -} -internal static class DiagnosticServices -{ - public static void ConfigureExtractDiagnostics(IServiceCollection services) + private static ListDiagnostics ListDiagnostics(IServiceProvider provider) { - ConfigureListDiagnostics(services); - ConfigureShouldExtractDiagnostic(services); - ConfigureWriteDiagnosticArtifacts(services); + var serviceUri = provider.GetRequiredService(); + var pipeline = provider.GetRequiredService(); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + return cancellationToken => + DiagnosticsUri.From(serviceUri) + .List(pipeline, cancellationToken); } - private static void ConfigureListDiagnostics(IServiceCollection services) + private static void ConfigureShouldExtractDiagnostic(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureShouldExtractFactory(services); + LoggerServices.ConfigureShouldExtractLogger(services); + + services.TryAddSingleton(ShouldExtractDiagnostic); } - private static void ConfigureShouldExtractDiagnostic(IServiceCollection services) + private static ShouldExtractDiagnostic ShouldExtractDiagnostic(IServiceProvider provider) { - LoggerServices.ConfigureShouldExtractLogger(services); + var shouldExtractFactory = provider.GetRequiredService(); + var shouldExtractLogger = provider.GetRequiredService(); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + var shouldExtractDiagnosticName = shouldExtractFactory.Create(); + + return (name, dto) => + shouldExtractDiagnosticName(name) + && DiagnosticModule.TryGetLoggerName(dto) + .Map(shouldExtractLogger.Invoke) + .IfNone(true); } private static void ConfigureWriteDiagnosticArtifacts(IServiceCollection services) { ConfigureWriteDiagnosticInformationFile(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(WriteDiagnosticArtifacts); + } + + private static WriteDiagnosticArtifacts WriteDiagnosticArtifacts(IServiceProvider provider) + { + var writeInformationFile = provider.GetRequiredService(); + + return async (name, dto, cancellationToken) => + await writeInformationFile(name, dto, cancellationToken); } private static void ConfigureWriteDiagnosticInformationFile(IServiceCollection services) { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + CommonServices.ConfigureManagementServiceDirectory(services); + + services.TryAddSingleton(WriteDiagnosticInformationFile); + } + + private static WriteDiagnosticInformationFile WriteDiagnosticInformationFile(IServiceProvider provider) + { + var serviceDirectory = provider.GetRequiredService(); + var loggerFactory = provider.GetRequiredService(); + + var logger = Common.GetLogger(loggerFactory); + + return async (name, dto, cancellationToken) => + { + var informationFile = DiagnosticInformationFile.From(name, serviceDirectory); + + logger.LogInformation("Writing diagnostic information file {DiagnosticInformationFile}...", informationFile); + await informationFile.WriteDto(dto, cancellationToken); + }; } } diff --git a/tools/code/extractor/ShouldExtractFactory.cs b/tools/code/extractor/ShouldExtractFactory.cs index 8d3b0b5a..d407b97e 100644 --- a/tools/code/extractor/ShouldExtractFactory.cs +++ b/tools/code/extractor/ShouldExtractFactory.cs @@ -12,21 +12,12 @@ namespace extractor; public sealed class ShouldExtractFactory(ConfigurationJson configurationJson, ILoggerFactory loggerFactory) { - private readonly FrozenDictionary> configurationNames = GetConfigurationNames(configurationJson); - private static readonly FrozenDictionary configurationSectionNames = GetConfigurationSectionNames(); + private static readonly FrozenDictionary typeSectionNames = GetTypeSectionNames(); + private static readonly FrozenDictionary sectionNameTypes = GetSectionNameTypes(typeSectionNames); + private readonly FrozenDictionary> resourcesToExtract = GetResourcesToExtract(configurationJson, sectionNameTypes); private readonly ILogger logger = loggerFactory.CreateLogger(); - private static FrozenDictionary> GetConfigurationNames(ConfigurationJson configurationJson) => - configurationJson.Value - // Get configuration sections that are JSON arrays - .ChooseValue(node => node.TryAsJsonArray()) - // Map each JSON array to a set of strings - .MapValue(jsonArray => jsonArray.Choose(node => node.TryAsString()) - .Where(value => string.IsNullOrWhiteSpace(value) is false) - .ToFrozenSet(StringComparer.OrdinalIgnoreCase)) - .ToFrozenDictionary(StringComparer.OrdinalIgnoreCase); - - private static FrozenDictionary GetConfigurationSectionNames() => + private static FrozenDictionary GetTypeSectionNames() => new Dictionary { [typeof(NamedValueName)] = "namedValueNames", @@ -44,16 +35,31 @@ private static FrozenDictionary GetConfigurationSectionNames() => } .ToFrozenDictionary(); + private static FrozenDictionary GetSectionNameTypes(FrozenDictionary typeSectionNames) => + typeSectionNames.ToFrozenDictionary(kvp => kvp.Value, kvp => kvp.Key, StringComparer.OrdinalIgnoreCase); + + private static FrozenDictionary> GetResourcesToExtract(ConfigurationJson configurationJson, FrozenDictionary sectionNameTypes) => + configurationJson.Value + // Get configuration sections that are JSON arrays + .ChooseValue(node => node.TryAsJsonArray()) + // Map each JSON array to a set of strings + .MapValue(jsonArray => jsonArray.Choose(node => node.TryAsString()) + .Where(value => string.IsNullOrWhiteSpace(value) is false) + .ToFrozenSet(StringComparer.OrdinalIgnoreCase)) + // Map each configuration section to a resource name type + .ChooseKey(sectionNameTypes.Find) + .ToFrozenDictionary(); + public static string GetConfigurationSectionName() => - configurationSectionNames.Find(typeof(T)) - .IfNone(() => throw new InvalidOperationException($"Resource type {typeof(T).Name} is not supported.")); + typeSectionNames.Find(typeof(T)) + .IfNone(() => throw new InvalidOperationException($"Resource type {typeof(T).Name} is not supported.")); public static string GetNameToFind(T name) where T : ResourceName => - configurationSectionNames.ContainsKey(typeof(T)) + typeSectionNames.ContainsKey(typeof(T)) ? name switch { - ApiName apiName => ApiName.GetRootName(apiName).ToString(), - _ => name.ToString()! + ApiName apiName => ApiName.GetRootName(apiName).Value, + _ => name.Value } : throw new InvalidOperationException($"Resource type {typeof(T).Name} is not supported."); @@ -61,10 +67,9 @@ public static string GetNameToFind(T name) where T : ResourceName => private bool ShouldExtract(TName name) where TName : ResourceName { - var sectionName = GetConfigurationSectionName(); var nameToFind = GetNameToFind(name); - var shouldExtract = configurationNames.Find(sectionName) + var shouldExtract = resourcesToExtract.Find(typeof(TName)) .Map(set => set.Contains(nameToFind)) .IfNone(true); diff --git a/tools/code/extractor/Tag.cs b/tools/code/extractor/Tag.cs index c927a882..54719bdd 100644 --- a/tools/code/extractor/Tag.cs +++ b/tools/code/extractor/Tag.cs @@ -15,7 +15,7 @@ namespace extractor; file delegate IAsyncEnumerable<(TagName Name, TagDto Dto)> ListTags(CancellationToken cancellationToken); -file delegate bool ShouldExtractTag(TagName name); +internal delegate bool ShouldExtractTag(TagName name); file delegate ValueTask WriteTagArtifacts(TagName name, TagDto dto, CancellationToken cancellationToken); @@ -84,7 +84,7 @@ private static void ConfigureListTags(IServiceCollection services) services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } - private static void ConfigureShouldExtractTag(IServiceCollection services) + public static void ConfigureShouldExtractTag(IServiceCollection services) { services.TryAddSingleton(); services.TryAddSingleton(provider => provider.GetRequiredService().Handle); diff --git a/tools/code/publisher/publisher.csproj b/tools/code/publisher/publisher.csproj index 407e7dbf..6ff50939 100644 --- a/tools/code/publisher/publisher.csproj +++ b/tools/code/publisher/publisher.csproj @@ -14,7 +14,7 @@ - + From bb4f26629fef53a09be02999b9814e8c079f39f5 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Tue, 25 Jun 2024 21:02:03 -0400 Subject: [PATCH 27/29] Update dev container --- .devcontainer/devcontainer.json | 39 ++++++++++++++++++------------ .devcontainer/postCreateCommand.sh | 9 +++++++ 2 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 .devcontainer/postCreateCommand.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ec4ebec6..85a0175a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,25 +1,32 @@ { - "name": "API Ops", - "image": "mcr.microsoft.com/devcontainers/dotnet:0-7.0", + "name": "C# (.NET)", + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", "features": { - "ghcr.io/devcontainers/features/azure-cli:1": {} + "ghcr.io/devcontainers/features/dotnet:2": { + "version": "8.0", + "additionalVersions": "9.0", + "workloads": "aspire" + }, + "ghcr.io/devcontainers/features/azure-cli:1": { + "version": "latest", + "installBicep": true + }, + "ghcr.io/devcontainers/features/powershell:1": { + "version": "latest" + }, + "ghcr.io/devcontainers/features/git:1": {} }, - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [5000, 5001], - // "portsAttributes": { - // "5001": { - // "protocol": "https" - // } - // } - - "postCreateCommand": "dotnet restore /workspaces/apiops/tools/code/code.sln", - + "postCreateCommand": "./.devcontainer/postCreateCommand.sh", "customizations": { "vscode": { "extensions": [ - "ms-dotnettools.csharp" + "ms-azuretools.vscode-bicep", + "ms-azure-devops.azure-pipelines", + "ms-dotnettools.csdevkit", + "github.copilot", + "github.copilot-chat", + "timonwong.shellcheck" ] } } -} +} \ No newline at end of file diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh new file mode 100644 index 00000000..eeff37ef --- /dev/null +++ b/.devcontainer/postCreateCommand.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Install Aspire +sudo /usr/share/dotnet/dotnet workload update --from-rollback-file "./.devcontainer/rollback.txt" +sudo /usr/share/dotnet/dotnet workload install aspire --from-rollback-file "./.devcontainer/rollback.txt" + +# Install PowerShell modules +pwsh -Command "Install-Module -Name Az -Force + Install-Module -Name Microsoft.Graph -Force" From d93572a0478acba6884415f95d67f298217f1467 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Tue, 25 Jun 2024 21:03:11 -0400 Subject: [PATCH 28/29] Version bumps --- tools/code/common/common.csproj | 10 +++++----- tools/code/publisher/publisher.csproj | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/code/common/common.csproj b/tools/code/common/common.csproj index b43d9bcf..7f946392 100644 --- a/tools/code/common/common.csproj +++ b/tools/code/common/common.csproj @@ -11,7 +11,7 @@ - + @@ -19,13 +19,13 @@ - - + + - - + + diff --git a/tools/code/publisher/publisher.csproj b/tools/code/publisher/publisher.csproj index 6ff50939..54917771 100644 --- a/tools/code/publisher/publisher.csproj +++ b/tools/code/publisher/publisher.csproj @@ -14,7 +14,7 @@ - + From 0e10dad040100fce6e6a3dce2bd41dbb89168a65 Mon Sep 17 00:00:00 2001 From: Guy Fankam Date: Tue, 2 Jul 2024 11:46:43 -0400 Subject: [PATCH 29/29] Prioritize backend processing --- tools/code/publisher/App.cs | 9 ++++--- tools/code/publisher/Backend.cs | 47 +++++++++++++++++++++++++++++---- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/tools/code/publisher/App.cs b/tools/code/publisher/App.cs index 8bb27ede..56250196 100644 --- a/tools/code/publisher/App.cs +++ b/tools/code/publisher/App.cs @@ -15,6 +15,8 @@ namespace publisher; file sealed class RunPublisherHandler(ProcessNamedValuesToPut processNamedValuesToPut, ProcessDeletedNamedValues processDeletedNamedValues, + ProcessBackendsToPut processBackendsToPut, + ProcessDeletedBackends processDeletedBackends, GetPublisherFiles getPublisherFiles, PublishFile publishFile, ILoggerFactory loggerFactory) @@ -24,10 +26,12 @@ file sealed class RunPublisherHandler(ProcessNamedValuesToPut processNamedValues public async ValueTask Handle(CancellationToken cancellationToken) { await processNamedValuesToPut(cancellationToken); + await processBackendsToPut(cancellationToken); await ProcessPublisherFiles(cancellationToken); await processDeletedNamedValues(cancellationToken); + await processDeletedBackends(cancellationToken); logger.LogInformation("Publisher completed."); } @@ -47,7 +51,6 @@ private async ValueTask ProcessPublisherFiles(CancellationToken cancellationToke file sealed class PublishFileHandler(FindTagAction findTagAction, FindGatewayAction findGatewayAction, FindVersionSetAction findVersionSetAction, - FindBackendAction findBackendAction, FindLoggerAction findLoggerAction, FindDiagnosticAction findDiagnosticAction, FindPolicyFragmentAction findPolicyFragmentAction, @@ -75,7 +78,6 @@ private Option FindPublisherAction(FileInfo file) => findTagAction(file) | findGatewayAction(file) | findVersionSetAction(file) - | findBackendAction(file) | findLoggerAction(file) | findDiagnosticAction(file) | findPolicyFragmentAction(file) @@ -100,6 +102,8 @@ public static void ConfigureRunPublisher(IServiceCollection services) { NamedValueServices.ConfigureProcessNamedValuesToPut(services); NamedValueServices.ConfigureProcessDeletedNamedValues(services); + BackendServices.ConfigureProcessBackendsToPut(services); + BackendServices.ConfigureProcessDeletedBackends(services); ConfigurePublishFile(services); services.TryAddSingleton(); @@ -111,7 +115,6 @@ private static void ConfigurePublishFile(IServiceCollection services) TagServices.ConfigureFindTagAction(services); GatewayServices.ConfigureFindGatewayAction(services); VersionSetServices.ConfigureFindVersionSetAction(services); - BackendServices.ConfigureFindBackendAction(services); LoggerServices.ConfigureFindLoggerAction(services); DiagnosticServices.ConfigureFindDiagnosticAction(services); PolicyFragmentServices.ConfigureFindPolicyFragmentAction(services); diff --git a/tools/code/publisher/Backend.cs b/tools/code/publisher/Backend.cs index 00ecea3a..c0dcb259 100644 --- a/tools/code/publisher/Backend.cs +++ b/tools/code/publisher/Backend.cs @@ -9,12 +9,14 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; namespace publisher; -internal delegate Option FindBackendAction(FileInfo file); +internal delegate ValueTask ProcessBackendsToPut(CancellationToken cancellationToken); +internal delegate ValueTask ProcessDeletedBackends(CancellationToken cancellationToken); file delegate Option TryParseBackendName(FileInfo file); @@ -32,6 +34,30 @@ namespace publisher; file delegate ValueTask DeleteBackendFromApim(BackendName name, CancellationToken cancellationToken); +file sealed class ProcessBackendsToPutHandler(GetPublisherFiles getPublisherFiles, + TryParseBackendName tryParseBackendName, + IsBackendNameInSourceControl isNameInSourceControl, + PutBackend putBackend) +{ + public async ValueTask Handle(CancellationToken cancellationToken) => + await getPublisherFiles() + .Choose(tryParseBackendName.Invoke) + .Where(isNameInSourceControl.Invoke) + .IterParallel(putBackend.Invoke, cancellationToken); +} + +file sealed class ProcessDeletedBackendsHandler(GetPublisherFiles getPublisherFiles, + TryParseBackendName tryParseBackendName, + IsBackendNameInSourceControl isNameInSourceControl, + DeleteBackend deleteBackend) +{ + public async ValueTask Handle(CancellationToken cancellationToken) => + await getPublisherFiles() + .Choose(tryParseBackendName.Invoke) + .Where(name => isNameInSourceControl(name) is false) + .IterParallel(deleteBackend.Invoke, cancellationToken); +} + file sealed class FindBackendActionHandler(TryParseBackendName tryParseName, ProcessBackend processBackend) { public Option Handle(FileInfo file) => @@ -189,13 +215,14 @@ public async ValueTask Handle(BackendName name, CancellationToken cancellationTo internal static class BackendServices { - public static void ConfigureFindBackendAction(IServiceCollection services) + public static void ConfigureProcessBackendsToPut(IServiceCollection services) { ConfigureTryParseBackendName(services); - ConfigureProcessBackend(services); + ConfigureIsBackendNameInSourceControl(services); + ConfigurePutBackend(services); - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } private static void ConfigureTryParseBackendName(IServiceCollection services) @@ -241,6 +268,16 @@ private static void ConfigurePutBackendInApim(IServiceCollection services) services.TryAddSingleton(provider => provider.GetRequiredService().Handle); } + public static void ConfigureProcessDeletedBackends(IServiceCollection services) + { + ConfigureTryParseBackendName(services); + ConfigureIsBackendNameInSourceControl(services); + ConfigureDeleteBackend(services); + + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService().Handle); + } + private static void ConfigureDeleteBackend(IServiceCollection services) { ConfigureDeleteBackendFromApim(services);