diff --git a/eng/pipelines/daily-verification.yml b/eng/pipelines/daily-verification.yml new file mode 100644 index 0000000000..96dcea1290 --- /dev/null +++ b/eng/pipelines/daily-verification.yml @@ -0,0 +1,54 @@ + +jobs: + - job: + displayName: Daily vcpkg PR + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + variables: + # The branch in the azure-sdk/vcpkg repo that is updated daily with the + # latest set of ports + TargetPrBranch: release/azure-sdk-for-cpp-daily-verification + + steps: + - checkout: self + + - template: /eng/pipelines/templates/steps/generate-daily-branch-name.yml + + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + Workspace: $(Pipeline.Workspace) + RepoOwner: azure-sdk + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + parameters: + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + + - pwsh: | + Write-Host "git checkout $(DailyBranchName)" + git checkout $(DailyBranchName) + if ($LASTEXITCODE -ne 0) { + Write-Error "Cannot check out $(DailyBranchName) LASTEXITCODE=$LASTEXITCODE" + } + + Write-Host "##vso[task.setvariable variable=HasChanges]$true" + workingDirectory: $(Pipeline.Workspace)/vcpkg + displayName: Update vcpkg Daily Branch + + # This step will skip creating the PR if the PR already exists + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + # Force push so this recreates the $(TargetPrBranch) + PushArgs: --force + BaseBranchName: $(DefaultBranch) + PRBranchName: $(TargetPrBranch) + RepoOwner: Microsoft + RepoName: vcpkg + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + CommitMsg: Update with changes from $(DailyBranchName) + PRTitle: "[DO NOT MERGE] Daily Azure SDK for C++ CI validation" + PRBody: This is a long-lived draft PR should not be merged. Its branch is be updated periodically to validate the Azure SDK for C++ against the vcpkg CI. + ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts + OpenAsDraft: true + SkipCheckingForChanges: true diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index efb67f2915..fb83fd5fdb 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -1,6 +1,7 @@ parameters: Artifacts: [] ArtifactName: 'not-specified' + ServiceDirectory: not-specified TestPipeline: false stages: @@ -9,7 +10,7 @@ stages: - stage: Release_${{ replace(artifact.Name, '-', '_') }} displayName: 'Release ${{artifact.name}}' dependsOn: Build - condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp')) + condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp'), ne(variables['Skip.AllRelease'], 'true')) jobs: - deployment: TagRepository @@ -122,52 +123,18 @@ stages: Write-Host "##vso[task.setvariable variable=PrTitle]$title" displayName: Set PR title - - task: Powershell@2 - inputs: - filePath: eng/scripts/Initialize-VcpkgRelease.ps1 - arguments: >- - -GitHubRepo "$(Build.Repository.Name)" - -PackageSpecPath $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json - -SourceDirectory $(Pipeline.Workspace)/packages/${{artifact.Name}}/vcpkg - -Verbose - pwsh: true - displayName: Initialize vcpkg release - # There are potential race conditions if this script runs # in parallel against the same branch name. Release only # one package at a time. - - pwsh: | - Write-Host "git clone https://github.com/azure-sdk/vcpkg $(Pipeline.Workspace)/vcpkg" - git clone https://github.com/azure-sdk/vcpkg $(Pipeline.Workspace)/vcpkg - if ($LASTEXITCODE -ne 0) { - Write-Error "Unable to check out vcpkg fork repo" - exit $LASTEXITCODE - } - Write-Host "##vso[task.setvariable variable=VcpkgWorkspace]$(Pipeline.Workspace)/vcpkg" - displayName: Clone vcpkg from upstream + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: azure-sdk + PRBranchName: $(PrBranchName) - # Check out the PR branch if it's already in remote. - # Ignore failures. - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/$(PrBranchName)" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout Previous PRBranch if it exist. - workingDirectory: $(VcpkgWorkspace) - - - task: Powershell@2 - inputs: - pwsh: true - targetType: filePath - filePath: eng/scripts/Update-VcpkgPort.ps1 - arguments: >- - -ReleaseArtifactSourceDirectory "$(Pipeline.Workspace)/packages/${{ artifact.Name }}" - -PortDestinationDirectory 'ports/${{ artifact.VcpkgPortName }}' - -VcpkgPortName '${{ artifact.VcpkgPortName }}' - -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' - workingDirectory: $(VcpkgWorkspace) - displayName: Update vcpkg port + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + parameters: + ArtifactName: ${{ artifact.Name }} + VcpkgPortName: ${{ artifact.VcpkgPortName }} # Set $(HasChanges) to $true so that # create-pull-request.yml completes the push and PR @@ -214,4 +181,54 @@ stages: PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) CommitMsg: "Increment package version after release of ${{ artifact.name }}" PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' \ No newline at end of file + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + + - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - stage: Integration + dependsOn: Build + condition: and(succeeded(), or(eq(variables['PublishDailyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule'))) + jobs: + - job: PublishDailyVcpkg + displayName: Publish to vcpkg daily branch + + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + steps: + - checkout: self + - download: current + artifact: packages + + - template: /eng/pipelines/templates/steps/generate-daily-branch-name.yml + + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: azure-sdk + PRBranchName: $(DailyBranchName) + + # Placing artifacts in the same branch in sequence reduces the + # potential for race conditions. If this job does fail to push + # because another job has already pushed to the same branch it + # can be re-run. + - ${{ each artifact in parameters.Artifacts }}: + # Only run vcpkg publish if the artifact has a "VcpkgPortName" + # property. Absence of VcpkgPortName implies that the artifact + # does not ship to vcpkg. + - ${{ if ne(artifact.VcpkgPortName, '') }}: + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + parameters: + ArtifactName: ${{ artifact.Name }} + VcpkgPortName: ${{ artifact.VcpkgPortName }} + DisplayNameExtension: ${{ artifact.Name }} + DailyRelease: true + DailyReleaseRef: $(Build.SourceVersion) + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DailyBranchName) + CommitMsg: Daily vcpkg ports for ${{ parameters.ServiceDirectory }} + TargetRepoOwner: azure-sdk + TargetRepoName: vcpkg + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts diff --git a/eng/pipelines/templates/steps/generate-daily-branch-name.yml b/eng/pipelines/templates/steps/generate-daily-branch-name.yml new file mode 100644 index 0000000000..18762e5e52 --- /dev/null +++ b/eng/pipelines/templates/steps/generate-daily-branch-name.yml @@ -0,0 +1,9 @@ +steps: + - pwsh: | + $branchName = $env:DAILYBRANCHOVERRIDE + if (!$branchName) { + $formattedDate = Get-Date -Format 'yyyy-MM-dd' + $branchName = "release/daily-$formattedDate" + } + Write-Host "##vso[task.setvariable variable=DailyBranchName]$branchName" + displayName: Set branch name in fork repo diff --git a/eng/pipelines/templates/steps/vcpkg-clone.yml b/eng/pipelines/templates/steps/vcpkg-clone.yml new file mode 100644 index 0000000000..94640bb22d --- /dev/null +++ b/eng/pipelines/templates/steps/vcpkg-clone.yml @@ -0,0 +1,24 @@ +parameters: + Workspace: $(Pipeline.Workspace) + RepoOwner: not-set + RepoName: vcpkg + PRBranchName: not-set + +steps: + - pwsh: | + Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg" + git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg + if ($LASTEXITCODE -ne 0) { + Write-Error "Unable to check out vcpkg fork repo" + exit $LASTEXITCODE + } + displayName: Clone vcpkg from upstream + + # Check out the PR branch if it's already in remote. Ignore failures. + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout Previous PRBranch if it exists + workingDirectory: ${{ parameters.Workspace }}/vcpkg \ No newline at end of file diff --git a/eng/pipelines/templates/steps/vcpkg-publish.yml b/eng/pipelines/templates/steps/vcpkg-publish.yml new file mode 100644 index 0000000000..0493eebc83 --- /dev/null +++ b/eng/pipelines/templates/steps/vcpkg-publish.yml @@ -0,0 +1,35 @@ +parameters: + GitHubRepo: $(Build.Repository.Name) + Workspace: $(Pipeline.Workspace) + ArtifactName: not-set + VcpkgPortName: not-set + DisplayNameExtension: + DailyReleaseRef: + DailyRelease: false + +steps: + - task: Powershell@2 + inputs: + filePath: eng/scripts/Initialize-VcpkgRelease.ps1 + arguments: >- + -GitHubRepo "${{ parameters.GitHubRepo }}" + -PackageSpecPath ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/package-info.json + -SourceDirectory ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/vcpkg + -DailyReleaseRef '${{ parameters.DailyReleaseRef }}' + -Verbose + pwsh: true + displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }} + + - task: Powershell@2 + inputs: + pwsh: true + targetType: filePath + filePath: eng/scripts/Update-VcpkgPort.ps1 + arguments: >- + -ReleaseArtifactSourceDirectory "${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}" + -PortDestinationDirectory 'ports/${{ parameters.VcpkgPortName }}' + -VcpkgPortName '${{ parameters.VcpkgPortName }}' + -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' + -DailyRelease:$${{ parameters.DailyRelease }} + workingDirectory: ${{ parameters.Workspace }}/vcpkg + displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }} \ No newline at end of file diff --git a/eng/scripts/Initialize-VcpkgRelease.ps1 b/eng/scripts/Initialize-VcpkgRelease.ps1 index f10cb582af..c62e54afd9 100644 --- a/eng/scripts/Initialize-VcpkgRelease.ps1 +++ b/eng/scripts/Initialize-VcpkgRelease.ps1 @@ -12,8 +12,12 @@ Location of the relevant package-info.json file .PARAMETER GitHubRepo Name of the GitHub repo (of the form Azure/azure-sdk-for-cpp) -#> +.PARAMETER DailyReleaseRef +If supplied update the portfile.cmake file's REF and SHA512 with values +associated with the given ref. +#> +[CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] @@ -25,7 +29,9 @@ param ( [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] - [string] $GitHubRepo + [string] $GitHubRepo, + + [string] $DailyReleaseRef ) # If there's nothing in the "port" folder to upload set SkipVcpkgUpdate to true @@ -39,6 +45,11 @@ if (!(Get-ChildItem -Path "$SourceDirectory/port/CONTROL")) { $packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json $tarGzUri = "https://github.com/$GitHubRepo/archive/$($packageSpec.packageName).tar.gz" +if ($DailyReleaseRef) { + Write-Verbose "Initializing Daily Release" + $tarGzUri = "https://github.com/$GitHubRepo/archive/$DailyReleaseRef.tar.gz" +} + Write-Host "Downloading tarball to compute hash from $tarGzUri" $localTarGzPath = New-TemporaryFile Invoke-WebRequest -Uri $tarGzUri -OutFile $localTarGzPath @@ -53,7 +64,12 @@ $portfileLocation = "$SourceDirectory/port/portfile.cmake" # recommended in vcpkg documentation # Before: " SHA512 1" # After: " SHA512 f6cf1c16c52" -$newContent = Get-Content -Raw -Path $portfileLocation ` - | ForEach-Object { $_ -replace '(SHA512\s+)1', "`${1}$sha512" } +$portFileContent = Get-Content -Raw -Path $portfileLocation +$newContent = $portFileContent -replace '(SHA512\s+)1', "`${1}$sha512" + +if ($DailyReleaseRef) { + Write-Verbose "Overriding REF with test release ref: $DailyReleaseRef" + $newContent = $newContent -replace '(?m)^(\s+)REF azure.*$', "`${1}REF $DailyReleaseRef" +} $newContent | Set-Content $portfileLocation -NoNewLine diff --git a/eng/scripts/Update-VcpkgPort.ps1 b/eng/scripts/Update-VcpkgPort.ps1 index e0e5c32d5d..35708e7201 100644 --- a/eng/scripts/Update-VcpkgPort.ps1 +++ b/eng/scripts/Update-VcpkgPort.ps1 @@ -11,9 +11,9 @@ param ( [ValidateNotNullOrEmpty()] [string] $VcpkgPortName, - [Parameter(Mandatory = $true)] - [ValidateNotNullOrEmpty()] - [string] $GitCommitParameters + [string] $GitCommitParameters, + + [switch] $DailyRelease ) ."$PSScriptRoot/../common/scripts/common.ps1" @@ -77,8 +77,13 @@ if ($LASTEXITCODE -ne 0) { exit 1 } -Write-Host "./vcpkg.exe x-add-version $VcpkgPortName" -./vcpkg.exe x-add-version $VcpkgPortName +$addVersionAdditionalParameters = '' +if ($DailyRelease) { + $addVersionAdditionalParameters = '--overwrite-version' +} + +Write-Host "./vcpkg.exe x-add-version $VcpkgPortName $addVersionAdditionalParameters" +./vcpkg.exe x-add-version $VcpkgPortName $addVersionAdditionalParameters if ($LASTEXITCODE -ne 0) { Write-Error "Failed to run vcpkg x-add-version $VcpkgPortName" @@ -89,30 +94,37 @@ if ($LASTEXITCODE -ne 0) { Write-Host "git reset HEAD^" git reset HEAD^ -# Grab content needed for commit message and place in a temporary file -$packageVersion = (Get-Content $ReleaseArtifactSourceDirectory/package-info.json -Raw | ConvertFrom-Json).version -$commitMessageFile = New-TemporaryFile -$chagelogEntry = Get-ChangeLogEntryAsString ` - -ChangeLogLocation $ReleaseArtifactSourceDirectory/CHANGELOG.md ` - -VersionString $PackageVersion - -"[$VcpkgPortName] Update to $PackageVersion`n$chagelogEntry" ` - | Set-Content $commitMessageFile - -Write-Host "Commit Message:" -Write-host (Get-Content $commitMessageFile -Raw) - - -Write-Host "git add -A" -git add -A +# Only perform the final commit if this is not a test release +if (!$DailyRelease) { + # Grab content needed for commit message and place in a temporary file + $packageVersion = (Get-Content $ReleaseArtifactSourceDirectory/package-info.json -Raw | ConvertFrom-Json).version + $commitMessageFile = New-TemporaryFile + $chagelogEntry = Get-ChangeLogEntryAsString ` + -ChangeLogLocation $ReleaseArtifactSourceDirectory/CHANGELOG.md ` + -VersionString $PackageVersion + + "[$VcpkgPortName] Update to $PackageVersion`n$chagelogEntry" ` + | Set-Content $commitMessageFile + + Write-Host "Commit Message:" + Write-host (Get-Content $commitMessageFile -Raw) + + Write-Host "git add -A" + git add -A + + # Final commit using commit message from the temporary file. Using the file + # enables the commit message to be formatted properly without having to write + # code to escape certain characters that might appear in the changelog file. + Write-Host "git $GitCommitParameters commit --file $commitMessageFile" + "git $GitCommitParameters commit --file $commitMessageFile" ` + | Invoke-Expression -Verbose ` + | Write-Host + + # Set $(HasChanges) to $true so that create-pull-request.yml completes the + # push and PR submission steps + Write-Host "##vso[task.setvariable variable=HasChanges]$true" +} -# Final commit using commit message from the temporary file. Using the file -# enables the commit message to be formatted properly without having to write -# code to escape certain characters that might appear in the changelog file. -Write-Host "git $GitCommitParameters commit --file $commitMessageFile" -"git $GitCommitParameters commit --file $commitMessageFile" ` - | Invoke-Expression -Verbose ` - | Write-Host <# .SYNOPSIS @@ -147,4 +159,9 @@ Additional parameters to supply to the `git commit` command. These are useful in the context of Azure DevOps where the git client does not have a configured user.name and user.email. +.PARAMETER DailyRelease +In the case of a test release set this to ensure that the x-add-version step +includes `--overwrite-version` to ensure daily packages are properly updated +in the vcpkg repo. + #>