Skip to content

Commit

Permalink
Add nightly vcpkg publishing (#2025)
Browse files Browse the repository at this point in the history
* Tool changes to support nightly test package release

* Initial wiring up and refactor vcpkg-publish.yml

* Use vcpkg-clone.yml

* Disable publishing for verification of nightly builds

* Complete the comment

* Dependency is enforced at the stage level, not the deployment/job level

* Remove environment, no approval needed

* Add ability to enable test release parameters

* deploy -> job

* deployment -> job

* Remove strategy

* Download pipeline artifacts

* task -> download

* Set appropriate working directory

* git status

* Set working directory

* Add pipeline for nightly vcpkg PR update

* Correct vcpkg-clone.yml path

* Add identity parameters to git merge commands

* Remove GitIdentityParameters, the merge command does not suppor tthem. Add instead in the pipeline

* Split lines

* Write-Host

* Template for nightly branch name, update comments, Check for scheduling or "PublishNightlyVcpkg"

* include archetype-cpp-release.yml changes

* Re-enable publishing stage

* Close the and

* Runtime condition for integration stage

* Move integration below package publishing stages

* try -- to disambiguate

* ^ -> ~

* Remove --

* Documentation and variable naming

* Move documentation comment

* Update eng/pipelines/templates/steps/generate-nightly-branch-name.yml

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

* Review feedback

* Add ability to skip all release steps

* Equal sign

* The right number of dashes

* Checkout before merge

* Checkout the default branch

* --no-ff

* git reset

* Skip publishing for artifacts which are not publishign to vcpkg

* Add git config --unset

* Apply suggestions from code review

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

* Review feedback

* Review feedback

* Remove extra reset

* TestRelease -> DailyRelease

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>
  • Loading branch information
danieljurek and weshaggard authored Apr 6, 2021
1 parent 38d3535 commit 5a99f38
Show file tree
Hide file tree
Showing 7 changed files with 248 additions and 76 deletions.
54 changes: 54 additions & 0 deletions eng/pipelines/daily-verification.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@

jobs:
- job:
displayName: Daily vcpkg PR
pool:
name: azsdk-pool-mms-win-2019-general
vmImage: MMS2019

variables:
# The branch in the azure-sdk/vcpkg repo that is updated daily with the
# latest set of ports
TargetPrBranch: release/azure-sdk-for-cpp-daily-verification

steps:
- checkout: self

- template: /eng/pipelines/templates/steps/generate-daily-branch-name.yml

- template: /eng/pipelines/templates/steps/vcpkg-clone.yml
parameters:
Workspace: $(Pipeline.Workspace)
RepoOwner: azure-sdk

- template: /eng/common/pipelines/templates/steps/set-default-branch.yml
parameters:
WorkingDirectory: $(Pipeline.Workspace)/vcpkg

- pwsh: |
Write-Host "git checkout $(DailyBranchName)"
git checkout $(DailyBranchName)
if ($LASTEXITCODE -ne 0) {
Write-Error "Cannot check out $(DailyBranchName) LASTEXITCODE=$LASTEXITCODE"
}
Write-Host "##vso[task.setvariable variable=HasChanges]$true"
workingDirectory: $(Pipeline.Workspace)/vcpkg
displayName: Update vcpkg Daily Branch
# This step will skip creating the PR if the PR already exists
- template: /eng/common/pipelines/templates/steps/create-pull-request.yml
parameters:
# Force push so this recreates the $(TargetPrBranch)
PushArgs: --force
BaseBranchName: $(DefaultBranch)
PRBranchName: $(TargetPrBranch)
RepoOwner: Microsoft
RepoName: vcpkg
WorkingDirectory: $(Pipeline.Workspace)/vcpkg
CommitMsg: Update with changes from $(DailyBranchName)
PRTitle: "[DO NOT MERGE] Daily Azure SDK for C++ CI validation"
PRBody: This is a long-lived draft PR should not be merged. Its branch is be updated periodically to validate the Azure SDK for C++ against the vcpkg CI.
ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts
OpenAsDraft: true
SkipCheckingForChanges: true
105 changes: 61 additions & 44 deletions eng/pipelines/templates/stages/archetype-cpp-release.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
parameters:
Artifacts: []
ArtifactName: 'not-specified'
ServiceDirectory: not-specified
TestPipeline: false

stages:
Expand All @@ -9,7 +10,7 @@ stages:
- stage: Release_${{ replace(artifact.Name, '-', '_') }}
displayName: 'Release ${{artifact.name}}'
dependsOn: Build
condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp'))
condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp'), ne(variables['Skip.AllRelease'], 'true'))

jobs:
- deployment: TagRepository
Expand Down Expand Up @@ -122,52 +123,18 @@ stages:
Write-Host "##vso[task.setvariable variable=PrTitle]$title"
displayName: Set PR title
- task: Powershell@2
inputs:
filePath: eng/scripts/Initialize-VcpkgRelease.ps1
arguments: >-
-GitHubRepo "$(Build.Repository.Name)"
-PackageSpecPath $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json
-SourceDirectory $(Pipeline.Workspace)/packages/${{artifact.Name}}/vcpkg
-Verbose
pwsh: true
displayName: Initialize vcpkg release

# There are potential race conditions if this script runs
# in parallel against the same branch name. Release only
# one package at a time.
- pwsh: |
Write-Host "git clone https://github.com/azure-sdk/vcpkg $(Pipeline.Workspace)/vcpkg"
git clone https://github.com/azure-sdk/vcpkg $(Pipeline.Workspace)/vcpkg
if ($LASTEXITCODE -ne 0) {
Write-Error "Unable to check out vcpkg fork repo"
exit $LASTEXITCODE
}
Write-Host "##vso[task.setvariable variable=VcpkgWorkspace]$(Pipeline.Workspace)/vcpkg"
displayName: Clone vcpkg from upstream
- template: /eng/pipelines/templates/steps/vcpkg-clone.yml
parameters:
RepoOwner: azure-sdk
PRBranchName: $(PrBranchName)

# Check out the PR branch if it's already in remote.
# Ignore failures.
- pwsh: |
$ErrorActionPreference = "Continue"
git checkout "origin/$(PrBranchName)" 2>&1 | Out-Null
$LASTEXITCODE = 0 # This ignores any error from git checkout
git status
displayName: Checkout Previous PRBranch if it exist.
workingDirectory: $(VcpkgWorkspace)
- task: Powershell@2
inputs:
pwsh: true
targetType: filePath
filePath: eng/scripts/Update-VcpkgPort.ps1
arguments: >-
-ReleaseArtifactSourceDirectory "$(Pipeline.Workspace)/packages/${{ artifact.Name }}"
-PortDestinationDirectory 'ports/${{ artifact.VcpkgPortName }}'
-VcpkgPortName '${{ artifact.VcpkgPortName }}'
-GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"'
workingDirectory: $(VcpkgWorkspace)
displayName: Update vcpkg port
- template: /eng/pipelines/templates/steps/vcpkg-publish.yml
parameters:
ArtifactName: ${{ artifact.Name }}
VcpkgPortName: ${{ artifact.VcpkgPortName }}

# Set $(HasChanges) to $true so that
# create-pull-request.yml completes the push and PR
Expand Down Expand Up @@ -214,4 +181,54 @@ stages:
PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId)
CommitMsg: "Increment package version after release of ${{ artifact.name }}"
PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases"
CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}'
CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}'

- ${{if eq(variables['System.TeamProject'], 'internal') }}:
- stage: Integration
dependsOn: Build
condition: and(succeeded(), or(eq(variables['PublishDailyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule')))
jobs:
- job: PublishDailyVcpkg
displayName: Publish to vcpkg daily branch

pool:
name: azsdk-pool-mms-win-2019-general
vmImage: MMS2019

steps:
- checkout: self
- download: current
artifact: packages

- template: /eng/pipelines/templates/steps/generate-daily-branch-name.yml

- template: /eng/pipelines/templates/steps/vcpkg-clone.yml
parameters:
RepoOwner: azure-sdk
PRBranchName: $(DailyBranchName)

# Placing artifacts in the same branch in sequence reduces the
# potential for race conditions. If this job does fail to push
# because another job has already pushed to the same branch it
# can be re-run.
- ${{ each artifact in parameters.Artifacts }}:
# Only run vcpkg publish if the artifact has a "VcpkgPortName"
# property. Absence of VcpkgPortName implies that the artifact
# does not ship to vcpkg.
- ${{ if ne(artifact.VcpkgPortName, '') }}:
- template: /eng/pipelines/templates/steps/vcpkg-publish.yml
parameters:
ArtifactName: ${{ artifact.Name }}
VcpkgPortName: ${{ artifact.VcpkgPortName }}
DisplayNameExtension: ${{ artifact.Name }}
DailyRelease: true
DailyReleaseRef: $(Build.SourceVersion)

- template: /eng/common/pipelines/templates/steps/git-push-changes.yml
parameters:
BaseRepoBranch: $(DailyBranchName)
CommitMsg: Daily vcpkg ports for ${{ parameters.ServiceDirectory }}
TargetRepoOwner: azure-sdk
TargetRepoName: vcpkg
WorkingDirectory: $(Pipeline.Workspace)/vcpkg
ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts
9 changes: 9 additions & 0 deletions eng/pipelines/templates/steps/generate-daily-branch-name.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
steps:
- pwsh: |
$branchName = $env:DAILYBRANCHOVERRIDE
if (!$branchName) {
$formattedDate = Get-Date -Format 'yyyy-MM-dd'
$branchName = "release/daily-$formattedDate"
}
Write-Host "##vso[task.setvariable variable=DailyBranchName]$branchName"
displayName: Set branch name in fork repo
24 changes: 24 additions & 0 deletions eng/pipelines/templates/steps/vcpkg-clone.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
parameters:
Workspace: $(Pipeline.Workspace)
RepoOwner: not-set
RepoName: vcpkg
PRBranchName: not-set

steps:
- pwsh: |
Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg"
git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg
if ($LASTEXITCODE -ne 0) {
Write-Error "Unable to check out vcpkg fork repo"
exit $LASTEXITCODE
}
displayName: Clone vcpkg from upstream
# Check out the PR branch if it's already in remote. Ignore failures.
- pwsh: |
$ErrorActionPreference = "Continue"
git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null
$LASTEXITCODE = 0 # This ignores any error from git checkout
git status
displayName: Checkout Previous PRBranch if it exists
workingDirectory: ${{ parameters.Workspace }}/vcpkg
35 changes: 35 additions & 0 deletions eng/pipelines/templates/steps/vcpkg-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
parameters:
GitHubRepo: $(Build.Repository.Name)
Workspace: $(Pipeline.Workspace)
ArtifactName: not-set
VcpkgPortName: not-set
DisplayNameExtension:
DailyReleaseRef:
DailyRelease: false

steps:
- task: Powershell@2
inputs:
filePath: eng/scripts/Initialize-VcpkgRelease.ps1
arguments: >-
-GitHubRepo "${{ parameters.GitHubRepo }}"
-PackageSpecPath ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/package-info.json
-SourceDirectory ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/vcpkg
-DailyReleaseRef '${{ parameters.DailyReleaseRef }}'
-Verbose
pwsh: true
displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }}

- task: Powershell@2
inputs:
pwsh: true
targetType: filePath
filePath: eng/scripts/Update-VcpkgPort.ps1
arguments: >-
-ReleaseArtifactSourceDirectory "${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}"
-PortDestinationDirectory 'ports/${{ parameters.VcpkgPortName }}'
-VcpkgPortName '${{ parameters.VcpkgPortName }}'
-GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"'
-DailyRelease:$${{ parameters.DailyRelease }}
workingDirectory: ${{ parameters.Workspace }}/vcpkg
displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }}
24 changes: 20 additions & 4 deletions eng/scripts/Initialize-VcpkgRelease.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,12 @@ Location of the relevant package-info.json file
.PARAMETER GitHubRepo
Name of the GitHub repo (of the form Azure/azure-sdk-for-cpp)
#>
.PARAMETER DailyReleaseRef
If supplied update the portfile.cmake file's REF and SHA512 with values
associated with the given ref.
#>
[CmdletBinding()]
param (
[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
Expand All @@ -25,7 +29,9 @@ param (

[Parameter(Mandatory = $true)]
[ValidateNotNullOrEmpty()]
[string] $GitHubRepo
[string] $GitHubRepo,

[string] $DailyReleaseRef
)

# If there's nothing in the "port" folder to upload set SkipVcpkgUpdate to true
Expand All @@ -39,6 +45,11 @@ if (!(Get-ChildItem -Path "$SourceDirectory/port/CONTROL")) {
$packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json
$tarGzUri = "https://github.com/$GitHubRepo/archive/$($packageSpec.packageName).tar.gz"

if ($DailyReleaseRef) {
Write-Verbose "Initializing Daily Release"
$tarGzUri = "https://github.com/$GitHubRepo/archive/$DailyReleaseRef.tar.gz"
}

Write-Host "Downloading tarball to compute hash from $tarGzUri"
$localTarGzPath = New-TemporaryFile
Invoke-WebRequest -Uri $tarGzUri -OutFile $localTarGzPath
Expand All @@ -53,7 +64,12 @@ $portfileLocation = "$SourceDirectory/port/portfile.cmake"
# recommended in vcpkg documentation
# Before: " SHA512 1"
# After: " SHA512 f6cf1c16c52"
$newContent = Get-Content -Raw -Path $portfileLocation `
| ForEach-Object { $_ -replace '(SHA512\s+)1', "`${1}$sha512" }
$portFileContent = Get-Content -Raw -Path $portfileLocation
$newContent = $portFileContent -replace '(SHA512\s+)1', "`${1}$sha512"

if ($DailyReleaseRef) {
Write-Verbose "Overriding REF with test release ref: $DailyReleaseRef"
$newContent = $newContent -replace '(?m)^(\s+)REF azure.*$', "`${1}REF $DailyReleaseRef"
}

$newContent | Set-Content $portfileLocation -NoNewLine
Loading

0 comments on commit 5a99f38

Please sign in to comment.