From f773f405102f7e90f049f371e2fe3964e3884b1e Mon Sep 17 00:00:00 2001 From: Jerome Humbert Date: Thu, 5 Mar 2020 17:36:09 +0000 Subject: [PATCH] Link against core PDBs when building interop DLL (#208) This change enables linking the interop DLL Microsoft.MixedReality.WebRTC.Native.dll with the PDBs from the core webrtc.lib and, on UWP platform, the UWP wrappers from Org.WebRtc.WrapperGlue.lib, to enable a better debugging experience for users. Given the large size of PDBs for each build variant (~400 MB and more), this changes uses some Universal Packages to store the PDBs in some private Artifacts feed on Azure DevOps when compiling webrtc.lib, and restore them when compiling Microsoft.MixedReality.WebRTC.Native.dll, as the frequency of the former is much lower than the frequency of the latter, and the former build is generally shared by multiple branches of the latter (e.g. master and release/1.0 at the moment). Additionally, because Azure build agents have a limit of 10GB per build, this change generates on-the-fly a custom packages.config for the current build triple (config-platform-arch) which allows restoring only the necessary core NuGet packages (containing webrtc.lib). This has the extra advantage of reducing the time needed to restore all platform packages: previously 4 on Desktop and 13 on UWP, now only 1 on Desktop and 3 on UWP. The .vcxproj is also modified on-the-fly and in-place to import only the restored packages. Note that those steps are executed via PowerShell scripts during the build and not stored in the committed .vcxproj/packages.config files because it is uncertain whether the Visual Studio IDE would understand platform- and config-dependent NuGet restore, which are rather unusual and only required because we are hitting the NuGet file size limit on nuget.org, therefore the files shipped on GitHub are left untouched with the full set of NuGet packages for all config/arch per platform, to avoid missing any NuGet restore which would prevent the user from compiling the solution. --- ...oft.MixedReality.WebRTC.Native.UWP.vcxproj | 28 ++++++ ...dReality.WebRTC.Native.UWP.vcxproj.filters | 2 +- ...t.MixedReality.WebRTC.Native.Win32.vcxproj | 4 +- ...eality.WebRTC.Native.Win32.vcxproj.filters | 2 +- tools/ci/computePdbPackageVars.ps1 | 73 +++++++++++++++ tools/ci/copyPdbsForBuilding.ps1 | 20 +++++ tools/ci/copyPdbsForPackaging.ps1 | 47 ++++++++++ tools/ci/generateCppPackagesConfig.ps1 | 66 ++++++++++++++ tools/ci/modifyCppProject.ps1 | 88 +++++++++++++++++++ tools/ci/release-core-nosign.yaml | 5 -- tools/ci/templates/jobs-cpp.yaml | 86 ++++++++++++++++-- tools/ci/templates/jobs-libwebrtc-uwp.yaml | 58 ++++++++++-- tools/ci/templates/jobs-libwebrtc-win32.yaml | 41 +++++++-- 13 files changed, 484 insertions(+), 36 deletions(-) create mode 100644 tools/ci/computePdbPackageVars.ps1 create mode 100644 tools/ci/copyPdbsForBuilding.ps1 create mode 100644 tools/ci/copyPdbsForPackaging.ps1 create mode 100644 tools/ci/generateCppPackagesConfig.ps1 create mode 100644 tools/ci/modifyCppProject.ps1 diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj index 068504c30..78127446a 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj @@ -91,6 +91,12 @@ $(ProjectDir)..\..\..\\Microsoft.MixedReality.WebRTC.Unity\Assets\Plugins\WSA\ARM + + true + + + false + @@ -114,6 +120,28 @@ Ole32.lib;Evr.lib;mf.lib;mfuuid.lib;mfplat.lib;webrtc.lib;WindowsApp.lib;Org.WebRtc.WrapperGlue.lib;%(AdditionalDependencies) ..\exports.def $(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\OUTPUT\webrtc\winuwp\$(PlatformTarget)\$(Configuration);$(WebRTCCoreRepoPath)webrtc\windows\projects\msvc\Org.WebRtc.WrapperGlue.Universal\Build\Output\Org.WebRtc.WrapperGlue\$(Configuration)\$(PlatformTarget);$(WebRTCCoreRepoPath)webrtc\windows\projects\msvc\Org.WebRtc.Universal\Build\Output\Org.WebRtc\$(Configuration)\$(PlatformTarget);%(AdditionalLibraryDirectories) + DebugFull + + + + + Disabled + _DEBUG;%(PreprocessorDefinitions) + MultiThreadedDebugDLL + + + + + MaxSpeed + true + true + NDEBUG;%(PreprocessorDefinitions) + true + MultiThreadedDLL + + + true + true diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters index 77193a6f9..7a1d65014 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/uwp/Microsoft.MixedReality.WebRTC.Native.UWP.vcxproj.filters @@ -25,7 +25,7 @@ interop - + media diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj index 0f96a2eb4..09c653046 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj @@ -96,6 +96,7 @@ ..\exports.def strmiids.lib;Msdmo.lib;dmoguids.lib;wmcodecdspuuid.lib;Secur32.lib;winmm.lib;Ole32.lib;Evr.lib;mfreadwrite.lib;mf.lib;mfuuid.lib;mfplat.lib;mfplay.lib;webrtc.lib;%(AdditionalDependencies) $(WebRTCCoreRepoPath)webrtc\xplatform\webrtc\OUTPUT\webrtc\win\$(PlatformTarget)\$(Configuration);%(AdditionalLibraryDirectories) + DebugFull @@ -104,9 +105,6 @@ _DEBUG;%(PreprocessorDefinitions) MultiThreadedDebug - - true - diff --git a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters index 2ecf1b59e..02881131c 100644 --- a/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters +++ b/libs/Microsoft.MixedReality.WebRTC.Native/src/win32/Microsoft.MixedReality.WebRTC.Native.Win32.vcxproj.filters @@ -25,7 +25,7 @@ interop - + media diff --git a/tools/ci/computePdbPackageVars.ps1 b/tools/ci/computePdbPackageVars.ps1 new file mode 100644 index 000000000..c0acb24a6 --- /dev/null +++ b/tools/ci/computePdbPackageVars.ps1 @@ -0,0 +1,73 @@ +# Compute the name and version of the Universal Package used for PDBs, +# and populate the relevant pipeline variables. + +# Compute package name +# Note the restrictions on a Universal Package name (from error message): +# Universal package names must be one or more lowercase alphanumeric +# segments separated by a dash, dot or underscore. The package name +# must be under 256 characters. +if (!$env:BUILDTRIPLE) +{ + $err = "Invalid build triple '$env:BUILDTRIPLE'" + Write-Error $err + Write-Host "##vso[task.complete result=Failed;]$err" + exit 1 +} +$PackageName = "mr-webrtc-core_pdbs_$env:BUILDTRIPLE".ToLowerInvariant() +if ($PackageName.Length -ge 256) { + $err = "Package name too long: '$PackageName'" + Write-Error $err + Write-Host "##vso[task.complete result=Failed;]$err" + exit 1 +} +Write-Host "PDB package name : $PackageName" +Write-Host "##vso[task.setvariable variable=MRWebRTC_PdbPackageName]$PackageName" + +# Compute version if needed, or check existing version +if ($env:WRITE_VERSION -eq 'true') +{ + # Compute package version based on build variables + if ($env:MRWEBRTC_PDBPACKAGEVERSION) + { + # Normally this should compute $(MRWebRTC_PdbPackageVersion) based on the build's version + # and optional release tag and build number. But the variable has also been already assigned + # in the build pipelines variables, which generally indicates a misconfigured pipeline. + $err = "Pipeline variable MRWebRTC_PdbPackageVersion is already set : '$env:MRWEBRTC_PDBPACKAGEVERSION'." + Write-Error $err + Write-Host "##vso[task.complete result=Failed;]$err" + exit 1 + } + if (!$env:MRWEBRTCVERSION) + { + $err = "Invalid build version '$env:MRWEBRTCVERSION'" + Write-Error $err + Write-Host "##vso[task.complete result=Failed;]$err" + exit 1 + } + if ($env:MRWEBRTCRELEASETAG) + { + $PackageVersion = "$env:MRWEBRTCVERSION-$env:MRWEBRTCRELEASETAG" + } + else + { + $PackageVersion = "$env:MRWEBRTCVERSION" + } + if ($env:MRWEBRTCWITHBUILDNUMBER -eq "true") + { + $PackageVersion = "$PackageVersion-$env:BUILD_BUILDNUMBER" + } + Write-Host "PDB package version (generated) : $PackageVersion" + Write-Host "##vso[task.setvariable variable=MRWebRTC_PdbPackageVersion]$PackageVersion" +} +else +{ + # Read and check version from pipeline variables, but do not modify + if (!$env:MRWEBRTC_PDBPACKAGEVERSION) + { + $err = "Invalid PDB package version '$env:MRWEBRTC_PDBPACKAGEVERSION'" + Write-Error $err + Write-Host "##vso[task.complete result=Failed;]$err" + exit 1 + } + Write-Host "PDB package version (from pipeline) : $env:MRWEBRTC_PDBPACKAGEVERSION" +} diff --git a/tools/ci/copyPdbsForBuilding.ps1 b/tools/ci/copyPdbsForBuilding.ps1 new file mode 100644 index 000000000..41328c997 --- /dev/null +++ b/tools/ci/copyPdbsForBuilding.ps1 @@ -0,0 +1,20 @@ +# Copy all PDBs from their packaging folder back next to webrtc.lib +# Note that this is not their original location, but this is where +# the linker currently looks for them. + +param( + [string]$SourcePath, + [string]$OutputPath +) + +# Move all PDBs +Write-Host "Moving PDBs..." +mkdir -Force $OutputPath | out-null +Move-Item -Path $(Join-Path $SourcePath "*") -Destination $OutputPath -Include "*.pdb" + +# List content of output folder +Write-Host "Content of output folder $OutputPath" +foreach ($f in $(Get-ChildItem -Path $OutputPath -Recurse)) +{ + Write-Host $f.FullName +} diff --git a/tools/ci/copyPdbsForPackaging.ps1 b/tools/ci/copyPdbsForPackaging.ps1 new file mode 100644 index 000000000..33a8b2ac5 --- /dev/null +++ b/tools/ci/copyPdbsForPackaging.ps1 @@ -0,0 +1,47 @@ +# Copy all PDBs into a single folder for packaging as Universal Package + +param( + [string]$CorePath, + [string]$WrapperPath, + [string]$WrapperGluePath, + [string]$OutputPath, + [ValidateSet('Debug','Release')] + [string]$BuildConfig, + [switch]$WithUwpWrapper +) + +# Copied from https://github.com/microsoft/MixedReality-Sharing/blob/master/tools/ci/utils.ps1 +function Ensure-Empty($DirectoryPath) { + mkdir -Force "$DirectoryPath" | out-null + Remove-Item "$DirectoryPath\*" -Force -Recurse +} + +# Ensure the output path exists and is empty +Ensure-Empty $OutputPath + +# Copy core PDBs +Write-Host "Copying PDBs for core webrtc.lib..." +Copy-Item -Path $(Join-Path $CorePath "*") -Destination $OutputPath -Include "*.pdb" + +if ($WithUwpWrapper) +{ + # Copy wrapper PDB (Org.WebRtc.pdb) + Write-Host "Copying PDB for Org.WebRtc.dll..." + Copy-Item -Path $(Join-Path $WrapperPath "Org.WebRtc.pdb") -Destination $OutputPath + + # Copy wrapper glue PDB (Org.WebRtc.WrapperGlue.pdb) + # In Release there is no PDB; the project does not specify /DEBUG so defaults to Debug-only PDBs. + if ($BuildConfig -eq "Release") { + Write-Host "Skipping PDB for Org.WebRtc.WrapperGlue.lib (not generated in Release)" + } else { + Write-Host "Copying PDB for Org.WebRtc.WrapperGlue.lib..." + Copy-Item -Path $(Join-Path $WrapperGluePath "Org.WebRtc.WrapperGlue.pdb") -Destination $OutputPath + } +} + +# List content of output folder +Write-Host "Content of output folder $OutputPath" +foreach ($f in $(Get-ChildItem -Path $OutputPath -Recurse)) +{ + Write-Host $f.FullName +} diff --git a/tools/ci/generateCppPackagesConfig.ps1 b/tools/ci/generateCppPackagesConfig.ps1 new file mode 100644 index 000000000..da3ce6ba6 --- /dev/null +++ b/tools/ci/generateCppPackagesConfig.ps1 @@ -0,0 +1,66 @@ +# Generate a custom packages.config to restore only the necessary Core +# NuGet packages for the given build triple, to save on disk space + +param( + [ValidateSet('Win32','UWP')] + [string]$BuildPlatform, + [ValidateSet('x86','x64','ARM')] + [string]$BuildArch, + [ValidateSet('Debug','Release')] + [string]$BuildConfig, + [string]$InputFile, + [string]$OutputFile +) + +Write-Host "Generating packages.config for build triple $BuildPlatform-$BuildArch-$BuildConfig..." + +$reader = [System.IO.File]::OpenText($InputFile) +$content = "" +try { + for() { + $line = $reader.ReadLine() + if ($line -eq $null) { break } + # Check if the package is arch/config dependent. + # Note that this will *not* match the generic UWP package containing the generated headers, + # "Microsoft.MixedReality.WebRTC.Native.Core.UWP" (without '.' after 'UWP'), which should + # always be restored on all UWP variants independent of the arch/config. + if ($line -match "Microsoft\.MixedReality\.WebRTC\.Native\.Core\.(Desktop|UWP|WinRT)\.") + { + # Once the package is known to be config/arch-dependent, validate that this is the + # correct arch and config for the current build. + if ($line -match "Microsoft\.MixedReality\.WebRTC\.Native\.Core\.(Desktop|UWP|WinRT)\.$BuildArch\.$BuildConfig") + { + $linePlatform = $Matches.1 # Desktop|UWP|WinRT + if ((($linePlatform -eq 'Desktop') -and ($BuildPlatform -eq 'Win32')) -or (($linePlatform -ne 'Desktop') -and ($BuildPlatform -eq 'UWP'))) + { + # Copy line for matching build triple as is + $content += $line + "`n"; + } + else + { + # Discard line - mismatching platform + } + } + else + { + # Discard line - mismatching arch/config + } + } + else + { + # Copy any other line as is, including "Microsoft.MixedReality.WebRTC.Native.Core.UWP" + $content += $line + "`n"; + } + } +} +finally { + $reader.Close() +} +Write-Output $content | Set-Content -Path $OutputFile -Encoding UTF8 + +Write-Host "== $OutputFile =======================================" +Get-Content -Encoding UTF8 -Path "$OutputFile" +Write-Host "==========================================================" + +# Write the filename of the new packages.config file to $(PackagesConfigFile) for later use +Write-Host "##vso[task.setvariable variable=PackagesConfigFile]$OutputFile" diff --git a/tools/ci/modifyCppProject.ps1 b/tools/ci/modifyCppProject.ps1 new file mode 100644 index 000000000..df10b026a --- /dev/null +++ b/tools/ci/modifyCppProject.ps1 @@ -0,0 +1,88 @@ +# Modify the .vcxproj project file to only import the NuGet packages for +# the current build triple, which are the only ones restored. + +param( + [ValidateSet('Win32','UWP')] + [string]$BuildPlatform, + [ValidateSet('x86','x64','ARM')] + [string]$BuildArch, + [ValidateSet('Debug','Release')] + [string]$BuildConfig, + [string]$ProjectFile +) + +Write-Host "Modifying .vcxproj for build triple $BuildPlatform-$BuildArch-$BuildConfig..." + +$reader = [System.IO.File]::OpenText($ProjectFile) +$content = "" +try { + for() { + $line = $reader.ReadLine() + if ($line -eq $null) { break } + if ($line -match "Import Project=") + { + if ($line -match "packages\\Microsoft\.MixedReality\.WebRTC\.Native\.Core\.(Desktop|UWP|WinRT)\.(x86|x64|ARM)\.(Debug|Release)") + { + $linePlatform = $Matches.1 # Desktop|UWP|WinRT + $lineArch = $Matches.2 # x86|x64|ARM + $lineConfig = $Matches.3 #Debug|Release + $matchPlatform = (($linePlatform -eq 'Desktop') -and ($BuildPlatform -eq 'Win32')) -or (($linePlatform -ne 'Desktop') -and ($BuildPlatform -eq 'UWP')) + $matchArch = ($lineArch -eq $BuildArch) + $matchConfig = ($lineConfig -eq $BuildConfig) + if ($matchPlatform -and $matchArch -and $matchConfig) + { + # Copy line for matching build triple as is + $content += $line + "`n"; + } + else + { + # Discard line - mismatching platform or arch or config + } + } + else + { + # Copy any other line as is + $content += $line + "`n"; + } + } + elseif ($line -match "Error Condition=") + { + if ($line -match "packages\\Microsoft\.MixedReality\.WebRTC\.Native\.Core\.(Desktop|UWP|WinRT)\.(x86|x64|ARM)\.(Debug|Release)") + { + $linePlatform = $Matches.1 # Desktop|UWP|WinRT + $lineArch = $Matches.2 # x86|x64|ARM + $lineConfig = $Matches.3 #Debug|Release + $matchPlatform = (($linePlatform -eq 'Desktop') -and ($BuildPlatform -eq 'Win32')) -or (($linePlatform -ne 'Desktop') -and ($BuildPlatform -eq 'UWP')) + $matchArch = ($lineArch -eq $BuildArch) + $matchConfig = ($lineConfig -eq $BuildConfig) + if ($matchPlatform -and $matchArch -and $matchConfig) + { + # Copy line for matching build triple as is + $content += $line + "`n"; + } + else + { + # Discard line - mismatching platform or arch or config + } + } + else + { + # Copy any other line as is + $content += $line + "`n"; + } + } + else + { + # Copy any other line as is + $content += $line + "`n"; + } + } +} +finally { + $reader.Close() +} +Write-Output $content | Set-Content -Path $ProjectFile -Encoding UTF8 + +Write-Host "== $ProjectFile =======================================" +Get-Content -Encoding UTF8 -Path "$ProjectFile" +Write-Host "==========================================================" diff --git a/tools/ci/release-core-nosign.yaml b/tools/ci/release-core-nosign.yaml index fc65f8beb..d2993d75c 100644 --- a/tools/ci/release-core-nosign.yaml +++ b/tools/ci/release-core-nosign.yaml @@ -14,11 +14,6 @@ pr: none # Give a unique name to the build each time it runs name: $(Date:yyyyMMdd)-$(Rev:r) -variables: - # Version string is "$(MRWebRTCVersion)-$(MRWebRTCReleaseTag)-$(Build.BuildId)" - MRWebRTCVersion: 0.0.1 # Major.Minor.Patch - MRWebRTCReleaseTag: 'alpha' # Optional, without '-' - stages: # Compile all platform/architecture/config variants as separate jobs diff --git a/tools/ci/templates/jobs-cpp.yaml b/tools/ci/templates/jobs-cpp.yaml index cce1ca6c3..ca977ded8 100644 --- a/tools/ci/templates/jobs-cpp.yaml +++ b/tools/ci/templates/jobs-cpp.yaml @@ -1,5 +1,5 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See LICENSE in the project root for license information. +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. # [TEMPLATE] Compile C++ library Microsoft.MixedReality.WebRTC.Native.dll @@ -9,7 +9,7 @@ parameters: buildArch: '' # x86|x64|ARM|ARM64 msbuildPlatform: '' # Win32|x64|ARM|ARM64 buildConfig: '' # Debug|Release - + jobs: # Compile Microsoft.MixedReality.WebRTC.Native.dll @@ -22,22 +22,75 @@ jobs: - group: MixedReality-WebRTC-ArtifactConfig - name: buildTriple value: ${{parameters.buildPlatform}}-${{parameters.buildArch}}-${{parameters.buildConfig}} + # This is overwritten later if restoring PDBs + - name: PackagesConfigFile + value: '$(Build.SourcesDirectory)/libs/Microsoft.MixedReality.WebRTC.Native/src/${{parameters.buildPlatform}}/packages.config' steps: - checkout: self submodules: recursive + # Map build* variables to script* ones + - task: PowerShell@2 + displayName: Map script variables + inputs: + targetType: filePath + filePath: tools/ci/mapVariables.ps1 + arguments: '${{parameters.buildPlatform}} ${{parameters.buildArch}} ${{parameters.buildConfig}}' + + # Compute the PDB package variables + - task: PowerShell@2 + displayName: 'Compute PDB package variables' + inputs: + targetType: filePath + filePath: tools/ci/computePdbPackageVars.ps1 + env: + # Read $(MRWebRTC_PdbPackageVersion) from pipeline variables + WRITE_VERSION: 'false' + condition: eq(variables['MRWebRTC_RestorePDBs'], 'true') + + # Download the PDBs + - task: UniversalPackages@0 + displayName: Download core PDBs + inputs: + command: download + vstsFeed: $(MRWebRTC_PdbFeed) + vstsFeedPackage: $(MRWebRTC_PdbPackageName) + vstsPackageVersion: $(MRWebRTC_PdbPackageVersion) + downloadDirectory: '$(Build.SourcesDirectory)\_pdbs' + condition: eq(variables['MRWebRTC_RestorePDBs'], 'true') + + # Move PDBs back into their original location for the linker to find them + - task: PowerShell@2 + displayName: Move core PDBs back in place + inputs: + targetType: filePath + filePath: tools/ci/copyPdbsForBuilding.ps1 + arguments: '-BuildConfig ${{parameters.buildConfig}} -BuildPlatform ${{parameters.buildPlatform}} -SourcePath "$(Build.SourcesDirectory)/_pdbs" -OutputPath "bin/${{parameters.buildPlatform}}/${{parameters.buildArch}}/${{parameters.buildConfig}}"' + condition: eq(variables['MRWebRTC_RestorePDBs'], 'true') + # Use NuGet 5.2.0 - task: NuGetToolInstaller@1 displayName: 'Use NuGet 5.2.0' inputs: versionSpec: 5.2.0 - # Restore the NuGet packages containing the input dependencies + # Generate custom packages.config to restore only the necessary packages for the current build triple. + # This helps both with decreasing restore time and with minimizing disk space to avoid the 10GB limit. + # This task sets $(PackagesConfigFile) to the filename of the generated 'packages.config' file. + - task: PowerShell@2 + displayName: Generate packages.config for build triple + inputs: + targetType: filePath + filePath: tools/ci/generateCppPackagesConfig.ps1 + arguments: '-BuildConfig ${{parameters.buildConfig}} -BuildPlatform ${{parameters.buildPlatform}} -BuildArch ${{parameters.buildArch}} -InputFile "$(Build.SourcesDirectory)/libs/Microsoft.MixedReality.WebRTC.Native/src/${{parameters.buildPlatform}}/packages.config" -OutputFile "bin/${{parameters.buildPlatform}}/${{parameters.buildArch}}/${{parameters.buildConfig}}/packages.config"' + condition: eq(variables['MRWebRTC_RestorePDBs'], 'true') + + # Restore the NuGet packages containing the input dependencies (non-PDB case) - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2 # NuGetCommand@2 displayName: 'NuGet restore C++ lib' inputs: command: restore - restoreSolution: '$(Build.SourcesDirectory)/libs/Microsoft.MixedReality.WebRTC.Native/src/${{parameters.buildPlatform}}/packages.config' + restoreSolution: '$(PackagesConfigFile)' restoreDirectory: '$(Build.SourcesDirectory)/packages' includeNuGetOrg: true feedsToUse: config @@ -68,15 +121,24 @@ jobs: arguments: '-a prepare -p $(scriptPlatform) --cpus $(scriptArch) -c $(scriptConfig)' timeoutInMinutes: 10 + # Generate custom .vcxproj to import only the necessary packages for the current build triple + - task: PowerShell@2 + displayName: 'Modify .vcxproj for build triple' + inputs: + targetType: filePath + filePath: tools/ci/modifyCppProject.ps1 + arguments: '-BuildConfig ${{parameters.buildConfig}} -BuildPlatform ${{parameters.buildPlatform}} -BuildArch ${{parameters.buildArch}} -ProjectFile "libs/Microsoft.MixedReality.WebRTC.Native/src/${{parameters.buildPlatform}}/Microsoft.MixedReality.WebRTC.Native.${{parameters.buildPlatform}}.vcxproj"' + condition: eq(variables['MRWebRTC_RestorePDBs'], 'true') + # Build Microsoft.MixedReality.WebRTC.Native.dll - task: MSBuild@1 displayName: 'Build C++ WebRTC lib ($(buildTriple))' inputs: - solution: libs/Microsoft.MixedReality.WebRTC.Native/src/${{parameters.buildPlatform}}/Microsoft.MixedReality.WebRTC.Native.${{parameters.buildPlatform}}.vcxproj + solution: 'libs/Microsoft.MixedReality.WebRTC.Native/src/${{parameters.buildPlatform}}/Microsoft.MixedReality.WebRTC.Native.${{parameters.buildPlatform}}.vcxproj' msbuildVersion: 15.0 msbuildArchitecture: x64 - platform: ${{parameters.msbuildPlatform}} - configuration: ${{parameters.buildConfig}} + platform: '${{parameters.msbuildPlatform}}' + configuration: '${{parameters.buildConfig}}' timeoutInMinutes: 20 # Publish Microsoft.MixedReality.WebRTC.Native.dll and associated files (PDB, ...) @@ -84,7 +146,13 @@ jobs: displayName: 'Publish C++ WebRTC lib ($(buildTriple))' inputs: artifactName: 'Microsoft.MixedReality.WebRTC.Native_$(buildTriple)' - targetPath: bin/${{parameters.buildPlatform}}/${{parameters.buildArch}}/${{parameters.buildConfig}} + targetPath: 'bin/${{parameters.buildPlatform}}/${{parameters.buildArch}}/${{parameters.buildConfig}}' + + # Delete PDBs to save disk space, otherwise the NuGet download below fails in Debug + - powershell: | + Remove-Item "bin/${{parameters.buildPlatform}}/${{parameters.buildArch}}/${{parameters.buildConfig}}/*" -Force -Include "*.pdb" -Exclude "Microsoft.MixedReality.WebRTC.*","NativeLib.*","NativeTests.*" + displayName: 'Delete PDBs after linking webrtc.lib' + condition: eq(variables['MRWebRTC_RestorePDBs'], 'true') # Restore the NuGet packages for the C++ tests - task: 333b11bd-d341-40d9-afcf-b32d5ce6f23b@2 # NuGetCommand@2 diff --git a/tools/ci/templates/jobs-libwebrtc-uwp.yaml b/tools/ci/templates/jobs-libwebrtc-uwp.yaml index 7dd05ba3d..87630e934 100644 --- a/tools/ci/templates/jobs-libwebrtc-uwp.yaml +++ b/tools/ci/templates/jobs-libwebrtc-uwp.yaml @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See LICENSE in the project root for license information. -# [TEMPLATE] Compile WebRTC Core and UWP wrappers +# [TEMPLATE] Compile WebRTC Core and UWP wrappers for UWP platform parameters: buildAgent: '' @@ -27,6 +27,16 @@ jobs: submodules: recursive clean: $(clean.git) + # Compute the PDB package variables + - task: PowerShell@2 + displayName: 'Compute PDB package variables' + inputs: + targetType: filePath + filePath: tools/ci/computePdbPackageVars.ps1 + env: + # Compute and write $(MRWebRTC_PdbPackageVersion) from build pipeline variables + WRITE_VERSION: 'true' + # Map build* variables to script* ones - task: PowerShell@2 displayName: Map script variables @@ -76,7 +86,7 @@ jobs: configuration: ${{parameters.buildConfig}} timeoutInMinutes: 180 - # Publish webrtc.lib + # Publish webrtc.lib as pipeline artifacts (limited retention) - task: PublishPipelineArtifact@0 displayName: 'Publish webrtc.lib ($(buildTriple))' inputs: @@ -84,13 +94,13 @@ jobs: targetPath: 'external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/OUTPUT/webrtc/$(scriptPlatform)/${{parameters.buildArch}}/${{parameters.buildConfig}}/webrtc.lib' timeoutInMinutes: 15 - # Publish PDBs for webrtc.lib - - task: PublishPipelineArtifact@0 - displayName: 'Publish PDBs for webrtc.lib ($(buildTriple))' - inputs: - artifactName: 'libwebrtc_pdbs_$(buildTriple)' - targetPath: 'external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/OUTPUT/webrtc/$(scriptPlatform)/${{parameters.buildArch}}/${{parameters.buildConfig}}/pdbs' - timeoutInMinutes: 15 + # # Publish PDBs for webrtc.lib as pipeline artifacts (limited retention) + # - task: PublishPipelineArtifact@0 + # displayName: 'Publish PDBs for webrtc.lib ($(buildTriple))' + # inputs: + # artifactName: 'libwebrtc_pdbs_$(buildTriple)' + # targetPath: 'external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/OUTPUT/webrtc/$(scriptPlatform)/${{parameters.buildArch}}/${{parameters.buildConfig}}/pdbs' + # timeoutInMinutes: 15 # Publish Org.WebRtc.dll and associated (PDB, ...) - task: PublishPipelineArtifact@0 @@ -116,3 +126,33 @@ jobs: artifactName: 'orgwebrtc_glue_$(buildTriple)' targetPath: 'external/webrtc-uwp-sdk/webrtc/windows/projects/msvc/Org.WebRtc.WrapperGlue.Universal/Build/Output/Org.WebRtc.WrapperGlue/${{parameters.buildConfig}}/${{parameters.buildArch}}' timeoutInMinutes: 15 + + # Copy all PDBs to a single directory for package publishing + - task: PowerShell@2 + displayName: Copy PDBs for packaging + inputs: + targetType: filePath + filePath: tools/ci/copyPdbsForPackaging.ps1 + arguments: '-WithUwpWrapper -CorePath "external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/OUTPUT/webrtc/$(scriptPlatform)/${{parameters.buildArch}}/${{parameters.buildConfig}}/pdbs" -WrapperPath "external/webrtc-uwp-sdk/webrtc/windows/projects/msvc/Org.WebRtc.Universal/Build/Output/Org.WebRtc/${{parameters.buildConfig}}/${{parameters.buildArch}}" -WrapperGluePath "external/webrtc-uwp-sdk/webrtc/windows/projects/msvc/Org.WebRtc.WrapperGlue.Universal/Build/Output/Org.WebRtc.WrapperGlue/${{parameters.buildConfig}}/${{parameters.buildArch}}" -OutputPath $(Build.ArtifactStagingDirectory)/pdbs -BuildConfig ${{parameters.buildConfig}}' + + # List content of PDB packaging folder + - powershell: | + foreach ($f in $(Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/pdbs -Recurse)) + { + Write-Host $f.FullName + } + displayName: 'List PDBs to package' + timeoutInMinutes: 5 + + # Publish PDBs for webrtc.lib as Universal Package (unlimited retention) + - task: UniversalPackages@0 + displayName: 'Publish PDBs for webrtc.lib ($(buildTriple))' + inputs: + command: publish + publishDirectory: '$(Build.ArtifactStagingDirectory)/pdbs' + vstsFeedPublish: $(MRWebRTC_PdbFeed) + vstsFeedPackagePublish: $(MRWebRTC_PdbPackageName) + versionOption: custom + versionPublish: $(MRWebRTC_PdbPackageVersion) + packagePublishDescription: 'PDBs for MixedReality-WebRTC core (webrtc.lib) $(buildTriple)' + timeoutInMinutes: 30 diff --git a/tools/ci/templates/jobs-libwebrtc-win32.yaml b/tools/ci/templates/jobs-libwebrtc-win32.yaml index 7f2492c47..dd48af3cf 100644 --- a/tools/ci/templates/jobs-libwebrtc-win32.yaml +++ b/tools/ci/templates/jobs-libwebrtc-win32.yaml @@ -1,11 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See LICENSE in the project root for license information. -# [TEMPLATE] Compile WebRTC Core +# [TEMPLATE] Compile WebRTC Core for Desktop (Win32) platform parameters: buildAgent: '' - buildArch: '' # x86|x64|ARM|ARM64 + buildArch: '' # x86|x64 buildConfig: '' # Debug|Release jobs: @@ -25,6 +25,16 @@ jobs: - checkout: self submodules: recursive + # Compute the PDB package variables + - task: PowerShell@2 + displayName: 'Compute PDB package variables' + inputs: + targetType: filePath + filePath: tools/ci/computePdbPackageVars.ps1 + env: + # Compute and write $(MRWebRTC_PdbPackageVersion) from build pipeline variables + WRITE_VERSION: 'true' + # Map build* variables to script* ones - task: PowerShell@2 displayName: Map script variables @@ -44,7 +54,7 @@ jobs: configuration: ${{parameters.buildConfig}} timeoutInMinutes: 120 - # Publish webrtc.lib + # Publish webrtc.lib as pipeline artifacts (limited retention) - task: PublishPipelineArtifact@0 displayName: 'Publish webrtc.lib ($(buildTriple))' inputs: @@ -52,11 +62,26 @@ jobs: targetPath: 'external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/OUTPUT/webrtc/$(scriptPlatform)/${{parameters.buildArch}}/${{parameters.buildConfig}}/webrtc.lib' timeoutInMinutes: 15 - # Publish PDBs for webrtc.lib - - task: PublishPipelineArtifact@0 + # Publish PDBs for webrtc.lib as pipeline artifacts (limited retention) + # - task: PublishPipelineArtifact@0 + # displayName: 'Publish PDBs for webrtc.lib ($(buildTriple))' + # inputs: + # artifactName: 'libwebrtc_pdbs_$(buildTriple)' + # targetPath: 'external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/OUTPUT/webrtc/$(scriptPlatform)/${{parameters.buildArch}}/${{parameters.buildConfig}}/pdbs' + # timeoutInMinutes: 15 + + # Publish PDBs for webrtc.lib as Universal Package (unlimited retention) + - task: UniversalPackages@0 displayName: 'Publish PDBs for webrtc.lib ($(buildTriple))' inputs: - artifactName: 'libwebrtc_pdbs_$(buildTriple)' - targetPath: 'external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/OUTPUT/webrtc/$(scriptPlatform)/${{parameters.buildArch}}/${{parameters.buildConfig}}/pdbs' - timeoutInMinutes: 15 + command: publish + publishDirectory: 'external/webrtc-uwp-sdk/webrtc/xplatform/webrtc/OUTPUT/webrtc/$(scriptPlatform)/${{parameters.buildArch}}/${{parameters.buildConfig}}/pdbs' + vstsFeedPublish: $(MRWebRTC_PdbFeed) + vstsFeedPackagePublish: $(MRWebRTC_PdbPackageName) + versionOption: custom + versionPublish: $(MRWebRTC_PdbPackageVersion) + packagePublishDescription: 'PDBs for MixedReality-WebRTC core (webrtc.lib) $(buildTriple)' + timeoutInMinutes: 30 + +