Changes to nuget functionality (#856)

With this release the implementation for delivering to NuGet packages
(by adding the NuGetContext secret), is similar to the functionality
behind delivering to GitHub packages and the implementation is final.

New Settings:
- `trustMicrosoftNuGetFeeds` Unless this setting is set to false, AL-Go
for GitHub will trust the NuGet feeds provided by Microsoft. The feeds
provided by Microsoft contains all Microsoft apps, all Microsoft symbols
and symbols for all AppSource apps.
- `trustedNuGetFeeds` - can be an array of NuGet feed specifications,
which AL-Go for GitHub will use for dependency resolution. Every feed
specification must include a url property and can optionally include a
few other properties:
- Url - The URL of the feed (examples:
https://pkgs.dev.azure.com/myorg/apps/\_packaging/myrepo/nuget/v3/index.json
or https://nuget.pkg.github.com/mygithuborg/index.json")
- Patterns - AL-Go for Github will only trust packages, where the ID
matches this pattern. Default is all packages (\*).
- FingerPrints - If specified, AL-Go for GitHub will only trust packages
signed with one of the fingerprints specified in this array of
fingerprints.
- AuthTokenSecret - If the NuGet feed specified by URL is private, the
authTokenSecret must be the name of a secret containing the
authentication token with permissions to search and read packages from
the NuGet feed

---------

Co-authored-by: freddydk <freddydk@users.noreply.github.com>

Actions/AL-Go-Helper.ps1

@@ -657,6 +657,7 @@ function ReadSettings {
             "defaultIndexMD"                            = "## Reference documentation\n\nThis is the generated reference documentation for [{REPOSITORY}](https://github.com/{REPOSITORY}).\n\nYou can use the navigation bar at the top and the table of contents to the left to navigate your documentation.\n\nYou can change this content by creating/editing the **{INDEXTEMPLATERELATIVEPATH}** file in your repository or use the alDoc:defaultIndexMD setting in your repository settings file (.github/AL-Go-Settings.json)\n\n{RELEASENOTES}"
             "defaultReleaseMD"                          = "## Release reference documentation\n\nThis is the generated reference documentation for [{REPOSITORY}](https://github.com/{REPOSITORY}).\n\nYou can use the navigation bar at the top and the table of contents to the left to navigate your documentation.\n\nYou can change this content by creating/editing the **{INDEXTEMPLATERELATIVEPATH}** file in your repository or use the alDoc:defaultReleaseMD setting in your repository settings file (.github/AL-Go-Settings.json)\n\n{RELEASENOTES}"
         }
+        "trustMicrosoftNuGetFeeds"                      = $true
     }
 
     # Read settings from files and merge them into the settings object

Actions/Deliver/Deliver.ps1

@@ -1,4 +1,4 @@
-Param(
+Param(
     [Parameter(HelpMessage = "The GitHub actor running the action", Mandatory = $false)]
     [string] $actor,
     [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)]
@@ -189,7 +189,6 @@ foreach ($thisProject in $projectList) {
             "RepoSettings"    = $settings
             "ProjectSettings" = $projectSettings
         }
-        #Calculate the folders per artifact type
 
         #Calculate the folders per artifact type
         'Apps', 'TestApps', 'Dependencies' | ForEach-Object {
@@ -226,9 +225,24 @@ foreach ($thisProject in $projectList) {
         Write-Host "Calling custom script: $customScript"
         . $customScript -parameters $parameters
     }
-    elseif ($deliveryTarget -eq "GitHubPackages") {
-        $githubPackagesCredential = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets.githubPackagesContext)) | ConvertFrom-Json
-        'Apps' | ForEach-Object {
+    elseif ($deliveryTarget -eq 'GitHubPackages' -or $deliveryTarget -eq 'NuGet') {
+        $preReleaseTag = ''
+        try {
+            $nuGetAccount = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$($deliveryTarget)Context")) | ConvertFrom-Json | ConvertTo-HashTable
+            if ($deliveryTarget -eq 'NuGet' -and $type -eq 'CD') {
+                # When doing continuous delivery to NuGet, we always use the preview tag
+                # When doing a release, we do not add a preview tag
+                $preReleaseTag = 'preview'
+            }
+            $nuGetServerUrl = $nuGetAccount.ServerUrl
+            Write-Host $nuGetAccount.ServerUrl
+            $nuGetToken = $nuGetAccount.Token
+            Write-Host "$($deliveryTarget)Context secret OK"
+        }
+        catch {
+            throw "$($deliveryTarget)Context secret is malformed. Needs to be formatted as Json, containing serverUrl and token as a minimum."
+        }
+        'Apps','TestApps' | ForEach-Object {
             $folder = @(Get-ChildItem -Path (Join-Path $artifactsFolder "$project-$refname-$($_)-*.*.*.*") | Where-Object { $_.PSIsContainer })
             if ($folder.Count -gt 1) {
                 $folder | Out-Host
@@ -237,93 +251,16 @@ foreach ($thisProject in $projectList) {
             elseif ($folder.Count -eq 1) {
                 Get-Item -Path (Join-Path $folder[0] "*.app") | ForEach-Object {
                     $parameters = @{
-                        "gitHubRepository"         = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY"
-                        "includeNuGetDependencies" = $true
-                        "dependencyIdTemplate"     = "AL-Go-{id}"
-                        "packageId"                = "AL-Go-{id}"
+                        "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY"
+                        "preReleaseTag"    = $preReleaseTag
+                        "appFile"          = $_.FullName
                     }
-                    $parameters.appFiles = $_.FullName
                     $package = New-BcNuGetPackage @parameters
-                    Push-BcNuGetPackage -nuGetServerUrl $gitHubPackagesCredential.serverUrl -nuGetToken $gitHubPackagesCredential.token -bcNuGetPackage $package
+                    Push-BcNuGetPackage -nuGetServerUrl $nuGetServerUrl -nuGetToken $nuGetToken -bcNuGetPackage $package
                 }
             }
         }
     }
-    elseif ($deliveryTarget -eq "NuGet") {
-        try {
-            $nuGetAccount = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets.nuGetContext)) | ConvertFrom-Json | ConvertTo-HashTable
-            $nuGetServerUrl = $nuGetAccount.ServerUrl
-            $nuGetToken = $nuGetAccount.Token
-            Write-Host "NuGetContext secret OK"
-        }
-        catch {
-            throw "NuGetContext secret is malformed. Needs to be formatted as Json, containing serverUrl and token as a minimum."
-        }
-        $appsfolder = @(Get-ChildItem -Path (Join-Path $artifactsFolder "$project-$refname-Apps-*.*.*.*") | Where-Object { $_.PSIsContainer })
-        if ($appsFolder.Count -eq 0) {
-            throw "Internal error - unable to locate apps folder"
-        }
-        elseif ($appsFolder.Count -gt 1) {
-            $appsFolder | Out-Host
-            throw "Internal error - multiple apps folders located"
-        }
-        $testAppsFolder = @(Get-ChildItem -Path (Join-Path $artifactsFolder "$project-$refname-TestApps-*.*.*.*") | Where-Object { $_.PSIsContainer })
-        if ($testAppsFolder.Count -gt 1) {
-            $testAppsFolder | Out-Host
-            throw "Internal error - multiple testApps folders located"
-        }
-        $dependenciesFolder = @(Get-ChildItem -Path (Join-Path $artifactsFolder "$project-$refname-Dependencies-*.*.*.*") | Where-Object { $_.PSIsContainer })
-        if ($dependenciesFolder.Count -gt 1) {
-            $dependenciesFolder | Out-Host
-            throw "Internal error - multiple dependencies folders located"
-        }
-
-        $parameters = @{
-            "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY"
-        }
-        $parameters.appFiles = @(Get-Item -Path (Join-Path $appsFolder[0] "*.app") | ForEach-Object { $_.FullName })
-        if ($testAppsFolder.Count -gt 0) {
-            $parameters.testAppFiles = @(Get-Item -Path (Join-Path $testAppsFolder[0] "*.app") | ForEach-Object { $_.FullName })
-        }
-        if ($dependenciesFolder.Count -gt 0) {
-            $parameters.dependencyAppFiles = @(Get-Item -Path (Join-Path $dependenciesFolder[0] "*.app") | ForEach-Object { $_.FullName })
-        }
-        if ($nuGetAccount.Keys -contains 'PackageName') {
-            $parameters.packageId = $nuGetAccount.PackageName.replace('{project}', $projectName).replace('{owner}', $ENV:GITHUB_REPOSITORY_OWNER).replace('{repo}', $settings.repoName)
-        }
-        else {
-            if ($thisProject -and ($thisProject -eq '.')) {
-                $parameters.packageId = "$($ENV:GITHUB_REPOSITORY_OWNER)-$($settings.repoName)"
-            }
-            else {
-                $parameters.packageId = "$($ENV:GITHUB_REPOSITORY_OWNER)-$($settings.repoName)-$ProjectName"
-            }
-        }
-        if ($type -eq 'CD') {
-            $parameters.packageId += "-preview"
-        }
-        $parameters.packageVersion = [System.Version]$appsFolder[0].Name.SubString($appsFolder[0].Name.IndexOf("-Apps-") + 6)
-        if ($nuGetAccount.Keys -contains 'PackageTitle') {
-            $parameters.packageTitle = $nuGetAccount.PackageTitle
-        }
-        else {
-            $parameters.packageTitle = $parameters.packageId
-        }
-        if ($nuGetAccount.Keys -contains 'PackageDescription') {
-            $parameters.packageDescription = $nuGetAccount.PackageDescription
-        }
-        else {
-            $parameters.packageDescription = $parameters.packageTitle
-        }
-        if ($nuGetAccount.Keys -contains 'PackageAuthors') {
-            $parameters.packageAuthors = $nuGetAccount.PackageAuthors
-        }
-        else {
-            $parameters.packageAuthors = $actor
-        }
-        $package = New-BcNuGetPackage @parameters
-        Push-BcNuGetPackage -nuGetServerUrl $nuGetServerUrl -nuGetToken $nuGetToken -bcNuGetPackage $package
-    }
     elseif ($deliveryTarget -eq "Storage") {
         InstallAzModuleIfNeeded -name 'Az.Storage'
         try {
@@ -457,9 +394,9 @@ foreach ($thisProject in $projectList) {
                 }
                 $depfolder = $depfolder[0].FullName
                 $libraryAppFiles += @(Get-ChildItem -path $depFolder | Where-Object {
-                        $name = $_.name
-                        $appSourceIncludeDependencies | Where-Object { $name -like $_ }
-                    } | ForEach-Object { $_.FullName })
+                    $name = $_.name
+                    $appSourceIncludeDependencies | Where-Object { $name -like $_ }
+                } | ForEach-Object { $_.FullName })
             }
 
             Write-Host "Main App File:"

Actions/DetermineDeliveryTargets/DetermineDeliveryTargets.ps1

@@ -5,6 +5,16 @@
     [bool] $checkContextSecrets
 )
 
+function ContinuousDelivery([string] $deliveryTarget) {
+    $settingsName = "DeliverTo$deliveryTarget"
+    if ($settings.Contains($settingsName) -and $settings."$settingsName".Contains('ContinuousDelivery')) {
+        return $settings."$settingsName".ContinuousDelivery
+    }
+    else {
+        return $true
+    }
+}
+
 function IncludeBranch([string] $deliveryTarget) {
     $settingsName = "DeliverTo$deliveryTarget"
     if ($settings.Contains($settingsName) -and $settings."$settingsName".Contains('Branches')) {
@@ -26,7 +36,7 @@ function IncludeDeliveryTarget([string] $deliveryTarget) {
         Write-Host "- Secret '$contextName' not found"
         return $false
     }
-    return (IncludeBranch -deliveryTarget $deliveryTarget)
+    return (IncludeBranch -deliveryTarget $deliveryTarget) -and (ContinuousDelivery -deliveryTarget $deliveryTarget)
 }
 
 . (Join-Path -Path $PSScriptRoot -ChildPath "..\AL-Go-Helper.ps1" -Resolve)

Actions/DownloadProjectDependencies/README.md

@@ -11,7 +11,7 @@ The action constructs arrays of paths to .app files, that are dependencies of th
 | Name | Description |
 | :-- | :-- |
 | Settings | env.Settings must be set by a prior call to the ReadSettings Action |
-| Secrets | env.Secrets must be read by a prior call to the ReadSecrets Action with appDependencyProbingPathsSecrets in getSecrets |
+| Secrets | env.Secrets must be read by a prior call to the ReadSecrets Action with appDependencySecrets in getSecrets |
 
 ### Parameters
 

Actions/ReadSecrets/README.md

@@ -2,7 +2,7 @@
 
 Read secrets from GitHub secrets or Azure Keyvault for AL-Go workflows
 The secrets read and added to the output are the secrets specified in the getSecrets parameter
-Additionally, the secrets specified by the authToken secret in AppDependencyProbingPaths are read if appDependencyProbingPathsSecrets is specified in getSecrets
+Additionally, the secrets specified by the authTokenSecret in AppDependencyProbingPaths and TrustedNuGetFeeds are read if appDependencySecrets is specified in getSecrets
 All secrets included in the Secrets output are Base64 encoded to avoid issues with national characters
 Secrets, which name is preceded by an asterisk (\*) are encrypted and Base64 encoded
 
@@ -20,7 +20,7 @@ Secrets, which name is preceded by an asterisk (\*) are encrypted and Base64 enc
 | :-- | :-: | :-- | :-- |
 | shell | | The shell (powershell or pwsh) in which the PowerShell script in this action should run | powershell |
 | gitHubSecrets | Yes | GitHub secrets in a json structure | |
-| getSecrets | Yes | Comma-separated list of secrets to get (add appDependencyProbingPathsSecrets to request secrets needed for resolving dependencies in AppDependencyProbingPaths, add TokenForPush in order to request a token to use for pull requests and commits). Secrets preceded by an asterisk are returned encrypted | |
+| getSecrets | Yes | Comma-separated list of secrets to get (add appDependencySecrets to request secrets needed for resolving dependencies in AppDependencyProbingPaths and TrustedNuGetFeeds, add TokenForPush in order to request a token to use for pull requests and commits). Secrets preceded by an asterisk are returned encrypted | |
 | useGhTokenWorkflowForPush | false | Determines whether you want to use the GhTokenWorkflow secret for TokenForPush | false |
 
 ## OUTPUT

Actions/ReadSecrets/ReadSecrets.ps1

@@ -27,7 +27,7 @@ try {
     $outSecrets = [ordered]@{}
     $settings = $env:Settings | ConvertFrom-Json | ConvertTo-HashTable
     $keyVaultCredentials = GetKeyVaultCredentials
-    $getAppDependencyProbingPathsSecrets = $false
+    $getAppDependencySecrets = $false
     $getTokenForPush = $false
     [System.Collections.ArrayList]$secretsCollection = @()
     foreach($secret in ($getSecrets.Split(',') | Select-Object -Unique)) {
@@ -38,8 +38,8 @@ try {
             $secret = 'ghTokenWorkflow'
         }
         $secretNameProperty = "$($secret.TrimStart('*'))SecretName"
-        if ($secret -eq 'AppDependencyProbingPathsSecrets') {
-            $getAppDependencyProbingPathsSecrets = $true
+        if ($secret -eq 'AppDependencySecrets') {
+            $getAppDependencySecrets = $true
         }
         else {
             $secretName = $secret
@@ -61,12 +61,19 @@ try {
         }
     }
 
-    # Loop through appDependencyProbingPaths and add secrets to the collection of secrets to get
-    if ($getAppDependencyProbingPathsSecrets -and $settings.Keys -contains 'appDependencyProbingPaths') {
-        foreach($appDependencyProbingPath in $settings.appDependencyProbingPaths) {
-            if ($appDependencyProbingPath.PsObject.Properties.name -eq "AuthTokenSecret") {
-                if ($secretsCollection -notcontains $appDependencyProbingPath.authTokenSecret) {
-                    $secretsCollection += $appDependencyProbingPath.authTokenSecret
+    if ($getAppDependencySecrets) {
+        # Loop through appDependencyProbingPaths and trustedNuGetFeeds and add secrets to the collection of secrets to get
+        $settingsCollection = @()
+        if ($settings.Keys -contains 'appDependencyProbingPaths') {
+            $settingsCollection += $settings.appDependencyProbingPaths
+        }
+        if ($settings.Keys -contains 'trustedNuGetFeeds') {
+            $settingsCollection += $settings.trustedNuGetFeeds
+        }
+        foreach($settingsItem in $settingsCollection) {
+            if ($settingsItem.PsObject.Properties.name -eq "AuthTokenSecret") {
+                if ($secretsCollection -notcontains $settingsItem.authTokenSecret) {
+                    $secretsCollection += $settingsItem.authTokenSecret
                 }
             }
         }

Actions/RunPipeline/RunPipeline.ps1

@@ -118,6 +118,38 @@ try {
         exit
     }
 
+    if ($bcContainerHelperConfig.ContainsKey('TrustedNuGetFeeds')) {
+        Write-Host "Reading TrustedNuGetFeeds"
+        foreach($trustedNuGetFeed in $bcContainerHelperConfig.TrustedNuGetFeeds) {
+            if ($trustedNuGetFeed.PSObject.Properties.Name -eq 'Token') {
+                if ($trustedNuGetFeed.Token -ne '') {
+                    OutputWarning -message "Auth token for NuGet feed is defined in settings. This is not recommended. Use a secret instead and specify the secret name in the AuthTokenSecret property"
+                }
+            }
+            else {
+                $trustedNuGetFeed | Add-Member -MemberType NoteProperty -Name 'Token' -Value ''
+            }
+            if ($trustedNuGetFeed.PSObject.Properties.Name -eq 'AuthTokenSecret' -and $trustedNuGetFeed.AuthTokenSecret) {
+                $authTokenSecret = $trustedNuGetFeed.AuthTokenSecret
+                if ($secrets.Keys -notcontains $authTokenSecret) {
+                    OutputWarning -message "Secret $authTokenSecret needed for trusted NuGetFeeds cannot be found"
+                }
+                else {
+                    $trustedNuGetFeed.Token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($secrets."$authTokenSecret"))
+                }
+            }
+        }
+    }
+    else {
+        $bcContainerHelperConfig.TrustedNuGetFeeds = @()
+    }
+    if ($settings.trustMicrosoftNuGetFeeds) {
+        $bcContainerHelperConfig.TrustedNuGetFeeds += @([PSCustomObject]@{
+            "url" = "https://dynamicssmb2.pkgs.visualstudio.com/DynamicsBCPublicFeeds/_packaging/AppSourceSymbols/nuget/v3/index.json"
+            "token" = ''
+        })
+    }
+
     $installApps = $settings.installApps
     $installTestApps = $settings.installTestApps
 
@@ -276,8 +308,13 @@ try {
         }
     }
 
-    if ($gitHubPackagesContext -and ($runAlPipelineParams.Keys -notcontains 'InstallMissingDependencies')) {
-        $gitHubPackagesCredential = $gitHubPackagesContext | ConvertFrom-Json
+    if ((($bcContainerHelperConfig.ContainsKey('TrustedNuGetFeeds') -and ($bcContainerHelperConfig.TrustedNuGetFeeds.Count -gt 0)) -or ($gitHubPackagesContext)) -and ($runAlPipelineParams.Keys -notcontains 'InstallMissingDependencies')) {
+        if ($githubPackagesContext) {
+            $gitHubPackagesCredential = $gitHubPackagesContext | ConvertFrom-Json
+        }
+        else {
+            $gitHubPackagesCredential = [PSCustomObject]@{ "serverUrl" = ''; "token" = '' }
+        }
         $runAlPipelineParams += @{
             "InstallMissingDependencies" = {
                 Param([Hashtable]$parameters)
@@ -289,7 +326,7 @@ try {
                     $publishParams = @{
                         "nuGetServerUrl" = $gitHubPackagesCredential.serverUrl
                         "nuGetToken" = $gitHubPackagesCredential.token
-                        "packageName" = "AL-Go-$appId"
+                        "packageName" = $appId
                         "version" = $version
                     }
                     if ($parameters.ContainsKey('CopyInstalledAppsToFolder')) {
@@ -298,12 +335,11 @@ try {
                         }
                     }
                     if ($parameters.ContainsKey('containerName')) {
-                        Publish-BcNuGetPackageToContainer -containerName $parameters.containerName -tenant $parameters.tenant -skipVerification @publishParams
+                        Publish-BcNuGetPackageToContainer -containerName $parameters.containerName -tenant $parameters.tenant -skipVerification -appSymbolsFolder $parameters.appSymbolsFolder @publishParams -ErrorAction SilentlyContinue
                     }
                     else {
-                        Copy-BcNuGetPackageToFolder -appSymbolsFolder $parameters.appSymbolsFolder @publishParams
+                        Download-BcNuGetPackageToFolder -folder $parameters.appSymbolsFolder @publishParams | Out-Null
                     }
-
                 }
             }
         }