Submodules support (#1346)

Fixes #1344

This PR removes the prior implementation of git submodules support and
replaces it with a version, which is optional.

In .github/AL-Go-Settings.json you can add the following settings:

  "useGitSubmodules": "recursive",
  "gitSubmodulesTokenSecretName": "ghTokenWorkflow"

to enable submodules support and use the ghTokenWorkflow secret for
checking out submodules.

If the GitSubmodulesTokenSecretName is not specified, AL-Go will look
for a secret called submodulesToken.
If a secret called submodulesToken is not provided, AL-Go will use the
github.token.
 
TODO:

- [x] Release Notes
- [x] Update secrets.md
- [x] Update settings.md

---------

Co-authored-by: freddydk <freddydk@users.noreply.github.com>
Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com>

Actions/AL-Go-Helper.ps1

@@ -666,6 +666,8 @@ function ReadSettings {
             "Account"                                   = ""
             "CertificateProfile"                        = ""
         }
+        "useGitSubmodules"                              = "false"
+        "gitSubmodulesTokenSecretName"                  = "gitSubmodulesToken"
     }
 
     # Read settings from files and merge them into the settings object

Actions/ReadSecrets/ReadSecrets.ps1

@@ -37,7 +37,7 @@ try {
             # If we are using the ghTokenWorkflow for commits, we need to get ghTokenWorkflow secret
             $secret = 'ghTokenWorkflow'
         }
-        $secretNameProperty = "$($secret.TrimStart('*'))SecretName"
+        $secretNameProperty = "$($secret.TrimStart('-*'))SecretName"
         if ($secret -eq 'AppDependencySecrets') {
             $getAppDependencySecrets = $true
         }
@@ -83,9 +83,11 @@ try {
     foreach($secret in @($secretsCollection)) {
         $secretSplit = $secret.Split('=')
         $secretsProperty = $secretSplit[0]
-        # Secret names preceded by an asterisk are returned encrypted (and base64 encoded)
-        $secretsPropertyName = $secretsProperty.TrimStart('*')
-        $encrypted = $secretsProperty.StartsWith('*')
+        # Secret names preceded by an asterisk are returned encrypted (and base64 encoded unless...)
+        # Secret names preceded by a minus are not base64 encoded
+        $secretsPropertyName = $secretsProperty.TrimStart('-*')
+        $encrypted = $secretsProperty.TrimStart('-').StartsWith('*')
+        $base64encoded = !($secretsProperty.TrimStart('*').StartsWith('-'))
         $secretName = $secretsPropertyName
         if ($secretSplit.Count -gt 1) {
             $secretName = $secretSplit[1]
@@ -120,20 +122,28 @@ try {
                         }
                     }
                 }
-                $base64value = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($secretValue))
-                $outSecrets += @{ "$secretsProperty" = $base64value }
+                if ($base64encoded) {
+                    Write-Host "Base64 encode secret"
+                    $secretValue = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($secretValue))
+                }
+                $outSecrets += @{ "$secretsPropertyName" = $secretValue }
                 Write-Host "$($secretsPropertyName) successfully read from secret $secretName"
                 $secretsCollection.Remove($secret)
             }
+            elseif ($secretsPropertyName -eq 'gitSubmodulesToken') {
+                Write-Host "Using GitHub token for gitSubmodulesToken"
+                $outSecrets += @{ "$secretsPropertyName" = GetGithubSecret -SecretName 'github_token' }
+            }
         }
     }
 
     if ($secretsCollection) {
         $unresolvedSecrets = ($secretsCollection | ForEach-Object {
             $secretSplit = @($_.Split('='))
             $secretsProperty = $secretSplit[0]
-            # Secret names preceded by an asterisk are returned encrypted (and base64 encoded)
-            $secretsPropertyName = $secretsProperty.TrimStart('*')
+            # Secret names preceded by an asterisk are returned encrypted (and base64 encoded unless...)
+            # Secret names preceded by a minus are not base64 encoded
+            $secretsPropertyName = $secretsProperty.TrimStart('-*')
             if ($secretSplit.Count -eq 1 -or ($secretSplit[1] -eq '')) {
                 $secretsPropertyName
             }

RELEASENOTES.md

@@ -1,6 +1,15 @@
 ### Issues
 
 - Issue 1296 Make property "appFolders" optional
+- Issue 1344 Experimental feature "git submodules" seems to be a breaking change
+
+### New Repository Settings
+
+- `useGitSubmodules` can be either `true` or `recursive` if you want to enable Git Submodules in your repository. If your Git submodules resides in a private repository, you need to create a secret called `gitSubmodulesToken` containing a PAT with access to the submodule repositories. Like with all other secrets, you can also create a setting called `gitSubmodulesTokenSecretName` and specify the name of another secret, with these permissions (f.ex. ghTokenWorkflow).
+
+### Support for Git submodules
+
+In v6.1 we added experimental support for Git submodules - this did however only work if the submodules was in a public repository. In this version, you can use the `useGitSubmodules` setting to control whether you want to use Git Submodules and the `gitSubmodulesToken` secret to allow permission to read these repositories.
 
 ## v6.1
 

Scenarios/secrets.md

@@ -55,7 +55,9 @@ If your GitHub organization might have many organizational secrets, please only
 
 # Secrets
 
-## <a id="Azure_Credentials"></a>**Azure_Credentials** -> Connect to Azure
+<a id="Azure_Credentials"></a>
+
+## **Azure_Credentials** -> Connect to Azure
 
 By creating a secret called Azure_Credentials you can give your GitHub repository access to an Azure Key Vault, from which you can read secrets and use for managed signing of your apps. You can use a managed identity or an app registration (service to service) for authentication.
 
@@ -87,7 +89,9 @@ Example: `{"keyVaultName":"MyKeyVault","clientId":"<clientId>","clientSecret":"<
 
 With this setup, you can create a setting called `keyVaultCodesignCertificateName` containing the name of the imported certificate in your Key Vault in order for AL-Go for GitHub to sign your apps.
 
-## <a id="AuthContext"></a>**AuthContext** -> Deploy to an environment
+<a id="AuthContext"></a>
+
+## **AuthContext** -> Deploy to an environment
 
 Whenever AL-Go for GitHub is doing to deploy to an environment, it will need an AuthContext secret. The AuthContext secret can be provided underneath the environment in GitHub. If you are using a private repository in the free GitHub plan, you do not have environments. Then you can create an AuthContext secret in the repository. If you have multiple environments, you can create different AuthContext secrets by using the environment name followed by an underscore and AuthContext (f.ex. **QA_AuthContext**).
 
@@ -119,7 +123,9 @@ Under Certificates & Secrets in the app registration, you can create a Client Se
 
 Example: `{"tenantId":"<tenantId>","scopes":"https://api.businesscentral.dynamics.com/","clientId":"<clientId>","clientSecret":"<clientSecret>"}`
 
-## <a id="AppSourceContext"></a>**AppSourceContext** -> Deliver to AppSource
+<a id="AppSourceContext"></a>
+
+## **AppSourceContext** -> Deliver to AppSource
 
 Adding a secret called AppSourceContext to an AL-Go for GitHub repository from the AppSource template, enables automatic delivery to AppSource.
 
@@ -143,7 +149,9 @@ Under Certificates & Secrets in the app registration, you can create a Client Se
 
 Example: `{"tenantId":"c645f7e7-0613-4b82-88ca-71f3dbb40045","scopes":"https://api.partner.microsoft.com/.default","clientId":"<clientId>","clientSecret":"<clientSecret>"}`
 
-## <a id="StorageContext"></a>**StorageContext** -> Deliver to storage
+<a id="StorageContext"></a>
+
+## **StorageContext** -> Deliver to storage
 
 Adding a secret called StorageContext to an AL-Go for GitHub repository, enables automatic delivery to an Azure storage account.
 
@@ -179,14 +187,30 @@ Using storageAccount Name and Key is by far the most unsecure way of authenticat
 
 Example: `{"storageAccountName":"MyStorageName","storageAccountKey":"<storageAccountKey>","containerName":"{project}","blobName":"{version}/{project}-{type}.zip"} `
 
-## <a id="GitHubPackagesContext"></a>**GitHubPackagesContext** -> Deliver to GitHub Packages
+<a id="GitHubPackagesContext"></a>
+
+## **GitHubPackagesContext** -> Deliver to GitHub Packages
 
 If you create a secret called GitHubPackagesContext, then AL-Go for GitHub will automagically deliver apps to this NuGet feed after every successful build. AL-Go for GitHub will also use this NuGet feed for dependency resolution when building apps, giving you automatic dependency resolution within all your repositories sharing this secret.
 
 Example: `{"token":"<gitHubToken>","serverUrl":"https://nuget.pkg.github.com/mygithuborg/index.json"}`
 
-## <a id="NuGetContext"></a>**NuGetContext** -> Deliver to NuGet
+<a id="NuGetContext"></a>
+
+## **NuGetContext** -> Deliver to NuGet
 
 If you create a secret called NuGetContext, then AL-Go for GitHub will automagically deliver apps to this NuGet feed after every successful build. AL-Go for GitHub will NOT use this NuGet feed for dependency resolution when building apps. If you want to use this feed for dependency resolution as well, you need to add this to the [trustedNuGetFeeds](https://aka.ms/algosettings#trustedNuGetFeeds) setting.
 
 Example: `{"token":"<nuGetToken>","serverUrl":"https://pkgs.dev.azure.com/myorg/apps/_packaging/myrepo/nuget/v3/index.json"}`
+
+<a id="GhTokenWorkflow"></a>
+
+## **GhTokenWorkflow** -> Update AL-Go System Files
+
+When running the Update AL-Go System Files workflow, you will need a secret called `GhTokenWorkflow`, which should be a personal access token with with permissions to modify Workflows. Read [this](UpdateAlGoSystemFiles.md) for more information.
+
+<a id="GitSubmodulesToken"></a>
+
+## **GitSubmodulesToken** -> Use Git Submodules from private repositories
+
+When using Git submodules from private repositories, the `GitSubmodulesToken` should be a personal access token with permissions to read these repositories. You can also define a setting called `gitSubmodulesTokenSecretName` with the name of a secret to use instead (f.ex. GhTokenWorkflow).

Scenarios/settings.md

@@ -78,6 +78,7 @@ The repository settings are only read from the repository settings file (.github
 | <a id="CICDSchedule"></a>CICDSchedule | CRON schedule for when CI/CD workflow should run. Default is no scheduled run, only manually triggered or triggered by Push or Pull Request. Build your CRON string here: [https://crontab.guru](https://crontab.guru). You need to run the Update AL-Go System Files workflow for the schedule to take effect. **Note:** If you configure a CICDSchedule, AL-Go will stop triggering CICDs on push unless you have also added CICDPushBranches to your settings. |
 | <a id="UpdateGitHubGoSystemFilesSchedule"></a>UpdateGitHubGoSystemFilesSchedule | CRON schedule for when Update AL-Go System Files should run. When Update AL-Go System Files runs on a schedule, it uses direct Commit instead of creating a PR. Default is no scheduled run, only manual trigger. Build your CRON string here: [https://crontab.guru](https://crontab.guru). You need to run the Update AL-Go System Files workflow for the schedule to take effect. |
 | <a id="buildModes"></a>buildModes | A list of build modes to use when building the AL-Go projects. Every AL-Go project will be built using each build mode. AL-Go ships with the following build modes out of the box:<br /> **Default**: Apps are compiled as they are in the source code.<br />**Clean**: _PreprocessorSymbols_ are enabled when compiling the apps. The values for the symbols correspond to the `cleanModePreprocessorSymbols` setting of the AL-Go project.<br />**Translated**: `TranslationFile` compiler feature is enabled when compiling the apps.<br /><br />It is also possible to specify custom build modes by adding a build mode that is different than 'Default', 'Clean' or 'Translated'. |
+| <a id="useGitSubmodules"></a>useGitSubmodules | If your repository is using Git Submodules, you can set the `useGitSubmodules` setting to `"true"` or `"recursive"` in order to use these submodules during build workflows. If `useGitSubmodules` is not set, git submodules are not initialized. If the submodules reside in private repositories, you need to define a `gitSubmodulesToken` secret. Read [this](https://aka.ms/algosecrets#gitSubmodulesToken) for more information. |
 
 ## Advanced settings
 

Templates/AppSource App/.github/workflows/CICD.yaml

@@ -53,7 +53,6 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           lfs: true
-          submodules: recursive
 
       - name: Initialize the workflow
         id: init
@@ -66,7 +65,24 @@ jobs:
         uses: microsoft/AL-Go-Actions/ReadSettings@main
         with:
           shell: powershell
-          get: type, powerPlatformSolutionFolder
+          get: type,powerPlatformSolutionFolder,useGitSubmodules
+
+      - name: Read submodules token
+        id: ReadSubmodulesToken
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: microsoft/AL-Go-Actions/ReadSecrets@main
+        with:
+          shell: powershell
+          gitHubSecrets: ${{ toJson(secrets) }}
+          getSecrets: '-gitSubmodulesToken'
+
+      - name: Checkout Submodules
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          lfs: true
+          submodules: ${{ env.useGitSubmodules }}
+          token: '${{ fromJson(steps.ReadSubmodulesToken.outputs.Secrets).gitSubmodulesToken }}'
 
       - name: Determine Workflow Depth
         id: DetermineWorkflowDepth

Templates/AppSource App/.github/workflows/Current.yaml

@@ -37,7 +37,6 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           lfs: true
-          submodules: recursive
 
       - name: Initialize the workflow
         id: init
@@ -50,6 +49,24 @@ jobs:
         uses: microsoft/AL-Go-Actions/ReadSettings@main
         with:
           shell: powershell
+          get: useGitSubmodules
+
+      - name: Read submodules token
+        id: ReadSubmodulesToken
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: microsoft/AL-Go-Actions/ReadSecrets@main
+        with:
+          shell: powershell
+          gitHubSecrets: ${{ toJson(secrets) }}
+          getSecrets: '-gitSubmodulesToken'
+
+      - name: Checkout Submodules
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          lfs: true
+          submodules: ${{ env.useGitSubmodules }}
+          token: '${{ fromJson(steps.ReadSubmodulesToken.outputs.Secrets).gitSubmodulesToken }}'
 
       - name: Determine Workflow Depth
         id: DetermineWorkflowDepth

Templates/AppSource App/.github/workflows/NextMajor.yaml

@@ -37,7 +37,6 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           lfs: true
-          submodules: recursive
 
       - name: Initialize the workflow
         id: init
@@ -50,6 +49,24 @@ jobs:
         uses: microsoft/AL-Go-Actions/ReadSettings@main
         with:
           shell: powershell
+          get: useGitSubmodules
+
+      - name: Read submodules token
+        id: ReadSubmodulesToken
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: microsoft/AL-Go-Actions/ReadSecrets@main
+        with:
+          shell: powershell
+          gitHubSecrets: ${{ toJson(secrets) }}
+          getSecrets: '-gitSubmodulesToken'
+
+      - name: Checkout Submodules
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          lfs: true
+          submodules: ${{ env.useGitSubmodules }}
+          token: '${{ fromJson(steps.ReadSubmodulesToken.outputs.Secrets).gitSubmodulesToken }}'
 
       - name: Determine Workflow Depth
         id: DetermineWorkflowDepth

Templates/AppSource App/.github/workflows/NextMinor.yaml

@@ -37,7 +37,6 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           lfs: true
-          submodules: recursive
 
       - name: Initialize the workflow
         id: init
@@ -50,6 +49,24 @@ jobs:
         uses: microsoft/AL-Go-Actions/ReadSettings@main
         with:
           shell: powershell
+          get: useGitSubmodules
+
+      - name: Read submodules token
+        id: ReadSubmodulesToken
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: microsoft/AL-Go-Actions/ReadSecrets@main
+        with:
+          shell: powershell
+          gitHubSecrets: ${{ toJson(secrets) }}
+          getSecrets: '-gitSubmodulesToken'
+
+      - name: Checkout Submodules
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          lfs: true
+          submodules: ${{ env.useGitSubmodules }}
+          token: '${{ fromJson(steps.ReadSubmodulesToken.outputs.Secrets).gitSubmodulesToken }}'
 
       - name: Determine Workflow Depth
         id: DetermineWorkflowDepth

Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml

@@ -91,14 +91,13 @@ jobs:
         with:
           ref: ${{ inputs.checkoutRef }}
           lfs: true
-          submodules: recursive
 
       - name: Read settings
         uses: microsoft/AL-Go-Actions/ReadSettings@main
         with:
           shell: ${{ inputs.shell }}
           project: ${{ inputs.project }}
-          get: useCompilerFolder,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,artifact,generateDependencyArtifact,trustedSigning
+          get: useCompilerFolder,keyVaultCodesignCertificateName,doNotSignApps,doNotRunTests,artifact,generateDependencyArtifact,trustedSigning,useGitSubmodules
 
       - name: Read secrets
         id: ReadSecrets
@@ -107,7 +106,16 @@ jobs:
         with:
           shell: ${{ inputs.shell }}
           gitHubSecrets: ${{ toJson(secrets) }}
-          getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS'
+          getSecrets: '${{ inputs.secrets }},appDependencySecrets,AZURE_CREDENTIALS,-gitSubmodulesToken'
+
+      - name: Checkout Submodules
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ inputs.checkoutRef }}
+          lfs: true
+          submodules: ${{ env.useGitSubmodules }}
+          token: '${{ fromJson(steps.ReadSecrets.outputs.Secrets).gitSubmodulesToken }}'
 
       - name: Determine ArtifactUrl
         uses: microsoft/AL-Go-Actions/DetermineArtifactUrl@main

Templates/Per Tenant Extension/.github/workflows/CICD.yaml

@@ -53,7 +53,6 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           lfs: true
-          submodules: recursive
 
       - name: Initialize the workflow
         id: init
@@ -66,7 +65,24 @@ jobs:
         uses: microsoft/AL-Go-Actions/ReadSettings@main
         with:
           shell: powershell
-          get: type, powerPlatformSolutionFolder
+          get: type,powerPlatformSolutionFolder,useGitSubmodules
+
+      - name: Read submodules token
+        id: ReadSubmodulesToken
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: microsoft/AL-Go-Actions/ReadSecrets@main
+        with:
+          shell: powershell
+          gitHubSecrets: ${{ toJson(secrets) }}
+          getSecrets: '-gitSubmodulesToken'
+
+      - name: Checkout Submodules
+        if: env.useGitSubmodules != 'false' && env.useGitSubmodules != ''
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          lfs: true
+          submodules: ${{ env.useGitSubmodules }}
+          token: '${{ fromJson(steps.ReadSubmodulesToken.outputs.Secrets).gitSubmodulesToken }}'
 
       - name: Determine Workflow Depth
         id: DetermineWorkflowDepth