Add Page Scripting support to AL-Go for GitHub - part 1 (#1247)

This PR adds pagescripting functionality to CI/CD in AL-Go for GitHub.

### CI/CD

In CI/CD you will be able to define which page scripting tests you want
to execute after a successful build to verify the build. This is
controlled by 3 project settings:
- `pageScriptingTests` should be an array of page scripting test file
specifications, relative to the AL-Go project. Examples of file
specifications: `recordings/my*.yml` (for all yaml files in the
recordings subfolder matching my*.yml), `recordings` (for all *.yml
files in the recordings subfolder) or `recordings/test.yml` (for a
single yml file)
- `doNotRunPageScriptingTests` can force the pipeline to NOT run the
page scripting tests specified in pageScriptingTests. Note this setting
can be set in a [workflow specific settings
file](#where-are-the-settings-located) to only apply to that workflow
- `restoreDatabases` should be an array of events, indicating when you
want to start with clean databases in the container. Possible events
are: `BeforeBcpTests`, `BeforePageScriptingTests`, `BeforeEachTestApp`,
`BeforeEachBcptTestApp`, `BeforeEachPageScriptingTest`

CI/CD (and test) workflows will have new build artifacts created:

- `PageScriptingTestResults` is a JUnit test results file with all
results combined.
- `PageScriptingTestResultDetails` are the detailed test results
(including videos) when any of the page scripting tests have failures.
If the page scripting tests succeed - the details are not published.

### TO:DO
- [x] Update release notes
- [x] Update settings documentation
- [x] Revert to BcContainerHelper preview in AL-Go-Helper.ps1

### What's next

Workshop scenarios

Part 2 is adding a visual results viewer - this requires injecting a
custom playwright-reporter, which the current bc-replay module doesn't
support yet

Part 3 is adding pagescripting to deployment - allowing people to run
the same or other page scripting tests after deployment to QA
environment. This part could also allow running AL tests and BCPT tests
in QA environments after deployment as a validation of the deployment.

---------

Co-authored-by: freddydk <freddydk@users.noreply.github.com>
Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com>

Actions/AL-Go-Helper.ps1

@@ -581,6 +581,8 @@ function ReadSettings {
         "testDependencies"                              = @()
         "testFolders"                                   = @()
         "bcptTestFolders"                               = @()
+        "pageScriptingTests"                            = @()
+        "restoreDatabases"                              = @()
         "installApps"                                   = @()
         "installTestApps"                               = @()
         "installOnlyReferencedApps"                     = $true
@@ -606,6 +608,7 @@ function ReadSettings {
         "doNotBuildTests"                               = $false
         "doNotRunTests"                                 = $false
         "doNotRunBcptTests"                             = $false
+        "doNotRunPageScriptingTests"                    = $false
         "doNotPublishApps"                              = $false
         "doNotSignApps"                                 = $false
         "configPackages"                                = @()

Actions/CalculateArtifactNames/CalculateArtifactNames.ps1

@@ -42,7 +42,7 @@ else {
     $suffix = "$($settings.repoVersion).$($settings.appBuild).$($settings.appRevision)"
 }
 
-'Apps', 'Dependencies', 'TestApps', 'TestResults', 'BcptTestResults', 'BuildOutput', 'ContainerEventLog', 'PowerPlatformSolution' | ForEach-Object {
+'Apps', 'Dependencies', 'TestApps', 'TestResults', 'BcptTestResults', 'PageScriptingTestResults', 'PageScriptingTestResultDetails', 'BuildOutput', 'ContainerEventLog', 'PowerPlatformSolution' | ForEach-Object {
     $name = "$($_)ArtifactsName"
     $value = "$($projectName)-$($branchName)-$buildMode$_-$suffix"
     Set-OutputVariable -name $name -value $value

Actions/CalculateArtifactNames/README.md

@@ -38,6 +38,8 @@ none
 | TestAppsArtifactsName | Artifacts name for TestApps |
 | TestResultsArtifactsName | Artifacts name for TestResults |
 | BcptTestResultsArtifactsName | Artifacts name for BcptTestResults |
+| PageScriptingTestResultsArtifactsName | Artifacts name for PageScriptingTestResults |
+| PageScriptingTestResultDetailsArtifactsName | Artifacts name for PageScriptingTestResultDetails |
 | BuildOutputArtifactsName | Artifacts name for BuildOutput |
 | ContainerEventLogArtifactsName | Artifacts name for ContainerEventLog |
 | BuildMode | Build mode used when building the artifacts |

Actions/CalculateArtifactNames/action.yaml

@@ -43,6 +43,12 @@ outputs:
   BcptTestResultsArtifactsName:
     description: Artifacts name for BcptTestResults
     value: ${{ steps.calculateartifactnames.outputs.BcptTestResultsArtifactsName }}
+  PageScriptingTestResultsArtifactsName:
+    description: Artifacts name for PageScriptingTestResults
+    value: ${{ steps.calculateartifactnames.outputs.PageScriptingTestResultsArtifactsName }}
+  PageScriptingTestResultDetailsArtifactsName:
+    description: Artifacts name for PageScriptingTestResultDetails
+    value: ${{ steps.calculateartifactnames.outputs.PageScriptingTestResultDetailsArtifactsName }}
   BuildOutputArtifactsName:
     description: Artifacts name for BuildOutput
     value: ${{ steps.calculateartifactnames.outputs.BuildOutputArtifactsName }}

Actions/RunPipeline/RunPipeline.ps1

@@ -353,6 +353,7 @@ try {
     "doNotBuildTests",
     "doNotRunTests",
     "doNotRunBcptTests",
+    "doNotRunPageScriptingTests",
     "doNotPublishApps",
     "installTestRunner",
     "installTestFramework",
@@ -414,6 +415,8 @@ try {
         -appFolders $settings.appFolders `
         -testFolders $settings.testFolders `
         -bcptTestFolders $settings.bcptTestFolders `
+        -pageScriptingTests $settings.pageScriptingTests `
+        -restoreDatabases $settings.restoreDatabases `
         -buildOutputFile $buildOutputFile `
         -containerEventLogFile $containerEventLogFile `
         -testResultsFile $testResultsFile `
@@ -428,6 +431,8 @@ try {
         -additionalCountries $additionalCountries `
         -obsoleteTagMinAllowedMajorMinor $settings.obsoleteTagMinAllowedMajorMinor `
         -buildArtifactFolder $buildArtifactFolder `
+        -pageScriptingTestResultsFile (Join-Path $buildArtifactFolder 'PageScriptingTestResults.xml') `
+        -pageScriptingTestResultsFolder (Join-Path $buildArtifactFolder 'PageScriptingTestResultDetails') `
         -CreateRuntimePackages:$CreateRuntimePackages `
         -appBuild $appBuild -appRevision $appRevision `
         -uninstallRemovedApps

RELEASENOTES.md

@@ -2,6 +2,19 @@
 
 - Issue 1241 Increment Version Number might produce wrong app.json
 
+### Support for Page Scripting Tests
+
+Page Scripting tests are now supported as part of CI/CD. By specifying pageScriptingTests in your project settings file, AL-Go for GitHub will automatically run these page scripting tests as part of your CI/CD workflow, generating the following build artifacts:
+
+- `PageScriptingTestResults` is a JUnit test results file with all results combined.
+- `PageScriptingTestResultDetails` are the detailed test results (including videos) when any of the page scripting tests have failures. If the page scripting tests succeed - the details are not published.
+
+### New Project Settings
+
+- `pageScriptingTests` should be an array of page scripting test file specifications, relative to the AL-Go project. Examples of file specifications: `recordings/my*.yml` (for all yaml files in the recordings subfolder matching my\*.yml), `recordings` (for all \*.yml files in the recordings subfolder) or `recordings/test.yml` (for a single yml file)
+- `doNotRunPageScriptingTests` can force the pipeline to NOT run the page scripting tests specified in pageScriptingTests. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow
+- `restoreDatabases` should be an array of events, indicating when you want to start with clean databases in the container. Possible events are: `BeforeBcpTests`, `BeforePageScriptingTests`, `BeforeEachTestApp`, `BeforeEachBcptTestApp`, `BeforeEachPageScriptingTest`
+
 ## v6.0
 
 ### Issues

Scenarios/settings.md

@@ -34,6 +34,9 @@ When running a workflow or a local script, the settings are applied by reading s
 | <a id="appFolders"></a>appFolders | appFolders should be an array of folders (relative to project root), which contains apps for this project. Apps in these folders are sorted based on dependencies and built and published in that order.<br />If appFolders are not specified, AL-Go for GitHub will try to locate appFolders in the root of the project. | \[ \] |
 | <a id="testFolders"></a>testFolders | testFolders should be an array of folders (relative to project root), which contains test apps for this project. Apps in these folders are sorted based on dependencies and built, published and tests are run in that order.<br />If testFolders are not specified, AL-Go for GitHub will try to locate testFolders in the root of the project. | \[ \] |
 | <a id="bcptTestFolders"></a>bcptTestFolders | bcptTestFolders should be an array of folders (relative to project root), which contains performance test apps for this project. Apps in these folders are sorted based on dependencies and built, published and bcpt tests are run in that order.<br />If bcptTestFolders are not specified, AL-Go for GitHub will try to locate bcptTestFolders in the root of the project. | \[ \] |
+| <a id="pageScriptingTests"></a>pageScriptingTests | pageScriptingTests should be an array of page scripting test file specifications, relative to the AL-Go project. Examples of file specifications: `recordings/my*.yml` (for all yaml files in the recordings subfolder matching my\*.yml), `recordings` (for all \*.yml files in the recordings subfolder) or `recordings/test.yml` (for a single yml file) | \[ \] |
+| <a id="doNotRunpageScriptingTests"></a>doNotRunpageScriptingTests | When true, this setting forces the pipeline to NOT run the page scripting tests specified in pageScriptingTests. Note this setting can be set in a [workflow specific settings file](#where-are-the-settings-located) to only apply to that workflow | false |
+| <a id="restoreDatabases"></a>restoreDatabases | restoreDatabases should be an array of events, indicating when you want to start with clean databases in the container. Possible events are: `BeforeBcpTests`, `BeforePageScriptingTests`, `BeforeEachTestApp`, `BeforeEachBcptTestApp`, `BeforeEachPageScriptingTest` | \[ \] |
 | <a id="appDependencyProbingPaths"></a>appDependencyProbingPaths | Array of dependency specifications, from which apps will be downloaded when the CI/CD workflow is starting. Every dependency specification consists of the following properties:<br />**repo** = repository<br />**version** = version (default latest)<br />**release_status** = latestBuild/release/prerelease/draft (default release)<br />**projects** = projects (default * = all)<br />**branch** = branch (default main)<br />**AuthTokenSecret** = Name of secret containing auth token (default none)<br /> | \[ \] |
 | <a id="cleanModePreprocessorSymbols"></a>cleanModePreprocessorSymbols | List of clean tags to be used in _Clean_ build mode | \[ \] |
 | <a id="bcptThresholds"></a>bcptThresholds | Structure with properties for the thresholds when running performance tests using the Business Central Performance Toolkit.<br />**DurationWarning** = a warning is shown if the duration of a bcpt test degrades more than this percentage (default 10)<br />**DurationError** - an error is shown if the duration of a bcpt test degrades more than this percentage (default 25)<br />**NumberOfSqlStmtsWarning** - a warning is shown if the number of SQL statements from a bcpt test increases more than this percentage (default 5)<br />**NumberOfSqlStmtsError** - an error is shown if the number of SQL statements from a bcpt test increases more than this percentage (default 10)<br />*Note that errors and warnings on the build in GitHub are only issued when a threshold is exceeded on the codeunit level, when an individual operation threshold is exceeded, it is only shown in the test results viewer.* |
@@ -312,6 +315,9 @@ This functionality is also available in AL-Go for GitHub, by adding a file to th
 | GetBcContainerAppRuntimePackage.ps1 | Get the runtime package specified by the $parameters hashtable |
 | RemoveBcContainer.ps1 | Cleanup based on the $parameters hashtable |
 | InstallMissingDependencies | Install missing dependencies |
+| BackupBcContainerDatabases | Backup Databases in container for subsequent restore(s) |
+| RestoreDatabasesInBcContainer | Restore Databases in container |
+| InstallMissingDependencies | Install missing dependencies |
 
 ## BcContainerHelper settings
 

Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml

@@ -235,18 +235,34 @@ jobs:
 
       - name: Publish artifacts - test results
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        if: (success() || failure()) && (hashFiles(format('{0}/TestResults.xml',inputs.project)) != '')
+        if: (success() || failure()) && (hashFiles(format('{0}/.buildartifacts/TestResults.xml',inputs.project)) != '')
         with:
           name: ${{ steps.calculateArtifactsNames.outputs.TestResultsArtifactsName }}
-          path: '${{ inputs.project }}/TestResults.xml'
+          path: '${{ inputs.project }}/.buildartifacts/TestResults.xml'
           if-no-files-found: ignore
 
       - name: Publish artifacts - bcpt test results
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        if: (success() || failure()) && (hashFiles(format('{0}/bcptTestResults.json',inputs.project)) != '')
+        if: (success() || failure()) && (hashFiles(format('{0}/.buildartifacts/bcptTestResults.json',inputs.project)) != '')
         with:
           name: ${{ steps.calculateArtifactsNames.outputs.BcptTestResultsArtifactsName }}
-          path: '${{ inputs.project }}/bcptTestResults.json'
+          path: '${{ inputs.project }}/.buildartifacts/bcptTestResults.json'
+          if-no-files-found: ignore
+
+      - name: Publish artifacts - page scripting test results
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: (success() || failure()) && (hashFiles(format('{0}/.buildartifacts/PageScriptingTestResults.xml',inputs.project)) != '')
+        with:
+          name: ${{ steps.calculateArtifactsNames.outputs.PageScriptingTestResultsArtifactsName }}
+          path: '${{ inputs.project }}/.buildartifacts/PageScriptingTestResults.xml'
+          if-no-files-found: ignore
+
+      - name: Publish artifacts - page scripting test result details
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: (success() || failure())
+        with:
+          name: ${{ steps.calculateArtifactsNames.outputs.PageScriptingTestResultDetailsArtifactsName }}
+          path: '${{ inputs.project }}/.buildartifacts/PageScriptingTestResultDetails/'
           if-no-files-found: ignore
 
       - name: Analyze Test Results

Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml

@@ -235,18 +235,34 @@ jobs:
 
       - name: Publish artifacts - test results
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        if: (success() || failure()) && (hashFiles(format('{0}/TestResults.xml',inputs.project)) != '')
+        if: (success() || failure()) && (hashFiles(format('{0}/.buildartifacts/TestResults.xml',inputs.project)) != '')
         with:
           name: ${{ steps.calculateArtifactsNames.outputs.TestResultsArtifactsName }}
-          path: '${{ inputs.project }}/TestResults.xml'
+          path: '${{ inputs.project }}/.buildartifacts/TestResults.xml'
           if-no-files-found: ignore
 
       - name: Publish artifacts - bcpt test results
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
-        if: (success() || failure()) && (hashFiles(format('{0}/bcptTestResults.json',inputs.project)) != '')
+        if: (success() || failure()) && (hashFiles(format('{0}/.buildartifacts/bcptTestResults.json',inputs.project)) != '')
         with:
           name: ${{ steps.calculateArtifactsNames.outputs.BcptTestResultsArtifactsName }}
-          path: '${{ inputs.project }}/bcptTestResults.json'
+          path: '${{ inputs.project }}/.buildartifacts/bcptTestResults.json'
+          if-no-files-found: ignore
+
+      - name: Publish artifacts - page scripting test results
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: (success() || failure()) && (hashFiles(format('{0}/.buildartifacts/PageScriptingTestResults.xml',inputs.project)) != '')
+        with:
+          name: ${{ steps.calculateArtifactsNames.outputs.PageScriptingTestResultsArtifactsName }}
+          path: '${{ inputs.project }}/.buildartifacts/PageScriptingTestResults.xml'
+          if-no-files-found: ignore
+
+      - name: Publish artifacts - page scripting test result details
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: (success() || failure())
+        with:
+          name: ${{ steps.calculateArtifactsNames.outputs.PageScriptingTestResultDetailsArtifactsName }}
+          path: '${{ inputs.project }}/.buildartifacts/PageScriptingTestResultDetails/'
           if-no-files-found: ignore
 
       - name: Analyze Test Results