Introduce new e2e testing scaffolding (#492)

e2e scaffodling using Microsoft playwright as the test runner.

.github/workflows/e2e-test.yml

@@ -0,0 +1,79 @@
+name: e2e test suite
+
+on:
+  push:
+    branches:
+      - master
+
+env:
+  GITHUB_TOKEN: ${{ github.token }}
+
+jobs:
+  check_commit:
+    runs-on: ubuntu-latest
+    outputs:
+      skip: ${{ steps.check.outputs.skip }}
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    - name: Check commit message
+      id: check
+      run: |
+        MESSAGE=$(git log --format=%B -n 1 ${{ github.event.after }})
+        if [[ "$MESSAGE" == *"[SKIP TEST]"* ]]; then
+          echo "Commit message contains [SKIP TEST]. We will skip running test."
+          echo "::set-output name=skip::true"
+        else
+          echo "::set-output name=skip::false"
+        fi
+  e2e-test-run:
+    needs: check_commit
+    if: needs.check_commit.outputs.skip == 'false'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18
+    - name: Set file/directory permissions
+      run: |
+        chmod 777 trust uploads modules templates_c cache mainfile.php ~/work/formulize/formulize
+    - name: Run Docker Compose up
+      run: |
+        docker-compose up -d
+    - name: Wait for mysql
+      uses: smurfpandey/wait-for-it@main
+      with:
+        host: localhost
+        port: 3306
+        timeout: 60
+    # - name: Connect to saucelabs
+    #   uses: saucelabs/sauce-connect-action@v2
+    #   with:
+    #     username: ${{ secrets.SAUCE_USERNAME }}
+    #     accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+    #     tunnelName: ${{ github.run_id }}
+    # - name: Saucectl RUN
+    #   uses: saucelabs/saucectl-run-action@v4
+    #   with:
+    #     sauce-username: ${{ secrets.SAUCE_USERNAME }}
+    #     sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }}
+    #     tunnel-name: ${{ github.run_id }}
+    - name: Install dependencies
+      run: npm ci
+      working-directory: tests/e2e
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+      working-directory: tests/e2e
+    - name: Run Playwright tests
+      run: npx playwright test
+      working-directory: tests/e2e
+    - uses: actions/upload-artifact@v4
+      if: ${{ !cancelled() }}
+      with:
+        name: playwright-report
+        path: tests/e2e/test-report/
+        retention-days: 30
+    # - name: Setup tmate session
+    #   uses: mxschmitt/action-tmate@v3

.gitignore

@@ -14,4 +14,7 @@ formulize-docs/_site
 .DS_Store
 libraries/php-saml/settings.php
 _site
-install.lock
+install.lock
+tests/e2e/test-results
+tests/e2e/test-report
+tests/e2e/node_modules

.sauce/config.yml

@@ -0,0 +1,44 @@
+apiVersion: v1alpha
+kind: playwright
+sauce:
+  region: us-west-1
+  concurrency: 10 # Controls how many suites are executed at the same time.
+  metadata:
+    tags:
+      - e2e
+      - release team
+      - other tag
+playwright:
+  version: package.json # See https://docs.saucelabs.com/web-apps/automated-testing/playwright/#supported-testing-platforms for a list of supported versions.
+  configFile: playwright.config.js # See https://docs.saucelabs.com/web-apps/automated-testing/playwright/yaml/#configfile for a list of supported configuration files.
+# Controls what files are available in the context of a test run (unless explicitly excluded by .sauceignore).
+rootDir: ./tests/e2e
+suites:
+  - name: "Firefox Win"
+    platformName: "Windows 11"
+    screenResolution: "1440x900"
+    testMatch: ['.*.js']
+    params:
+      browserName: "firefox"
+      project: "firefox"  # Runs the project that's defined in `playwright.config.js`
+  # - name: "Chromium Mac"
+  #   platformName: "macOS 12"
+  #   screenResolution: "1440x900"
+  #   testMatch: ['.*.js']
+  #   params:
+  #     browserName: "chromium"
+  #     project: "chromium"
+  # - name: "Webkit Win"
+  #   platformName: "Windows 11"
+  #   screenResolution: "1440x900"
+  #   testMatch: ['.*.js']
+  #   params:
+  #     browserName: "webkit"
+  #     project: "webkit"
+# Controls what artifacts to fetch when the suites have finished.
+# artifacts:
+#   download:
+#     when: always
+#     match:
+#       - console.log
+#     directory: ./artifacts/

.sauceignore

@@ -0,0 +1,5 @@
+# Ignore all files by default.
+/*
+
+# Re-include files we selectively want as part of the payload by prefixing the lines with '!'.
+!/tests/e2e

tests/README.md

@@ -0,0 +1,24 @@
+# Testing
+
+## e2e testings
+
+We use the [Playwright](https://playwright.dev/) framework for configuring and executing e2e tests across browsers.
+
+Core formulize e2e test are located in the `/tests/e2e/formulize-core` directory.
+
+When writing your own tests for your site/application place them in an appropriately named directory inside `/tests/e2e`.
+
+### Running tests in CI
+
+e2e tests are automatically run in CI with a merge to the master branch as specifed in the `.github/workflows/e2e-test.yml` file. If you are performing a small merge to master and don't want the tests to execute then add `[SKIP TEST]` to your commit message.
+
+### Running core tests locally
+
+#### Requirements
+* Node.js (18+)
+* Docker (optional)
+
+#### Steps
+1. Get a local instance of the application set up which is accessible at `http://localhost:8080`. We recommend using the included docker-compose file to quickly spin up an environment using docker.
+2. Navigate to the tests folder `/tests/e2e` and perform an `npm install`
+3. While still in the `/tests/e2e` directory run the test with `npm t` This will run the tests in headless mode. If you'd like to have the browser load to watch the progress use `npm run test:debug`.

tests/e2e/formulize-core/001-install-formulize.spec.js

@@ -0,0 +1,71 @@
+const { test, expect } = require('@playwright/test')
+
+test.describe('Installer', () => {
+	test('Installation of Formulize', async ({ page }) => {
+		await page.goto('/install/index.php');
+  	// Welcome page
+		await expect(page.locator('h2')).toContainText('Welcome to the Formulize installation assistant');
+		await page.getByRole('button', { name: 'Next' }).click();
+  	// Check server configuration
+		await expect(page.locator('h2')).toContainText('Checking your server configuration');
+		await page.getByRole('button', { name: 'Next' }).click();
+		// Path settings
+		await expect(page.locator('h2')).toContainText('Paths settings');
+  	await page.getByLabel('ImpressCMS physical trust path').fill('/var/www/html/trust');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// Database connection
+		await expect(page.locator('h2')).toContainText('Database connection');
+  	await page.getByLabel('Server hostname').fill('mariadb');
+  	await page.getByLabel('User name').fill('user');
+  	await page.getByLabel('Password').fill('password');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// Database configuration
+		await expect(page.locator('h2')).toContainText('Database configuration');
+  	await page.getByLabel('Database name').fill('formulize');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// Saving configuration
+		await expect(page.locator('h2')).toContainText('Saving your system configuration');
+  	await expect(page.getByRole('paragraph')).toContainText('The installer is now ready to save the specified settings to mainfile.php.Press next to proceed.');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// DB table creation
+		await expect(page.locator('h2')).toContainText('Database tables creation');
+		await expect(page.getByRole('paragraph')).toContainText('No ImpressCMS tables were detected.The installer is now ready to create the ImpressCMS system tables.Press next to proceed.');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// DB table creation results
+		await expect(page.locator('h2')).toContainText('Database tables creation');
+		await expect(page.locator('#tablescreate')).toContainText('_avatar created.');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// Initial settings
+		await expect(page.locator('h2')).toContainText('Please enter your initial settings');
+		await page.getByLabel('Admin Display Name').fill('admin');
+		await page.getByLabel('Admin login').fill('admin');
+		await page.getByLabel('Admin e-mail').fill('formulize@example.com');
+		await page.getByLabel('Admin password').fill('password');
+		await page.getByLabel('Confirm password').fill('password');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// Saving to DB
+		await expect(page.locator('h2')).toContainText('Saving your settings to the database');
+		await expect(page.getByRole('paragraph')).toContainText('The installer is now ready to insert initial data into your database.THIS COULD TAKE A REALLY LONG TIME DEPENDING ON YOUR SERVER SOFTWARE AND CONFIGURATION!');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// DB saving results
+		await expect(page.locator('h2')).toContainText('Saving your settings to the database');
+		await expect(page.locator('#tablesfill')).toContainText('1 entries inserted to table');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// Install modules
+		await expect(page.locator('h2')).toContainText('Installation of modules');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// Install modules results
+		await expect(page.locator('h2')).toContainText('Installation of modules');
+		// await expect(page.locator('#modulesinstall')).toContainText('Module Content installed successfully');
+		// await expect(page.locator('#modulesinstall')).toContainText('Module Profile installed successfully.');
+		// await expect(page.locator('#modulesinstall')).toContainText('Module Forms installed successfully');
+		// await expect(page.locator('#modulesinstall')).toContainText('Module Protector installed successfully');
+  	await page.getByRole('button', { name: 'Next' }).click();
+		// Install finished
+		await expect(page.locator('h2')).toContainText('Installation completed');
+		await expect(page.getByRole('button', { name: 'Show my site' })).toBeVisible();
+  	await page.getByRole('button', { name: 'Show my site' }).click();
+		// Running instance
+		await expect(page.locator('#main-logo').getByRole('link', { name: 'Logo image' })).toBeVisible();
+	})
+})

tests/e2e/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "e2e-tests",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "playwright test",
+    "test:debug": "playwright test --headed --timeout=0",
+    "test:core": "playwright test formulize-core"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@playwright/test": "1.41.0",
+    "@types/node": "^20.11.19"
+  }
+}