From c582b4ba374f29f0e106581181f44082c0225d37 Mon Sep 17 00:00:00 2001 From: Arvin Singla Date: Fri, 3 May 2024 11:23:09 -0400 Subject: [PATCH] Introduce new e2e testing scaffolding (#492) e2e scaffodling using Microsoft playwright as the test runner. --- .github/workflows/e2e-test.yml | 79 ++++++++++++++ .gitignore | 5 +- .sauce/config.yml | 44 ++++++++ .sauceignore | 5 + tests/README.md | 24 +++++ .../001-install-formulize.spec.js | 71 ++++++++++++ tests/e2e/package-lock.json | 91 ++++++++++++++++ tests/e2e/package.json | 18 ++++ tests/e2e/playwright.config.js | 101 ++++++++++++++++++ 9 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e-test.yml create mode 100644 .sauce/config.yml create mode 100644 .sauceignore create mode 100644 tests/README.md create mode 100644 tests/e2e/formulize-core/001-install-formulize.spec.js create mode 100644 tests/e2e/package-lock.json create mode 100644 tests/e2e/package.json create mode 100644 tests/e2e/playwright.config.js diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml new file mode 100644 index 000000000..eea1c6161 --- /dev/null +++ b/.github/workflows/e2e-test.yml @@ -0,0 +1,79 @@ +name: e2e test suite + +on: + push: + branches: + - master + +env: + GITHUB_TOKEN: ${{ github.token }} + +jobs: + check_commit: + runs-on: ubuntu-latest + outputs: + skip: ${{ steps.check.outputs.skip }} + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Check commit message + id: check + run: | + MESSAGE=$(git log --format=%B -n 1 ${{ github.event.after }}) + if [[ "$MESSAGE" == *"[SKIP TEST]"* ]]; then + echo "Commit message contains [SKIP TEST]. We will skip running test." + echo "::set-output name=skip::true" + else + echo "::set-output name=skip::false" + fi + e2e-test-run: + needs: check_commit + if: needs.check_commit.outputs.skip == 'false' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Set file/directory permissions + run: | + chmod 777 trust uploads modules templates_c cache mainfile.php ~/work/formulize/formulize + - name: Run Docker Compose up + run: | + docker-compose up -d + - name: Wait for mysql + uses: smurfpandey/wait-for-it@main + with: + host: localhost + port: 3306 + timeout: 60 + # - name: Connect to saucelabs + # uses: saucelabs/sauce-connect-action@v2 + # with: + # username: ${{ secrets.SAUCE_USERNAME }} + # accessKey: ${{ secrets.SAUCE_ACCESS_KEY }} + # tunnelName: ${{ github.run_id }} + # - name: Saucectl RUN + # uses: saucelabs/saucectl-run-action@v4 + # with: + # sauce-username: ${{ secrets.SAUCE_USERNAME }} + # sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + # tunnel-name: ${{ github.run_id }} + - name: Install dependencies + run: npm ci + working-directory: tests/e2e + - name: Install Playwright Browsers + run: npx playwright install --with-deps + working-directory: tests/e2e + - name: Run Playwright tests + run: npx playwright test + working-directory: tests/e2e + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report + path: tests/e2e/test-report/ + retention-days: 30 + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 diff --git a/.gitignore b/.gitignore index d782c117b..ae8f3e46b 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,7 @@ formulize-docs/_site .DS_Store libraries/php-saml/settings.php _site -install.lock \ No newline at end of file +install.lock +tests/e2e/test-results +tests/e2e/test-report +tests/e2e/node_modules diff --git a/.sauce/config.yml b/.sauce/config.yml new file mode 100644 index 000000000..96807dc2e --- /dev/null +++ b/.sauce/config.yml @@ -0,0 +1,44 @@ +apiVersion: v1alpha +kind: playwright +sauce: + region: us-west-1 + concurrency: 10 # Controls how many suites are executed at the same time. + metadata: + tags: + - e2e + - release team + - other tag +playwright: + version: package.json # See https://docs.saucelabs.com/web-apps/automated-testing/playwright/#supported-testing-platforms for a list of supported versions. + configFile: playwright.config.js # See https://docs.saucelabs.com/web-apps/automated-testing/playwright/yaml/#configfile for a list of supported configuration files. +# Controls what files are available in the context of a test run (unless explicitly excluded by .sauceignore). +rootDir: ./tests/e2e +suites: + - name: "Firefox Win" + platformName: "Windows 11" + screenResolution: "1440x900" + testMatch: ['.*.js'] + params: + browserName: "firefox" + project: "firefox" # Runs the project that's defined in `playwright.config.js` + # - name: "Chromium Mac" + # platformName: "macOS 12" + # screenResolution: "1440x900" + # testMatch: ['.*.js'] + # params: + # browserName: "chromium" + # project: "chromium" + # - name: "Webkit Win" + # platformName: "Windows 11" + # screenResolution: "1440x900" + # testMatch: ['.*.js'] + # params: + # browserName: "webkit" + # project: "webkit" +# Controls what artifacts to fetch when the suites have finished. +# artifacts: +# download: +# when: always +# match: +# - console.log +# directory: ./artifacts/ diff --git a/.sauceignore b/.sauceignore new file mode 100644 index 000000000..7715b16ff --- /dev/null +++ b/.sauceignore @@ -0,0 +1,5 @@ +# Ignore all files by default. +/* + +# Re-include files we selectively want as part of the payload by prefixing the lines with '!'. +!/tests/e2e diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..ad7b8814e --- /dev/null +++ b/tests/README.md @@ -0,0 +1,24 @@ +# Testing + +## e2e testings + +We use the [Playwright](https://playwright.dev/) framework for configuring and executing e2e tests across browsers. + +Core formulize e2e test are located in the `/tests/e2e/formulize-core` directory. + +When writing your own tests for your site/application place them in an appropriately named directory inside `/tests/e2e`. + +### Running tests in CI + +e2e tests are automatically run in CI with a merge to the master branch as specifed in the `.github/workflows/e2e-test.yml` file. If you are performing a small merge to master and don't want the tests to execute then add `[SKIP TEST]` to your commit message. + +### Running core tests locally + +#### Requirements +* Node.js (18+) +* Docker (optional) + +#### Steps +1. Get a local instance of the application set up which is accessible at `http://localhost:8080`. We recommend using the included docker-compose file to quickly spin up an environment using docker. +2. Navigate to the tests folder `/tests/e2e` and perform an `npm install` +3. While still in the `/tests/e2e` directory run the test with `npm t` This will run the tests in headless mode. If you'd like to have the browser load to watch the progress use `npm run test:debug`. diff --git a/tests/e2e/formulize-core/001-install-formulize.spec.js b/tests/e2e/formulize-core/001-install-formulize.spec.js new file mode 100644 index 000000000..9c0fd21d9 --- /dev/null +++ b/tests/e2e/formulize-core/001-install-formulize.spec.js @@ -0,0 +1,71 @@ +const { test, expect } = require('@playwright/test') + +test.describe('Installer', () => { + test('Installation of Formulize', async ({ page }) => { + await page.goto('/install/index.php'); + // Welcome page + await expect(page.locator('h2')).toContainText('Welcome to the Formulize installation assistant'); + await page.getByRole('button', { name: 'Next' }).click(); + // Check server configuration + await expect(page.locator('h2')).toContainText('Checking your server configuration'); + await page.getByRole('button', { name: 'Next' }).click(); + // Path settings + await expect(page.locator('h2')).toContainText('Paths settings'); + await page.getByLabel('ImpressCMS physical trust path').fill('/var/www/html/trust'); + await page.getByRole('button', { name: 'Next' }).click(); + // Database connection + await expect(page.locator('h2')).toContainText('Database connection'); + await page.getByLabel('Server hostname').fill('mariadb'); + await page.getByLabel('User name').fill('user'); + await page.getByLabel('Password').fill('password'); + await page.getByRole('button', { name: 'Next' }).click(); + // Database configuration + await expect(page.locator('h2')).toContainText('Database configuration'); + await page.getByLabel('Database name').fill('formulize'); + await page.getByRole('button', { name: 'Next' }).click(); + // Saving configuration + await expect(page.locator('h2')).toContainText('Saving your system configuration'); + await expect(page.getByRole('paragraph')).toContainText('The installer is now ready to save the specified settings to mainfile.php.Press next to proceed.'); + await page.getByRole('button', { name: 'Next' }).click(); + // DB table creation + await expect(page.locator('h2')).toContainText('Database tables creation'); + await expect(page.getByRole('paragraph')).toContainText('No ImpressCMS tables were detected.The installer is now ready to create the ImpressCMS system tables.Press next to proceed.'); + await page.getByRole('button', { name: 'Next' }).click(); + // DB table creation results + await expect(page.locator('h2')).toContainText('Database tables creation'); + await expect(page.locator('#tablescreate')).toContainText('_avatar created.'); + await page.getByRole('button', { name: 'Next' }).click(); + // Initial settings + await expect(page.locator('h2')).toContainText('Please enter your initial settings'); + await page.getByLabel('Admin Display Name').fill('admin'); + await page.getByLabel('Admin login').fill('admin'); + await page.getByLabel('Admin e-mail').fill('formulize@example.com'); + await page.getByLabel('Admin password').fill('password'); + await page.getByLabel('Confirm password').fill('password'); + await page.getByRole('button', { name: 'Next' }).click(); + // Saving to DB + await expect(page.locator('h2')).toContainText('Saving your settings to the database'); + await expect(page.getByRole('paragraph')).toContainText('The installer is now ready to insert initial data into your database.THIS COULD TAKE A REALLY LONG TIME DEPENDING ON YOUR SERVER SOFTWARE AND CONFIGURATION!'); + await page.getByRole('button', { name: 'Next' }).click(); + // DB saving results + await expect(page.locator('h2')).toContainText('Saving your settings to the database'); + await expect(page.locator('#tablesfill')).toContainText('1 entries inserted to table'); + await page.getByRole('button', { name: 'Next' }).click(); + // Install modules + await expect(page.locator('h2')).toContainText('Installation of modules'); + await page.getByRole('button', { name: 'Next' }).click(); + // Install modules results + await expect(page.locator('h2')).toContainText('Installation of modules'); + // await expect(page.locator('#modulesinstall')).toContainText('Module Content installed successfully'); + // await expect(page.locator('#modulesinstall')).toContainText('Module Profile installed successfully.'); + // await expect(page.locator('#modulesinstall')).toContainText('Module Forms installed successfully'); + // await expect(page.locator('#modulesinstall')).toContainText('Module Protector installed successfully'); + await page.getByRole('button', { name: 'Next' }).click(); + // Install finished + await expect(page.locator('h2')).toContainText('Installation completed'); + await expect(page.getByRole('button', { name: 'Show my site' })).toBeVisible(); + await page.getByRole('button', { name: 'Show my site' }).click(); + // Running instance + await expect(page.locator('#main-logo').getByRole('link', { name: 'Logo image' })).toBeVisible(); + }) +}) diff --git a/tests/e2e/package-lock.json b/tests/e2e/package-lock.json new file mode 100644 index 000000000..8b1aced12 --- /dev/null +++ b/tests/e2e/package-lock.json @@ -0,0 +1,91 @@ +{ + "name": "ci", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ci", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@playwright/test": "1.41.0", + "@types/node": "^20.11.19" + } + }, + "node_modules/@playwright/test": { + "version": "1.41.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.41.0.tgz", + "integrity": "sha512-Grvzj841THwtpBOrfiHOeYTJQxDRnKofMSzCiV8XeyLWu3o89qftQ4BCKfkziJhSUQRd0utKhrddtIsiraIwmw==", + "dev": true, + "dependencies": { + "playwright": "1.41.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@types/node": { + "version": "20.12.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.7.tgz", + "integrity": "sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.41.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.41.0.tgz", + "integrity": "sha512-XOsfl5ZtAik/T9oek4V0jAypNlaCNzuKOwVhqhgYT3os6kH34PzbRb74F0VWcLYa5WFdnmxl7qyAHBXvPv7lqQ==", + "dev": true, + "dependencies": { + "playwright-core": "1.41.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.41.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.41.0.tgz", + "integrity": "sha512-UGKASUhXmvqm2Lxa1fNr8sFwAtqjpgBRr9jQ7XBI8Rn5uFiEowGUGwrruUQsVPIom4bk7Lt+oLGpXobnXzrBIw==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + } + } +} diff --git a/tests/e2e/package.json b/tests/e2e/package.json new file mode 100644 index 000000000..00af15f6a --- /dev/null +++ b/tests/e2e/package.json @@ -0,0 +1,18 @@ +{ + "name": "e2e-tests", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "playwright test", + "test:debug": "playwright test --headed --timeout=0", + "test:core": "playwright test formulize-core" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "1.41.0", + "@types/node": "^20.11.19" + } +} diff --git a/tests/e2e/playwright.config.js b/tests/e2e/playwright.config.js new file mode 100644 index 000000000..12f9a1f95 --- /dev/null +++ b/tests/e2e/playwright.config.js @@ -0,0 +1,101 @@ +// @ts-check +const { defineConfig, devices } = require('@playwright/test'); + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * @see https://playwright.dev/docs/test-configuration + */ +module.exports = defineConfig({ + testDir: './', + outputDir: './test-results', + /* Run tests in files in parallel */ + fullyParallel: false, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: [ + ['html', { outputFolder: './test-report' }] + ], + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: 'http://localhost:8080', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + + /* Video */ + video: 'on' + }, + + expect: { + // Maximum time expect() should wait for the condition to be met. + timeout: 5000, + + toHaveScreenshot: { + // An acceptable amount of pixels that could be different, unset by default. + maxDiffPixels: 10, + }, + + toMatchSnapshot: { + // An acceptable ratio of pixels that are different to the + // total amount of pixels, between 0 and 1. + maxDiffPixelRatio: 0.1, + }, + }, + + /* Configure projects for major browsers */ + projects: [ + // { + // name: 'chromium', + // use: { ...devices['Desktop Chrome'] }, + // }, + + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + // { + // name: 'webkit', + // use: { ...devices['Desktop Safari'] }, + // }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); +