feat: Record time difference in tests

.cspell/code-terms.txt

@@ -88,6 +88,7 @@ rels
 reqs
 rewritelinks
 rgba
+runtimes
 RIGHTOF
 sankey
 sequencenumber

.github/workflows/e2e.yml

@@ -15,6 +15,7 @@ on:
 
 permissions:
   contents: read
+  pull-requests: write
 
 env:
   # For PRs and MergeQueues, the target commit is used, and for push events, github.event.previous is used.
@@ -48,15 +49,35 @@ jobs:
         with:
           ref: ${{ env.targetHash }}
 
+      - name: Install dependencies
+        if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }}
+        uses: cypress-io/github-action@v6
+        with:
+          # just perform install
+          runTests: false
+
+      - name: Build
+        if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' && github.event_name == 'pull_request' }}
+        run: |
+          pnpm run build:viz
+          mkdir -p cypress/snapshots/stats/base
+          mv stats cypress/snapshots/stats/base
+
       - name: Cypress run
-        uses: cypress-io/github-action@v4
+        uses: cypress-io/github-action@v6
         id: cypress-snapshot-gen
         if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }}
         with:
+          install: false
           start: pnpm run dev
           wait-on: 'http://localhost:9000'
           browser: chrome
 
+      - name: Move runtime data
+        if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }}
+        run: |
+          mv cypress/snapshots/runtimes/current cypress/snapshots/runtimes/base
+
   e2e:
     runs-on: ubuntu-latest
     container:
@@ -86,15 +107,42 @@ jobs:
           path: ./cypress/snapshots
           key: ${{ runner.os }}-snapshots-${{ env.targetHash }}
 
+      - name: Install dependencies
+        uses: cypress-io/github-action@v6
+        with:
+          runTests: false
+
+      - name: Build
+        id: size
+        if: ${{ github.event_name == 'pull_request' && matrix.containers == 1 }}
+        run: |
+          pnpm run build:viz
+          mv stats cypress/snapshots/stats/head
+          {
+            echo 'size_diff<<EOF'
+            npx tsx scripts/size.ts
+            echo EOF
+          } >> "$GITHUB_OUTPUT"
+
+      # Size diff only needs to be posted from one job, on PRs.
+      - name: Comment PR size difference
+        if: ${{ github.event_name == 'pull_request' && matrix.containers == 1 }}
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            ${{ steps.size.outputs.size_diff }}
+          comment_tag: size-diff
+
       # Install NPM dependencies, cache them correctly
       # and run all Cypress tests
       - name: Cypress run
-        uses: cypress-io/github-action@v4
+        uses: cypress-io/github-action@v6
         id: cypress
         # If CYPRESS_RECORD_KEY is set, run in parallel on all containers
         # Otherwise (e.g. if running from fork), we run on a single container only
         if: ${{ ( env.CYPRESS_RECORD_KEY != '' ) || ( matrix.containers == 1 ) }}
         with:
+          install: false
           start: pnpm run dev:coverage
           wait-on: 'http://localhost:9000'
           browser: chrome
@@ -133,6 +181,16 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ always() }}
     steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v2
+        # uses version from "packageManager" field in package.json
+
+      - name: Setup Node.js 18.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+
       # Download all snapshot artifacts and merge them into a single folder
       - name: Download All Artifacts
         uses: actions/download-artifact@v4
@@ -141,6 +199,27 @@ jobs:
           pattern: snapshots-*
           merge-multiple: true
 
+      - name: Build
+        id: runtime
+        if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }}
+        run: |
+          mv ./snapshots/runtimes/current ./snapshots/runtimes/head
+          npm config set ignore-scripts true
+          pnpm install --frozen-lockfile
+          {
+            echo 'runtime_diff<<EOF'
+            npx tsx scripts/runTime.ts ./snapshots
+            echo EOF
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Comment PR runtime difference
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            ${{ steps.runtime.outputs.runtime_diff }}
+          comment_tag: size-diff
+
       # For successful push events, we save the snapshots cache
       - name: Save snapshots cache
         id: cache-upload

.gitignore

@@ -29,6 +29,7 @@ Gemfile.lock
 
 cypress/screenshots/
 cypress/snapshots/
+cypress/runtimes/
 
 # eslint --cache file
 .eslintcache
@@ -50,4 +51,4 @@ demos/dev/**
 tsx-0/**
 
 # autogenereated by langium-cli
-generated/
+generated/

cypress.config.ts

@@ -1,4 +1,6 @@
 import { defineConfig } from 'cypress';
+import fs from 'fs';
+import path from 'path';
 import { addMatchImageSnapshotPlugin } from 'cypress-image-snapshot/plugin';
 import coverage from '@cypress/code-coverage/task';
 import eyesPlugin from '@applitools/eyes-cypress';
@@ -17,6 +19,19 @@ export default eyesPlugin(
           }
           return launchOptions;
         });
+        on('task', {
+          recordRenderTime({ fileName, testName, timeTaken }) {
+            const resultsPath = path.join('cypress', 'snapshots', 'runtimes', 'current');
+            if (!fs.existsSync(resultsPath)) {
+              fs.mkdirSync(resultsPath, { recursive: true });
+            }
+            fs.appendFileSync(
+              path.join(resultsPath, `${fileName}.csv`),
+              `${testName},${timeTaken}\n`
+            );
+            return true;
+          },
+        });
         addMatchImageSnapshotPlugin(on, config);
         // copy any needed variables from process.env to config.env
         config.env.useAppli = process.env.USE_APPLI ? true : false;

cypress/helpers/util.ts

@@ -110,6 +110,14 @@ export const openURLAndVerifyRendering = (
 
   cy.visit(url);
   cy.window().should('have.property', 'rendered', true);
+  cy.window().then((win) => {
+    cy.task('recordRenderTime', {
+      fileName: Cypress.spec.name,
+      testName: name,
+      // @ts-ignore Dynamically added property.
+      timeTaken: win.renderTime,
+    });
+  });
   cy.get('svg').should('be.visible');
 
   if (validation) {

cypress/platform/viewer.js

@@ -8,6 +8,7 @@ function b64ToUtf8(str) {
 
 // Adds a rendered flag to window when rendering is done, so cypress can wait for it.
 function markRendered() {
+  window.renderTime = Date.now() - window.loadTime;
   if (window.Cypress) {
     window.rendered = true;
   }
@@ -131,6 +132,7 @@ if (typeof document !== 'undefined') {
   window.addEventListener(
     'load',
     function () {
+      this.window.loadTime = Date.now();
       if (this.location.href.match('xss.html')) {
         this.console.log('Using api');
         void contentLoadedApi().finally(markRendered);

package.json

@@ -111,6 +111,7 @@
     "jsdom": "^22.0.0",
     "langium-cli": "3.0.1",
     "lint-staged": "^13.2.1",
+    "markdown-table": "^3.0.3",
     "nyc": "^15.1.0",
     "path-browserify": "^1.0.1",
     "pnpm": "^8.6.8",

scripts/runTime.ts

@@ -0,0 +1,135 @@
+/* eslint-disable no-console */
+import { readFile } from 'fs/promises';
+import { globby } from 'globby';
+
+interface RunTimes {
+  [key: string]: number;
+}
+interface TestResult {
+  [key: string]: RunTimes;
+}
+
+const getRuntimes = (csv: string): RunTimes => {
+  const lines = csv.split('\n');
+  const runtimes: RunTimes = {};
+  for (const line of lines) {
+    const [testName, timeTaken] = line.split(',');
+    if (testName && timeTaken) {
+      runtimes[testName] = Number(timeTaken);
+    }
+  }
+  return runtimes;
+};
+
+const readStats = async (path: string): Promise<TestResult> => {
+  const files = await globby(path);
+  const contents = await Promise.all(
+    files.map(async (file) => [file, await readFile(file, 'utf-8')])
+  );
+  const sizes = contents.map(([file, content]) => [file.split('/').pop(), getRuntimes(content)]);
+  return Object.fromEntries(sizes);
+};
+
+const percentChangeThreshold = 5;
+const percentageDifference = (
+  oldValue: number,
+  newValue: number
+): { change: string; crossedThreshold: boolean } => {
+  const difference = Math.abs(newValue - oldValue);
+  const avg = (newValue + oldValue) / 2;
+  const percentage = (difference / avg) * 100;
+  const roundedPercentage = percentage.toFixed(2); // Round to two decimal places
+  if (roundedPercentage === '0.00') {
+    return { change: '0.00%', crossedThreshold: false };
+  }
+  const sign = newValue > oldValue ? '+' : '-';
+  return {
+    change: `${sign}${roundedPercentage}%`,
+    crossedThreshold: percentage > percentChangeThreshold,
+  };
+};
+
+const main = async () => {
+  const base = process.argv[2] || './cypress/snapshots';
+  const oldStats = await readStats(`${base}/runtimes/base/**/*.csv`);
+  const newStats = await readStats(`${base}/runtimes/head/**/*.csv`);
+  const fullData: string[][] = [];
+  const changed: string[][] = [];
+  let oldRuntimeSum = 0;
+  let newRuntimeSum = 0;
+  let testCount = 0;
+  for (const [fileName, runtimes] of Object.entries(newStats)) {
+    const oldStat = oldStats[fileName];
+    if (!oldStat) {
+      continue;
+    }
+    for (const [testName, timeTaken] of Object.entries(runtimes)) {
+      const oldTimeTaken = oldStat[testName];
+      if (!oldTimeTaken) {
+        continue;
+      }
+      oldRuntimeSum += oldTimeTaken;
+      newRuntimeSum += timeTaken;
+      testCount++;
+      const delta = timeTaken - oldTimeTaken;
+
+      const { change, crossedThreshold } = percentageDifference(oldTimeTaken, timeTaken);
+      const out = [
+        fileName,
+        testName.replace('#', ''),
+        `${oldTimeTaken}/${timeTaken}`,
+        `${delta.toString()}ms ${change}`,
+      ];
+      if (crossedThreshold && Math.abs(delta) > 25) {
+        changed.push(out);
+      }
+      fullData.push(out);
+    }
+  }
+  const oldAverage = oldRuntimeSum / testCount;
+  const newAverage = newRuntimeSum / testCount;
+  const { change, crossedThreshold } = percentageDifference(oldAverage, newAverage);
+
+  const headers = ['File', 'Test', 'Time Old/New', 'Change (%)'];
+  console.log(`## Runtime Changes
+Old runtime average: ${oldAverage.toFixed(2)}ms
+New runtime average: ${newAverage.toFixed(2)}ms
+Change: ${change} ${crossedThreshold ? '⚠️' : ''}
+  `);
+  console.log(`
+  <details>
+  <summary>Changed tests</summary>
+  ${htmlTable([headers, ...changed])}
+</details>
+`);
+  console.log(`
+  <details>
+  <summary>Full Data</summary>
+  ${htmlTable([headers, ...fullData])}
+</details>
+`);
+};
+
+const htmlTable = (data: string[][]): string => {
+  let table = `<table border='1' style="border-collapse: collapse">`;
+
+  // Generate table header
+  table += `<tr>
+    ${data
+      .shift()!
+      .map((header) => `<th>${header}</th>`)
+      .join('')}
+    </tr>`;
+
+  // Generate table rows
+  for (const row of data) {
+    table += `<tr>
+    ${row.map((cell) => `<td>${cell}</td>`).join('')}
+    </tr>`;
+  }
+
+  table += '</table>';
+  return table;
+};
+
+void main().catch((e) => console.error(e));