CI and image checking

.github/workflows/check.yml

@@ -72,7 +72,7 @@ jobs:
  steps:
  - uses: actions/checkout@v4
  - id: set-matrix
- run: echo "matrix=$(find . -name '*_test.ts' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+ run: echo "matrix=$(find . -name '*_test.ts' | jq -R -s -c 'gsub("./"; "") | gsub(".ts"; "") | split("\n")[:-1]')" >> $GITHUB_OUTPUT
  working-directory: react/test
 
  run-e2e-tests:
@@ -98,5 +98,73 @@ jobs:
  fluxbox >/dev/null 2>&1 & # needed for window resizing in Testcafe
  npm ci
  npx ts-node test/ci_proxy.ts &
- npx testcafe -e chromium test/${{ matrix.test-file }}
+ npx testcafe -e "chromium '--window-size=1400,800' --hide-scrollbars --disable-gpu" \
+ -s thumbnails=false \
+ --video videos \
+ test/${{ matrix.test-file }}.ts
  working-directory: react
+ - uses: actions/upload-artifact@v4
+ if: "!cancelled()"
+ with:
+ name: screenshots-${{ matrix.test-file }}
+ path: react/screenshots
+ - uses: actions/upload-artifact@v4
+ if: "!cancelled()"
+ with:
+ name: videos-${{ matrix.test-file }}
+ path: react/videos
+
+ check-screenshots:
+ runs-on: ubuntu-22.04
+ needs: [build-and-push-image, list-e2e-tests, run-e2e-tests]
+ if: ${{ !cancelled() && needs.build-and-push-image.result == 'success' && needs.list-e2e-tests.result == 'success' }} # Should run even if e2e tests fail
+ container:
+ image: ghcr.io/${{ needs.build-and-push-image.outputs.image_tag }}
+ strategy:
+ fail-fast: false
+ matrix:
+ test-file: ${{ fromJson(needs.list-e2e-tests.outputs.matrix) }}
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/download-artifact@v4
+ with:
+ name: screenshots-${{ matrix.test-file }}
+ path: react/screenshots
+ continue-on-error: true
+ - name: Check screenshots
+ run: python3 tests/check_images.py --test=${{ matrix.test-file }}
+ - uses: actions/upload-artifact@v4
+ if: "!cancelled()"
+ with:
+ name: delta-${{ matrix.test-file }}
+ path: react/delta
+ - uses: actions/upload-artifact@v4
+ if: "!cancelled()"
+ with:
+ name: changed_screenshots-${{ matrix.test-file }}
+ path: react/changed_screenshots
+
+ merge-screenshots:
+ runs-on: ubuntu-22.04
+ needs: [run-e2e-tests, check-screenshots]
+ if: "!cancelled()"
+ steps:
+ - uses: actions/upload-artifact/merge@v4
+ with:
+ name: screenshots
+ pattern: "screenshots-*"
+ - uses: actions/upload-artifact/merge@v4
+ with:
+ name: delta
+ pattern: "delta-*"
+ continue-on-error: true
+ - uses: actions/upload-artifact/merge@v4
+ with:
+ name: changed_screenshots
+ pattern: "changed_screenshots-*"
+ continue-on-error: true
+ - uses: actions/upload-artifact/merge@v4
+ with:
+ name: combined
+ pattern: "{screenshots,delta,changed_screenshots}"
+ separate-directories: true

react/.gitignore

@@ -1,3 +1,5 @@
 node_modules
 screenshots
-delta
+delta
+changed_screenshots
+videos

react/test/Dockerfile

@@ -22,6 +22,9 @@ RUN apt-get -y install fluxbox
 # needed for downloads directory in TestCafe
 RUN apt-get -y install xdg-user-dirs
 
+# needed for video recording in TestCafe
+RUN apt-get -y install ffmpeg
+
 COPY ./requirements.txt ./
 RUN pip3 install --break-system-packages -r requirements.txt
 

tests/check_images.py

@@ -59,34 +59,67 @@ def compute_delta_image(ref, act):
  delta = np.concatenate([ref, indicator], axis=1)
  return diff_mask.any(), delta
 
-def test_paths(reference, actual, delta_path):
+def test_paths(reference, actual, delta_path, changed_path):
  ref = np.array(Image.open(reference))
  act = np.array(Image.open(actual))
  diff, delta = compute_delta_image(ref, act)
  if diff:
- try:
- os.makedirs(os.path.dirname(delta_path))
- except FileExistsError:
- pass
+ os.makedirs(os.path.dirname(delta_path), exist_ok=True)
  Image.fromarray(delta).save(delta_path)
  print(f"{reference} and {actual} are different")
+ os.makedirs(os.path.dirname(changed_path), exist_ok=True)
+ shutil.copy(actual, changed_path)
  return False
  else:
  return True
 
-def test_all_same(reference="reference_test_screeshots", actual="react/screenshots"):
- shutil.rmtree("react/delta", ignore_errors=True)
+def test_all_same(reference, actual, delta, changed):
+ shutil.rmtree(delta, ignore_errors=True)
  errors = 0
+ for root, dirs, files in os.walk(actual):
+ for file in files:
+ actual_path = os.path.join(root, file)
+ relative = os.path.relpath(actual_path, actual)
+ reference_path = os.path.join(reference, relative)
+ changed_path = os.path.join(changed, relative)
+ if not os.path.isfile(reference_path):
+ errors += 1
+ print(f"Expected reference file {reference_path} not found")
+ os.makedirs(os.path.dirname(changed_path), exist_ok=True)
+ shutil.copy(actual_path, changed_path)
  for root, dirs, files in os.walk(reference):
  for file in files:
  reference_path = os.path.join(root, file)
  relative = os.path.relpath(reference_path, reference)
  actual_path = os.path.join(actual, relative)
- delta_path = os.path.join("react/delta", relative)
- errors += not test_paths(reference_path, actual_path, delta_path)
+ changed_path = os.path.join(changed, relative)
+ if not os.path.isfile(actual_path):
+ errors += 1
+ print(f"Expected actual file {actual_path} not found")
+ continue
+ delta_path = os.path.join(delta, relative)
+ errors += not test_paths(reference_path, actual_path, delta_path, changed_path)
  if errors:
  print(f"{errors} errors found")
+ exit(1)
  else:
  print("All tests passed")
 if __name__ == "__main__":
- test_all_same()
+ import argparse
+ p = argparse.ArgumentParser()
+ p.add_argument("--test", required=False)
+ args = p.parse_args()
+ if args.test:
+ test_all_same(
+ reference=f"reference_test_screenshots/{args.test}",
+ actual=f"react/screenshots/{args.test}", 
+ delta=f"react/delta/{args.test}",
+ changed=f"react/changed_screenshots/{args.test}"
+ )
+ else:
+ test_all_same(
+ reference="reference_test_screenshots", 
+ actual="react/screenshots", 
+ delta="react/delta",
+ changed=f"react/changed_screenshots"
+ )