(3/3) Image checking CI (#343)

- Refactor e2e test names from "./test_file.ts" to "test_file" this
simplifies operations throughout the pipeline.
- Add the following to the `testcafe` runner:
  - Fixed window size
  - Disable scorllbars
  - Disable GPU acceleration
  - Disable thumbnails
  - Enable videos (available as artifacts from test runs)
- Add `check_screenshots` job that runs `check_images.py` for each e2e
test
- Add `merge_screenshots` job that merges screenshots and deltas from
all e2e tests into one convenient artifact
- Modify `check_images.py`
- Include a mode where you can pass the specific test to check with
`--test=`
- Add explicit checks that expected reference and actual files exist,
error when one is missing
- Copy screenshots that have changed from reference to a
`changed_screenshots` folder

Co-authored-by: Luke Brody <lukebrody@pavonine.co>

.github/workflows/check.yml

@@ -72,7 +72,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - id: set-matrix
-        run: echo "matrix=$(find . -name '*_test.ts' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+        run: echo "matrix=$(find . -name '*_test.ts' | jq -R -s -c 'gsub("./"; "") | gsub(".ts"; "") | split("\n")[:-1]')" >> $GITHUB_OUTPUT
         working-directory: react/test
 
   run-e2e-tests:
@@ -98,5 +98,73 @@ jobs:
           fluxbox >/dev/null 2>&1 & # needed for window resizing in Testcafe
           npm ci
           npx ts-node test/ci_proxy.ts &
-          npx testcafe -e chromium test/${{ matrix.test-file }}
+          npx testcafe -e "chromium '--window-size=1400,800' --hide-scrollbars --disable-gpu" \
+            -s thumbnails=false \
+            --video videos \
+            test/${{ matrix.test-file }}.ts
         working-directory: react
+      - uses: actions/upload-artifact@v4
+        if: "!cancelled()"
+        with:
+          name: screenshots-${{ matrix.test-file }}
+          path: react/screenshots
+      - uses: actions/upload-artifact@v4
+        if: "!cancelled()"
+        with:
+          name: videos-${{ matrix.test-file }}
+          path: react/videos
+
+  check-screenshots:
+    runs-on: ubuntu-22.04
+    needs: [build-and-push-image, list-e2e-tests, run-e2e-tests]
+    if: ${{ !cancelled() && needs.build-and-push-image.result == 'success' && needs.list-e2e-tests.result == 'success' }} # Should run even if e2e tests fail
+    container:
+      image: ghcr.io/${{ needs.build-and-push-image.outputs.image_tag }}
+    strategy:
+      fail-fast: false
+      matrix:
+        test-file: ${{ fromJson(needs.list-e2e-tests.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: screenshots-${{ matrix.test-file }}
+          path: react/screenshots
+        continue-on-error: true
+      - name: Check screenshots
+        run: python3 tests/check_images.py --test=${{ matrix.test-file }}
+      - uses: actions/upload-artifact@v4
+        if: "!cancelled()"
+        with:
+          name: delta-${{ matrix.test-file }}
+          path: react/delta
+      - uses: actions/upload-artifact@v4
+        if: "!cancelled()"
+        with:
+          name: changed_screenshots-${{ matrix.test-file }}
+          path: react/changed_screenshots
+
+  merge-screenshots:
+    runs-on: ubuntu-22.04
+    needs: [run-e2e-tests, check-screenshots]
+    if: "!cancelled()"
+    steps:
+      - uses: actions/upload-artifact/merge@v4
+        with:
+          name: screenshots
+          pattern: "screenshots-*"
+      - uses: actions/upload-artifact/merge@v4
+        with:
+          name: delta
+          pattern: "delta-*"
+        continue-on-error: true
+      - uses: actions/upload-artifact/merge@v4
+        with:
+          name: changed_screenshots
+          pattern: "changed_screenshots-*"
+        continue-on-error: true
+      - uses: actions/upload-artifact/merge@v4
+        with:
+          name: combined
+          pattern: "{screenshots,delta,changed_screenshots}"
+          separate-directories: true

react/.gitignore

@@ -1,3 +1,5 @@
 node_modules
 screenshots
-delta
+delta
+changed_screenshots
+videos

react/test/Dockerfile

@@ -22,6 +22,9 @@ RUN apt-get -y install fluxbox
 # needed for downloads directory in TestCafe
 RUN apt-get -y install xdg-user-dirs
 
+# needed for video recording in TestCafe
+RUN apt-get -y install ffmpeg
+
 COPY ./requirements.txt ./
 RUN pip3 install --break-system-packages -r requirements.txt
 

tests/check_images.py

@@ -59,34 +59,67 @@ def compute_delta_image(ref, act):
     delta = np.concatenate([ref, indicator], axis=1)
     return diff_mask.any(), delta
 
-def test_paths(reference, actual, delta_path):
+def test_paths(reference, actual, delta_path, changed_path):
     ref = np.array(Image.open(reference))
     act = np.array(Image.open(actual))
     diff, delta = compute_delta_image(ref, act)
     if diff:
-        try:
-            os.makedirs(os.path.dirname(delta_path))
-        except FileExistsError:
-            pass
+        os.makedirs(os.path.dirname(delta_path), exist_ok=True)
         Image.fromarray(delta).save(delta_path)
         print(f"{reference} and {actual} are different")
+        os.makedirs(os.path.dirname(changed_path), exist_ok=True)
+        shutil.copy(actual, changed_path)
         return False
     else:
         return True
 
-def test_all_same(reference="reference_test_screeshots", actual="react/screenshots"):
-    shutil.rmtree("react/delta", ignore_errors=True)
+def test_all_same(reference, actual, delta, changed):
+    shutil.rmtree(delta, ignore_errors=True)
     errors = 0
+    for root, dirs, files in os.walk(actual):
+        for file in files:
+            actual_path = os.path.join(root, file)
+            relative = os.path.relpath(actual_path, actual)
+            reference_path = os.path.join(reference, relative)
+            changed_path = os.path.join(changed, relative)
+            if not os.path.isfile(reference_path):
+                errors += 1
+                print(f"Expected reference file {reference_path} not found")
+                os.makedirs(os.path.dirname(changed_path), exist_ok=True)
+                shutil.copy(actual_path, changed_path)
     for root, dirs, files in os.walk(reference):
         for file in files:
             reference_path = os.path.join(root, file)
             relative = os.path.relpath(reference_path, reference)
             actual_path = os.path.join(actual, relative)
-            delta_path = os.path.join("react/delta", relative)
-            errors += not test_paths(reference_path, actual_path, delta_path)
+            changed_path = os.path.join(changed, relative)
+            if not os.path.isfile(actual_path):
+                errors += 1
+                print(f"Expected actual file {actual_path} not found")
+                continue
+            delta_path = os.path.join(delta, relative)
+            errors += not test_paths(reference_path, actual_path, delta_path, changed_path)
     if errors:
         print(f"{errors} errors found")
+        exit(1)
     else:
         print("All tests passed")
 if __name__ == "__main__":
-    test_all_same()
+    import argparse
+    p = argparse.ArgumentParser()
+    p.add_argument("--test", required=False)
+    args = p.parse_args()
+    if args.test:
+        test_all_same(
+            reference=f"reference_test_screenshots/{args.test}",
+            actual=f"react/screenshots/{args.test}", 
+            delta=f"react/delta/{args.test}",
+            changed=f"react/changed_screenshots/{args.test}"
+        )
+    else:
+        test_all_same(
+            reference="reference_test_screenshots", 
+            actual="react/screenshots", 
+            delta="react/delta",
+            changed=f"react/changed_screenshots"
+        )