Merge pull request #512 from dodona-edu/tests/js-integration

Add JavaScript integration tests
dodona-edu · May 30, 2024 · d842e31 · d842e31
2 parents 771588a + 5b3c125
commit d842e31
Show file tree

Hide file tree

Showing 12 changed files with 19,253 additions and 98 deletions.
diff --git a/.github/dodona-image-integration.dockerfile b/.github/dodona-image-integration.dockerfile
@@ -0,0 +1,12 @@
+# Inherit from the Docker image for Dodona.
+FROM dodona/dodona-tested
+
+# Go back to being root.
+USER root
+WORKDIR /
+
+# Install some additional dependencies needed for testing.
+RUN pip install --no-cache-dir --upgrade pytest pytest-mock pytest-xdist jinja2 marko syrupy
+
+# The source of the judge is available in TESTED_SOURCE.
+CMD pytest -n auto ${TESTED_SOURCE}/tests/test_integration_javascript.py
diff --git a/.github/dodona-image.dockerfile b/.github/dodona-image.dockerfile
@@ -6,7 +6,7 @@ USER root
 WORKDIR /
 
 # Install some additional dependencies needed for testing.
-RUN pip install --no-cache-dir --upgrade pytest pytest-mock pytest-xdist jinja2 marko
+RUN pip install --no-cache-dir --upgrade pytest pytest-mock pytest-xdist jinja2 marko syrupy
 
 # The source of the judge is available in TESTED_SOURCE.
-CMD pytest -x -n auto ${TESTED_SOURCE}/tests/
+CMD pytest -x -n auto --ignore=${TESTED_SOURCE}/tests/test_integration_javascript.py ${TESTED_SOURCE}/tests/
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -1,15 +1,56 @@
 name: Integration tests
 
-on: [ workflow_dispatch ]
+on: [ pull_request ]
+
+env:
+  EXERCISES_COMMIT: 4a2094135abe972eb38072be129171f685c18ec3
 
 jobs:
   # Runs the test suite in a slightly modified Docker image used by Dodona.
   # This is the closest to actually running the production environment there is.
-  dodona-docker:
+  tests-dodona:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - run: docker build -t "local-image" -f ${{ github.workspace }}/.github/dodona-image.dockerfile --network=host .
         name: Build Dodona Docker image
       - run: docker run -v ${{ github.workspace }}:/github/workspace -e TESTED_SOURCE=/github/workspace local-image
         name: Run tests in Dodona Docker image
+  # Runs in the Docker image to check if the JS exercises still work.
+  javascript-dodona:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          echo "$JAVASCRIPT_EXERCISES_KEY" > private
+          chmod 0600 private
+          GIT_SSH_COMMAND='ssh -o "StrictHostKeyChecking no" -i private' git clone git@github.ugent.be:Scriptingtalen/javascript-oefeningen.git
+          rm private
+        env:
+          JAVASCRIPT_EXERCISES_KEY: ${{ secrets.JAVASCRIPT_EXERCISES_KEY }}
+      - run: git checkout $EXERCISES_COMMIT
+        working-directory: ./javascript-oefeningen
+      - run: docker build -t "integration-image" -f ${{ github.workspace }}/.github/dodona-image-integration.dockerfile --network=host .
+        name: Build Dodona Docker image
+      - run: docker run -v ${{ github.workspace }}:/github/workspace -e TESTED_SOURCE=/github/workspace -e EXERCISE_REPO=/github/workspace/javascript-oefeningen integration-image
+        name: Run integration tests in Dodona Docker image
+  # Runs in the repo to check if the JS exercises still work.
+  javascript-repository:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: echo "${GITHUB_WORKSPACE}" >> $GITHUB_PATH
+      - run: |
+          echo "$JAVASCRIPT_EXERCISES_KEY" > private
+          chmod 0600 private
+          GIT_SSH_COMMAND='ssh -o "StrictHostKeyChecking no" -i private' git clone git@github.ugent.be:Scriptingtalen/javascript-oefeningen.git
+          rm private
+        env:
+          JAVASCRIPT_EXERCISES_KEY: ${{ secrets.JAVASCRIPT_EXERCISES_KEY }}
+      - run: git checkout $EXERCISES_COMMIT
+        working-directory: ./javascript-oefeningen
+      - run: nix run .#devShell -- run-intergation-tests
+        env:
+          EXERCISE_REPO: ${{ github.workspace }}/javascript-oefeningen
diff --git a/flake.nix b/flake.nix
@@ -43,11 +43,15 @@
           );
         };
 
+        nodejs_base = pkgs.nodejs_22;
+
         # This one isn't in Nix, so do it manually.
         ast = pkgs.buildNpmPackage rec {
           pname = "abstract-syntax-tree";
           version = "2.22.0";
 
+          nodejs = nodejs_base;
+
           src = pkgs.fetchFromGitHub {
             owner = "buxlabs";
             repo = pname;
@@ -71,7 +75,7 @@
           (pkgs.haskell.packages.ghc96.ghcWithPackages (p: [ p.aeson ]))
           pkgs.hlint
         ];
-        node-deps = [ pkgs.nodejs_22 pkgs.nodePackages.eslint ast ];
+        node-deps = [ nodejs_base pkgs.nodePackages.eslint ast ];
         bash-deps = [ pkgs.shellcheck ];
         c-deps = [ pkgs.cppcheck pkgs.gcc13 ];
         java-deps = [ pkgs.openjdk21 pkgs.checkstyle ];
@@ -94,29 +98,23 @@
           inherit (python-base-env) projectDir python overrides;
           propagatedBuildInputs = all-other-dependencies;
         };
-
-        unit-test = pkgs.writeShellApplication {
-          name = "unit-test";
-          runtimeInputs = [ python-dev-env pkgs.poetry ] ++ all-other-dependencies;
-          text = ''
-            DOTNET_CLI_HOME="$(mktemp -d)"
-            export DOTNET_CLI_HOME
-            poetry run pytest -n auto --cov=tested --cov-report=xml tests/
-          '';
+        poetry-package = (pkgs.poetry.override {python3 = python;}).overridePythonAttrs {
+          doCheck = false;
         };
-
       in {
         checks = rec {
           default = simple-tests;
           simple-tests = pkgs.stdenvNoCC.mkDerivation {
             name = "simple-tests";
             src = self;
             doCheck = true;
-            checkInputs = [ python-dev-env pkgs.poetry ] ++ all-other-dependencies;
+            checkInputs = [ python-dev-env poetry-package ] ++ all-other-dependencies;
             checkPhase = ''
               DOTNET_CLI_HOME="$(mktemp -d)"
               export DOTNET_CLI_HOME
-              poetry run pytest -n auto --cov=tested --cov-report=xml tests/
+              NODE_PATH=${ast}/lib/node_modules
+              export NODE_PATH
+              poetry run pytest -n auto --cov=tested --cov-report=xml tests/test_functionality.py
             '';
             installPhase = ''
               touch $out # it worked!
@@ -130,15 +128,20 @@
             inherit (tested-env)
               projectDir python overrides propagatedBuildInputs;
             doCheck = false;
+            postInstall = ''
+              wrapProgram "$out/bin/tested" \
+                --prefix NODE_PATH : ${ast}/lib/node_modules
+            '';
           };
+          devShell = self.outputs.devShells.${system}.default;
         };
 
         devShells = rec {
           default = tested;
           tested = pkgs.devshell.mkShell {
             name = "TESTed";
 
-            packages = [ python-dev-env pkgs.nodePackages.pyright pkgs.poetry ]
+            packages = [ python-dev-env pkgs.nodePackages.pyright poetry-package ]
               ++ all-other-dependencies;
 
             devshell.startup.link.text = ''
@@ -149,18 +152,32 @@
             env = [
               {
                 name = "DOTNET_ROOT";
-                eval = "${pkgs.dotnetCorePackages.sdk_6_0}";
+                eval = "${pkgs.dotnetCorePackages.sdk_8_0}";
               }
               {
                 name = "NODE_PATH";
-                prefix = "$(npm get prefix)";
+                prefix = "${ast}/lib/node_modules";
+              }
+            ];
+            commands = [
+              {
+                name = "run-intergation-tests";
+                command = "poetry run pytest -x -n auto tests/test_integration_javascript.py";
+                help = "Run JavaScript integration tests";
+                category = "tests";
+              }
+              {
+                name = "run-tests";
+                command = "poetry run pytest -n auto tests/";
+                help = "Run unit tests";
+                category = "tests";
               }
             ];
           };
           types = pkgs.mkShell {
             packages = [ python-dev-env pkgs.nodePackages.pyright ];
           };
-          format = pkgs.mkShell { packages = [ python-dev-env pkgs.poetry ]; };
+          format = pkgs.mkShell { packages = [ python-dev-env poetry-package ]; };
         };
       });
 }