From ad5cefcba9dd21efe09562fcf1bc4e92bba164e6 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Wed, 29 May 2024 16:55:04 +0200 Subject: [PATCH] Debug runner --- .github/workflows/integration.yml | 36 ++++++++++++++-------------- tested/judge/core.py | 8 +++++++ tested/judge/execution.py | 9 +++++++ tests/test_integration_javascript.py | 18 ++++++++++++++ 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7e469966..18c565c0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -12,28 +12,28 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: docker build -t "local-image" -f ${{ github.workspace }}/.github/dodona-image.dockerfile --network=host . - name: Build Dodona Docker image - - run: docker run -v ${{ github.workspace }}:/github/workspace -e TESTED_SOURCE=/github/workspace local-image - name: Run tests in Dodona Docker image +# - run: docker build -t "local-image" -f ${{ github.workspace }}/.github/dodona-image.dockerfile --network=host . +# name: Build Dodona Docker image +# - run: docker run -v ${{ github.workspace }}:/github/workspace -e TESTED_SOURCE=/github/workspace local-image +# name: Run tests in Dodona Docker image # Runs in the Docker image to check if the JS exercises still work. javascript-dodona: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: | - echo "$JAVASCRIPT_EXERCISES_KEY" > private - chmod 0600 private - GIT_SSH_COMMAND='ssh -o "StrictHostKeyChecking no" -i private' git clone git@github.ugent.be:Scriptingtalen/javascript-oefeningen.git - rm private - env: - JAVASCRIPT_EXERCISES_KEY: ${{ secrets.JAVASCRIPT_EXERCISES_KEY }} - - run: git checkout $EXERCISES_COMMIT - working-directory: ./javascript-oefeningen - - run: docker build -t "integration-image" -f ${{ github.workspace }}/.github/dodona-image-integration.dockerfile --network=host . - name: Build Dodona Docker image - - run: docker run -v ${{ github.workspace }}:/github/workspace -e TESTED_SOURCE=/github/workspace -e EXERCISE_REPO=/github/workspace/javascript-oefeningen integration-image - name: Run integration tests in Dodona Docker image +# - run: | +# echo "$JAVASCRIPT_EXERCISES_KEY" > private +# chmod 0600 private +# GIT_SSH_COMMAND='ssh -o "StrictHostKeyChecking no" -i private' git clone git@github.ugent.be:Scriptingtalen/javascript-oefeningen.git +# rm private +# env: +# JAVASCRIPT_EXERCISES_KEY: ${{ secrets.JAVASCRIPT_EXERCISES_KEY }} +# - run: git checkout $EXERCISES_COMMIT +# working-directory: ./javascript-oefeningen +# - run: docker build -t "integration-image" -f ${{ github.workspace }}/.github/dodona-image-integration.dockerfile --network=host . +# name: Build Dodona Docker image +# - run: docker run -v ${{ github.workspace }}:/github/workspace -e TESTED_SOURCE=/github/workspace -e EXERCISE_REPO=/github/workspace/javascript-oefeningen integration-image +# name: Run integration tests in Dodona Docker image # Runs in the repo to check if the JS exercises still work. javascript-repository: runs-on: ubuntu-latest @@ -51,4 +51,4 @@ jobs: JAVASCRIPT_EXERCISES_KEY: ${{ secrets.JAVASCRIPT_EXERCISES_KEY }} - run: git checkout $EXERCISES_COMMIT working-directory: ./javascript-oefeningen - - run: nix develop -c bash -c 'EXERCISE_REPO=${{ github.workspace }}/javascript-oefeningen poetry run pytest -n auto ${{ github.workspace }}/tests/test_integration_javascript.py' + - run: nix develop -c bash -c "EXERCISE_REPO=./javascript-oefeningen poetry run pytest -x tests/test_integration_javascript.py" diff --git a/tested/judge/core.py b/tested/judge/core.py index 5c2ca35c..424eb256 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -242,6 +242,14 @@ def _execute_one_unit( else: execution_result = None + if execution_result is not None: + print("---------------- STDOUT EXECUTION -------------------") + print(execution_result.stdout) + print("---------------- STDERR EXECUTION -------------------") + print(execution_result.stderr) + print("---------------- EXIT CODE -------------------") + print(execution_result.exit) + return local_compilation_results, execution_result, execution_dir diff --git a/tested/judge/execution.py b/tested/judge/execution.py index c5ee6f42..cf51cf8b 100644 --- a/tested/judge/execution.py +++ b/tested/judge/execution.py @@ -1,5 +1,6 @@ import itertools import logging +import subprocess from pathlib import Path from attrs import define @@ -259,6 +260,14 @@ def execute_unit( values = _get_contents_or_empty(value_file(bundle, execution_dir)) exceptions = _get_contents_or_empty(exception_file(bundle, execution_dir)) + print("---------------- VALUES AFTER EXECUTION --------------") + print(values) + print("---------------- EXCEPTIONS AFTER EXECUTION --------------") + print(exceptions) + print("---------------- LS --------------------------------------") + return_code = subprocess.call(f"tree {str(execution_dir)}", shell=True, text=True) + print(return_code) + return ExecutionResult( stdout=base_result.stdout, stderr=base_result.stderr, diff --git a/tests/test_integration_javascript.py b/tests/test_integration_javascript.py index 9e814ae9..ddb6f4df 100644 --- a/tests/test_integration_javascript.py +++ b/tests/test_integration_javascript.py @@ -4,7 +4,9 @@ """ import json +import logging import os +import sys from pathlib import Path import pytest @@ -81,6 +83,18 @@ def get_exercises() -> list[Path]: @pytest.mark.parametrize("exercise", ALL_EXERCISES, ids=lambda ex: ex.name) def test_javascript_exercise(exercise: Path, tmp_path: Path, pytestconfig, snapshot): + log = logging.getLogger() + log.setLevel(logging.DEBUG) + ch = logging.StreamHandler(stream=sys.stdout) + formatter = logging.Formatter("%(name)s:%(levelname)s:%(message)s") + ch.setFormatter(formatter) + log.addHandler(ch) + + # Some modules are very verbose, hide those by default. + logger = logging.getLogger("tested.judge.collector") + logger.setLevel(logging.INFO) + logger = logging.getLogger("tested.parsing") + logger.setLevel(logging.INFO) conf = exercise_configuration( pytestconfig, exercise, @@ -89,6 +103,10 @@ def test_javascript_exercise(exercise: Path, tmp_path: Path, pytestconfig, snaps "suite.yaml", "solution.nl", ) + print("------------ SOURCE -------------") + with open(conf.source, 'r') as s: + print(s.read()) + print("END OF ------------ SOURCE -------------") result = execute_config(conf) updates = assert_valid_output(result, pytestconfig) status = updates.find_status_enum()