From 6397722ac6e9b4a37e17a34a9d2df2fe6546e0cb Mon Sep 17 00:00:00 2001 From: Github Actions Date: Thu, 12 Sep 2024 18:47:04 +0100 Subject: [PATCH] ENG-1293: Running tests with `--e2e-results`, and saving results on failure, to assist in troubleshooting --- .github/workflows/continous-integration.yml | 59 ++++++++++++++++++--- Makefile | 16 +++--- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml index 2bf17b3..55884de 100644 --- a/.github/workflows/continous-integration.yml +++ b/.github/workflows/continous-integration.yml @@ -174,6 +174,10 @@ jobs: run: | make run-duckling + - name: Make test results directory + run: | + mkdir tests + - name: Run e2e passing tests env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} @@ -181,7 +185,14 @@ jobs: RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_INTENTLESS: true run: | - make test-passing + make test-passing + + - name: Save test-passing results + if: failure() + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-passing-results + path: tests/ - name: Run e2e flaky tests if: always() @@ -191,7 +202,7 @@ jobs: RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_INTENTLESS: true run: | - make test-flaky || true + make test-flaky || true - name: Run e2e failing tests if: always() @@ -203,6 +214,13 @@ jobs: run: | make test-failing | grep '0 passed' + - name: Save test-failing results + if: failure() + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-failing-results + path: tests/*passed.yml + - name: Stop Duckling server run: | make stop-duckling @@ -282,6 +300,10 @@ jobs: run: | make run-duckling + - name: Make test results directory + run: | + mkdir tests + - name: Run e2e passing tests with assertions env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} @@ -289,7 +311,14 @@ jobs: RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_E2E_ASSERTIONS: true run: | - make test-passing-assertions + make test-passing-assertions + + - name: Save test-passing-assertions results + if: failure() + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-passing-assertions-results + path: tests/ - name: Run e2e flaky tests with assertions if: always() @@ -299,7 +328,7 @@ jobs: RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_E2E_ASSERTIONS: true run: | - make test-flaky-assertions || true + make test-flaky-assertions || true - name: Run e2e failing tests with assertions if: always() @@ -309,7 +338,14 @@ jobs: RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}} RASA_PRO_BETA_E2E_ASSERTIONS: true run: | - make test-failing-assertions | grep '0 passed' + make test-failing-assertions | grep '0 passed' + + - name: Save test-failing-assertions results + if: failure() + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-failing-assertions-results + path: tests/*passed.yml - name: Stop Duckling server run: | @@ -381,6 +417,10 @@ jobs: run: | make run-duckling + - name: Make test results directory + run: | + mkdir tests + - name: Run e2e passing tests with stub custom actions env: OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}} @@ -389,7 +429,14 @@ jobs: RASA_PRO_BETA_E2E_ASSERTIONS: true RASA_PRO_BETA_STUB_CUSTOM_ACTION: true run: | - make test-passing-stub-custom-actions + make test-passing-stub-custom-actions + + - name: Save test-stub-custom-actions-passing results + if: failure() + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: test-stub-custom-actions-passing-results + path: tests/ - name: Stop Duckling server run: | diff --git a/Makefile b/Makefile index 856e69c..838b7fa 100644 --- a/Makefile +++ b/Makefile @@ -75,16 +75,16 @@ actions: poetry run rasa run actions test-passing: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests/passing + poetry run rasa test e2e e2e_tests/passing --e2e-results test-flaky: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests/flaky + poetry run rasa test e2e e2e_tests/flaky --e2e-results test-failing: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests/failing + poetry run rasa test e2e e2e_tests/failing --e2e-results test-multistep: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests/multistep + poetry run rasa test e2e e2e_tests/multistep --e2e-results test-one: .EXPORT_ALL_VARIABLES poetry run rasa test e2e $(target) --debug @@ -93,13 +93,13 @@ stop-duckling: docker stop duckling_container test-passing-assertions: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests_with_assertions/passing + poetry run rasa test e2e e2e_tests_with_assertions/passing --e2e-results test-flaky-assertions: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests_with_assertions/flaky + poetry run rasa test e2e e2e_tests_with_assertions/flaky --e2e-results test-failing-assertions: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests_with_assertions/failing + poetry run rasa test e2e e2e_tests_with_assertions/failing --e2e-results make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES - poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing + poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing --e2e-results