ENG-1293: Running tests with --e2e-results, and saving results on f…

…ailure, to assist in troubleshooting
RasaHQ · Sep 13, 2024 · 6397722 · 6397722
1 parent 5512be9
commit 6397722
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 14 deletions.
diff --git a/.github/workflows/continous-integration.yml b/.github/workflows/continous-integration.yml
@@ -174,14 +174,25 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
         RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_INTENTLESS: true
       run: |
-        make test-passing
+        make test-passing 
+
+    - name: Save test-passing results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-passing-results
+        path: tests/
 
     - name: Run e2e flaky tests
       if: always()
@@ -191,7 +202,7 @@ jobs:
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_INTENTLESS: true
       run: |
-        make test-flaky || true
+        make test-flaky  || true
 
     - name: Run e2e failing tests
       if: always()
@@ -203,6 +214,13 @@ jobs:
       run: |
         make test-failing | grep '0 passed'
 
+    - name: Save test-failing results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-failing-results
+        path: tests/*passed.yml
+
     - name: Stop Duckling server
       run: |
           make stop-duckling
@@ -282,14 +300,25 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests with assertions
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
         RASA_PRO_LICENSE: ${{secrets.RASA_PRO_LICENSE}}
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_E2E_ASSERTIONS: true
       run: |
-        make test-passing-assertions
+        make test-passing-assertions 
+
+    - name: Save test-passing-assertions results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-passing-assertions-results
+        path: tests/
 
     - name: Run e2e flaky tests with assertions
       if: always()
@@ -299,7 +328,7 @@ jobs:
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_E2E_ASSERTIONS: true
       run: |
-        make test-flaky-assertions || true
+        make test-flaky-assertions  || true
 
     - name: Run e2e failing tests with assertions
       if: always()
@@ -309,7 +338,14 @@ jobs:
         RASA_DUCKLING_HTTP_URL: ${{secrets.DUCKLING_URL}}
         RASA_PRO_BETA_E2E_ASSERTIONS: true
       run: |
-        make test-failing-assertions | grep '0 passed'
+        make test-failing-assertions  | grep '0 passed'
+
+    - name: Save test-failing-assertions results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-failing-assertions-results
+        path: tests/*passed.yml
 
     - name: Stop Duckling server
       run: |
@@ -381,6 +417,10 @@ jobs:
       run: |
         make run-duckling
 
+    - name: Make test results directory
+      run: |
+        mkdir tests
+
     - name: Run e2e passing tests with stub custom actions
       env:
         OPENAI_API_KEY: ${{secrets.OPENAI_API_KEY}}
@@ -389,7 +429,14 @@ jobs:
         RASA_PRO_BETA_E2E_ASSERTIONS: true
         RASA_PRO_BETA_STUB_CUSTOM_ACTION: true
       run: |
-        make test-passing-stub-custom-actions
+        make test-passing-stub-custom-actions 
+
+    - name: Save test-stub-custom-actions-passing results
+      if: failure()
+      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+      with:
+        name: test-stub-custom-actions-passing-results
+        path: tests/
 
     - name: Stop Duckling server
       run: |

diff --git a/Makefile b/Makefile
@@ -75,16 +75,16 @@ actions:
 	poetry run rasa run actions
 
 test-passing: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/passing
+	poetry run rasa test e2e e2e_tests/passing --e2e-results
 
 test-flaky: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/flaky
+	poetry run rasa test e2e e2e_tests/flaky --e2e-results
 
 test-failing: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/failing
+	poetry run rasa test e2e e2e_tests/failing --e2e-results
 
 test-multistep: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests/multistep
+	poetry run rasa test e2e e2e_tests/multistep --e2e-results
 
 test-one: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e $(target) --debug
@@ -93,13 +93,13 @@ stop-duckling:
 	docker stop duckling_container
 
 test-passing-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/passing
+	poetry run rasa test e2e e2e_tests_with_assertions/passing --e2e-results
 
 test-flaky-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/flaky
+	poetry run rasa test e2e e2e_tests_with_assertions/flaky --e2e-results
 
 test-failing-assertions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_assertions/failing
+	poetry run rasa test e2e e2e_tests_with_assertions/failing --e2e-results
 
 make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES
-	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing
+	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing --e2e-results