dev: improve logging and loading (kkrt-labs#543)

* log failing output at end of test

* only deserialize if test should not be skipped

* add config for nextest, timing out after 20 min test

.config/nextest.toml

@@ -0,0 +1,2 @@
+[profile.default]
+slow-timeout = { period = "5m", terminate-after = 4 }

Makefile

@@ -41,8 +41,8 @@ unit:
 
 # Runs the repo tests
 tests:
-	cargo nextest run
+	cargo nextest run --failure-output final
 
 # Runs ef tests only
 ef-test:
-	cargo nextest run --package ef-testing --test tests
+	cargo nextest run --package ef-testing --test tests --failure-output final

crates/ef-testing/src/models/case.rs

@@ -31,7 +31,6 @@ pub struct BlockchainTestCase {
     pub name: String,
     pub tests: BTreeMap<String, BlockchainTest>,
     pub transaction: BlockchainTestTransaction,
-    skip: bool,
 }
 
 #[derive(Deserialize)]
@@ -242,6 +241,10 @@ impl Case for BlockchainTestCase {
     /// Load a test case from a path. This is a path to a directory containing
     /// the BlockChainTest
     fn load(path: &Path) -> Result<Self, RunnerError> {
+        if Self::should_skip(path) {
+            return Err(RunnerError::Skipped);
+        }
+
         let general_state_tests_path = path
             .components()
             .filter(|x| !x.as_os_str().eq_ignore_ascii_case("BlockchainTests"))
@@ -281,15 +284,10 @@ impl Case for BlockchainTestCase {
                 deserialize_into(&case.to_string(), general_state_tests_path)?
             },
             name: test_name.to_string(),
-            skip: Self::should_skip(path),
         })
     }
 
     async fn run(&self) -> Result<(), RunnerError> {
-        if self.skip {
-            return Err(RunnerError::Skipped);
-        }
-
         let test_regexp: Option<String> = std::env::var("TARGET").ok();
         let test_regexp = match test_regexp {
             Some(x) => Some(Regex::new(x.as_str())?),

crates/ef-testing/src/traits/mod.rs

@@ -53,7 +53,13 @@ pub trait Suite {
         let mut test_cases = Vec::new();
 
         for test_case_path in test_cases_paths {
-            let case = Self::Case::load(&test_case_path).expect("test case should load");
+            let case = match Self::Case::load(&test_case_path) {
+                Ok(case) => case,
+                Err(e) => match e {
+                    RunnerError::Skipped => continue,
+                    _ => panic!("Failed to load test case: {:?}", e),
+                },
+            };
             test_cases.push((test_case_path, case));
         }
 

crates/ef-testing/tests/tests.rs

@@ -31,11 +31,6 @@ macro_rules! blockchain_tests {
 mod blockchain_tests {
     use super::*;
 
-    // TODO: Uncomment tests to add entire folder to test flow.
-    // All failing tests should be added to the skip session at "blockchain-tests-skip.yml"
-    // and a respective issue at https://github.com/kkrt-labs/ef-tests/issues should be created.
-    // Tests from the same folder with the same error can be aggregated in one issue
-
     blockchain_tests!(shanghai, Shanghai);
     blockchain_tests!(st_args_zero_one_balance, stArgsZeroOneBalance);
     blockchain_tests!(st_attack_test, stAttackTest);