chore: show RunResources in test-skip.yml, print revert reason in test (

#674)

* chore: utils

* fix indent in yaml generation

* fix: fmt

* rename diff -> errors

crates/ef-testing/src/models/case.rs

@@ -117,7 +117,7 @@ impl BlockchainTestCase {
         let sender_address = wallet.address().to_fixed_bytes();
 
         let maybe_revert_reason = String::from_utf8(output.return_data.as_slice().to_vec());
-        let eth_validation_failed = maybe_revert_reason.map_or(false, |revert_reason| {
+        let eth_validation_failed = maybe_revert_reason.as_ref().map_or(false, |revert_reason| {
             revert_reason == "Kakarot: eth validation failed"
         });
 
@@ -161,12 +161,12 @@ impl BlockchainTestCase {
         let post_state = self.post.clone().expect("Post state not found");
         let post_state = update_post_state(post_state, self.pre.clone());
 
-        let mut diff: Vec<String> = vec![];
+        let mut errors: Vec<String> = vec![];
 
         let actual_gas_used = output.gas_used;
         let expected_gas_u64: u64 = expected_gas_used.try_into().unwrap();
         if expected_gas_u64 != actual_gas_used {
-            diff.push(format!(
+            errors.push(format!(
                 "gas used mismatch: expected {expected_gas_u64}, got {actual_gas_used}"
             ));
         }
@@ -180,7 +180,7 @@ impl BlockchainTestCase {
                         "storage mismatch for {:#20x} at {:#32x}: expected {:#32x}, got {:#32x}",
                         address, k, v, actual
                     );
-                    diff.push(storage_diff);
+                    errors.push(storage_diff);
                 }
             }
 
@@ -197,7 +197,7 @@ impl BlockchainTestCase {
                     "nonce mismatch for {:#20x}: expected {:#32x}, got {:#32x}",
                     address, expected_state.nonce, actual
                 );
-                diff.push(nonce_diff);
+                errors.push(nonce_diff);
             }
 
             // Bytecode
@@ -207,7 +207,7 @@ impl BlockchainTestCase {
                     "code mismatch for {:#20x}: expected {:#x}, got {:#x}",
                     address, expected_state.code, actual
                 );
-                diff.push(bytecode_diff);
+                errors.push(bytecode_diff);
             }
 
             // Balance
@@ -225,12 +225,15 @@ impl BlockchainTestCase {
                     "balance mismatch for {:#20x}: expected {:#32x}, got {:#32x}",
                     address, expected_state.balance, actual
                 );
-                diff.push(balance_diff);
+                errors.push(balance_diff);
             }
         }
 
-        if !diff.is_empty() {
-            return Err(RunnerError::Other(diff.into()));
+        if !errors.is_empty() {
+            if let Ok(revert_reason) = maybe_revert_reason {
+                errors.push(format!("revert reason: {}", revert_reason));
+            }
+            return Err(RunnerError::Other(errors.into()));
         }
 
         Ok(())

scripts/generate_skip_file.py

@@ -3,6 +3,34 @@
 from collections import defaultdict
 
 
+def extract_runresource_failures(input_file):
+    failing_tests = []
+    # Buffer to keep track of the last 8 lines
+    buffer = []
+    with open(input_file, "r") as file:
+        for line in file:
+            buffer.append(line)
+            if len(buffer) > 8:
+                buffer.pop(0)
+            if "RunResources has no remaining steps." in line:
+                # Extract the test name, which is 7 lines above the error message
+                try:
+                    test_name_line = buffer[
+                        -8
+                    ]  # The 8th item from the end is 7 lines above the error line
+                    # Extract the test name from the line
+                    if "reverted:" in test_name_line:
+                        test_name = (
+                            test_name_line.split("reverted:")[0].split("::")[-1].strip()
+                        )
+                        failing_tests.append(test_name)
+                except IndexError:
+                    # This happens if the error is found within the first 7 lines of the file
+                    # or the buffer doesn't have enough lines yet, just skip it
+                    continue
+    return failing_tests
+
+
 def parse_and_write_to_yaml(input_file, output_file):
     with open(input_file, "r") as f:
         data = f.read()
@@ -20,6 +48,7 @@ def parse_and_write_to_yaml(input_file, output_file):
         )
         for m in re.findall(r"thread '(.*)' panicked at", data)
     ]
+
     summary = next(
         re.finditer(
             r"test result: (?P<result>\w+). (?P<passed>\d+) passed; (?P<failed>\d+) failed; (?P<ignored>\d+) ignored",
@@ -30,6 +59,8 @@ def parse_and_write_to_yaml(input_file, output_file):
     if len(matches_failed) != int(summary["failed"]):
         raise ValueError("Failed to parse file")
 
+    runresources_errors = extract_runresource_failures(input_file)
+
     skip_dict = defaultdict(list)
     for [folder, file] in matches_failed:
         skip_dict[folder].append(file)
@@ -39,7 +70,10 @@ def parse_and_write_to_yaml(input_file, output_file):
         skip += f"  {folder}:\n"
         skip_dict[folder] = sorted(skip_dict[folder])
         for file in skip_dict[folder]:
-            skip += f"      - {file[5:]}\n"
+            if file[5:] in runresources_errors:
+                skip += f"      - {file[5:]}  #RunResources error\n"
+            else:
+                skip += f"      - {file[5:]}\n"
 
     with open(output_file, "w") as f:
         f.write(skip)
@@ -51,7 +85,6 @@ def parse_and_write_to_yaml(input_file, output_file):
     )
     parser.add_argument("input_file", help="Input file path")
     parser.add_argument("output_file", help="Output file path")
-
     args = parser.parse_args()
 
     input_file = args.input_file