From c32b5064144020ac1bd8fa5bffdabb04ea1b08fa Mon Sep 17 00:00:00 2001 From: Mike Karlesky Date: Tue, 14 May 2024 11:06:29 -0400 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Crash=20detection=20instea?= =?UTF-8?q?d=20of=20only=20segfault=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Renamed LogLabels to address crashes - Reworked test executable handling for general crash detection and useful logging - Updated spec tests for segfault detection to reference crash detection instead --- ...le_sigsegv.c => test_example_file_crash.c} | 2 +- lib/ceedling/constants.rb | 2 +- lib/ceedling/generator.rb | 43 +++++++------ lib/ceedling/generator_helper.rb | 60 +++++++++++++------ lib/ceedling/loginator.rb | 4 +- spec/gcov/gcov_test_cases_spec.rb | 32 +++++----- spec/spec_system_helper.rb | 58 +++++++++--------- 7 files changed, 116 insertions(+), 85 deletions(-) rename assets/{test_example_file_sigsegv.c => test_example_file_crash.c} (91%) diff --git a/assets/test_example_file_sigsegv.c b/assets/test_example_file_crash.c similarity index 91% rename from assets/test_example_file_sigsegv.c rename to assets/test_example_file_crash.c index e10a6ec6..dcc86066 100644 --- a/assets/test_example_file_sigsegv.c +++ b/assets/test_example_file_crash.c @@ -18,7 +18,7 @@ void test_add_numbers_adds_numbers(void) { } void test_add_numbers_will_fail(void) { - // Platform-independent way of creating a segmentation fault + // Platform-independent way of forcing a crash uint32_t* nullptr = (void*) 0; uint32_t i = *nullptr; TEST_ASSERT_EQUAL_INT(2, add_numbers(i,2)); diff --git a/lib/ceedling/constants.rb b/lib/ceedling/constants.rb index 8c09afdd..afe410c8 100644 --- a/lib/ceedling/constants.rb +++ b/lib/ceedling/constants.rb @@ -34,7 +34,7 @@ class LogLabels EXCEPTION = 5 # decorator + 'EXCEPTION:' CONSTRUCT = 6 # decorator only RUN = 7 # decorator only - SEGFAULT = 8 # decorator only + CRASH = 8 # decorator only PASS = 9 # decorator only FAIL = 10 # decorator only TITLE = 11 # decorator only diff --git a/lib/ceedling/generator.rb b/lib/ceedling/generator.rb index 996c1e81..6a9f0f21 100644 --- a/lib/ceedling/generator.rb +++ b/lib/ceedling/generator.rb @@ -29,6 +29,11 @@ class Generator :unity_utils + def setup() + # Alias + @helper = @generator_helper + end + def generate_mock(context:, mock:, test:, input_filepath:, output_path:) arg_hash = { :header_file => input_filepath, @@ -299,42 +304,44 @@ def generate_test_results(tool:, context:, test_name:, test_filepath:, executabl # Apply additional test case filters command[:line] += @unity_utils.collect_test_runner_additional_args - # Enable collecting GCOV results even when segmenatation fault is appearing - # The gcda and gcno files will be generated for a test cases which doesn't - # cause segmentation fault + # Enable collecting GCOV results even for crashes + # The gcda and gcno files will be generated for test executable that doesn't cause a crash @debugger_utils.enable_gcov_with_gdb_and_cmdargs(command) - # Run the test itself (allow it to fail. we'll analyze it in a moment) + # Run the test executable itself + # We allow it to fail without an exception. + # We'll analyze its results apart from tool_executor command[:options][:boom] = false shell_result = @tool_executor.exec( command ) - # Handle SegFaults - if @tool_executor.segfault?( shell_result ) - @loginator.log( "Test executable #{test_name} encountered a segmentation fault", Verbosity::OBNOXIOUS, LogLabels::SEGFAULT ) + # Handle crashes + if @helper.test_crash?( shell_result ) + @helper.log_test_results_crash( test_name, executable, shell_result ) + if @configurator.project_config_hash[:project_use_backtrace] && @configurator.project_config_hash[:test_runner_cmdline_args] # If we have the options and tools to learn more, dig into the details - shell_result = @debugger_utils.gdb_output_collector(shell_result) + shell_result = @debugger_utils.gdb_output_collector( shell_result ) else - # Otherwise, call a segfault a single failure so it shows up in the report + # Otherwise, call a crash a single failure so it shows up in the report source = File.basename(executable).ext(@configurator.extension_source) - shell_result[:output] = "#{source}:1:test_Unknown:FAIL:Segmentation Fault" + shell_result[:output] = "#{source}:1:test_Unknown:FAIL:Test Executable Crashed" shell_result[:output] += "\n-----------------------\n1 Tests 1 Failures 0 Ignored\nFAIL\n" shell_result[:exit_code] = 1 end - else - # Don't Let The Failure Count Make Us Believe Things Aren't Working - @generator_helper.test_results_error_handler(executable, shell_result) end - processed = @generator_test_results.process_and_write_results( shell_result, - arg_hash[:result_file], - @file_finder.find_test_from_file_path(arg_hash[:executable]) ) + processed = @generator_test_results.process_and_write_results( + shell_result, + arg_hash[:result_file], + @file_finder.find_test_from_file_path(arg_hash[:executable]) + ) arg_hash[:result_file] = processed[:result_file] arg_hash[:results] = processed[:results] - arg_hash[:shell_result] = shell_result # for raw output display if no plugins for formatted display + # For raw output display if no plugins enabled for nice display + arg_hash[:shell_result] = shell_result - @plugin_manager.post_test_fixture_execute(arg_hash) + @plugin_manager.post_test_fixture_execute( arg_hash ) end end diff --git a/lib/ceedling/generator_helper.rb b/lib/ceedling/generator_helper.rb index afdfa7ee..6efdfbe2 100644 --- a/lib/ceedling/generator_helper.rb +++ b/lib/ceedling/generator_helper.rb @@ -13,32 +13,54 @@ class GeneratorHelper constructor :loginator + def test_crash?(shell_result) + return true if (shell_result[:output] =~ /\s*Segmentation\sfault.*/i) - def test_results_error_handler(executable, shell_result) - notice = '' - error = false - + # Unix Signal 11 ==> SIGSEGV + # Applies to Unix-like systems including MSYS on Windows + return true if (shell_result[:status].termsig == 11) + + # No test results found in test executable output + return true if (shell_result[:output] =~ TEST_STDOUT_STATISTICS_PATTERN).nil? + + return false + end + + def log_test_results_crash(test_name, executable, shell_result) + runner = File.basename(executable) + + notice = "Test executable #{test_name} [`#{runner}`] seems to have crashed" + @loginator.log( notice, Verbosity::ERRORS, LogLabels::CRASH ) + + log = false + + # Check for empty output if (shell_result[:output].nil? or shell_result[:output].strip.empty?) - error = true - # mirror style of generic tool_executor failure output - notice = "Test executable \"#{File.basename(executable)}\" failed.\n" + - "> Produced no output to $stdout.\n" + # Mirror style of generic tool_executor failure output + notice = "Test executable `#{runner}` failed.\n" + + "> Produced no output\n" + + log = true + + # Check for no test results elsif ((shell_result[:output] =~ TEST_STDOUT_STATISTICS_PATTERN).nil?) - error = true - # mirror style of generic tool_executor failure output - notice = "Test executable \"#{File.basename(executable)}\" failed.\n" + - "> Produced no final test result counts in $stdout:\n" + - "#{shell_result[:output].strip}\n" + # Mirror style of generic tool_executor failure output + notice = "Test executable `#{runner}` failed.\n" + + "> Output contains no test result counts\n" + + log = true end - if (error) - # since we told the tool executor to ignore the exit code, handle it explicitly here - notice += "> And exited with status: [#{shell_result[:exit_code]}] (count of failed tests).\n" if (shell_result[:exit_code] != nil) - notice += "> And then likely crashed.\n" if (shell_result[:exit_code] == nil) + if (log) + if (shell_result[:exit_code] != nil) + notice += "> And terminated with exit code: [#{shell_result[:exit_code]}] (failed test case count).\n" + end - notice += "> This is often a symptom of a bad memory access in source or test code.\n\n" + notice += "> Causes can include a bad memory access, stack overflow, heap error, or bad branch in source or test code.\n" - raise CeedlingException.new( notice ) + @loginator.log( '', Verbosity::OBNOXIOUS ) + @loginator.log( notice, Verbosity::OBNOXIOUS, LogLabels::ERROR ) + @loginator.log( '', Verbosity::OBNOXIOUS ) end end diff --git a/lib/ceedling/loginator.rb b/lib/ceedling/loginator.rb index eb396b2b..92fbc4f7 100644 --- a/lib/ceedling/loginator.rb +++ b/lib/ceedling/loginator.rb @@ -129,7 +129,7 @@ def decorate(str, label=LogLabels::NONE) prepend = '๐Ÿงจ ' when LogLabels::CONSTRUCT prepend = '๐Ÿšง ' - when LogLabels::SEGFAULT + when LogLabels::CRASH prepend = 'โ˜ ๏ธ ' when LogLabels::RUN prepend = '๐Ÿ‘Ÿ ' @@ -212,6 +212,8 @@ def format(string, verbosity, label, decorate) prepend += 'ERROR: ' when LogLabels::EXCEPTION prepend += 'EXCEPTION: ' + when LogLabels::CRASH + prepend += 'ERROR: ' # Otherwise no headings for decorator-only messages end diff --git a/spec/gcov/gcov_test_cases_spec.rb b/spec/gcov/gcov_test_cases_spec.rb index e9e9fce6..6aef13bd 100644 --- a/spec/gcov/gcov_test_cases_spec.rb +++ b/spec/gcov/gcov_test_cases_spec.rb @@ -206,19 +206,19 @@ def can_create_gcov_html_report_from_crashing_test_runner_with_enabled_debug_and Dir.chdir @proj_name do FileUtils.cp test_asset_path("example_file.h"), 'src/' FileUtils.cp test_asset_path("example_file.c"), 'src/' - FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' + FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/' FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml' @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling gcov:all 2>&1` - expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called - expect(output).to match(/Segmentation fault/i) + expect($?.exitstatus).to match(1) # Test should fail because of crash + expect(output).to match(/Test Executable Crashed/i) expect(output).to match(/Unit test failures./) - expect(File.exist?('./build/gcov/results/test_example_file_sigsegv.fail')) - output_rd = File.read('./build/gcov/results/test_example_file_sigsegv.fail') - expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ ) + expect(File.exist?('./build/gcov/results/test_example_file_crash.fail')) + output_rd = File.read('./build/gcov/results/test_example_file_crash.fail') + expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ ) expect(output).to match(/TESTED:\s+2/) expect(output).to match(/PASSED:\s+(?:0|1)/) expect(output).to match(/FAILED:\s+(?:1|2)/) @@ -237,19 +237,19 @@ def can_create_gcov_html_report_from_crashing_test_runner_with_enabled_debug_and Dir.chdir @proj_name do FileUtils.cp test_asset_path("example_file.h"), 'src/' FileUtils.cp test_asset_path("example_file.c"), 'src/' - FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' + FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/' FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml' @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling gcov:all --exclude_test_case=test_add_numbers_adds_numbers 2>&1` - expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called - expect(output).to match(/Segmentation fault/i) + expect($?.exitstatus).to match(1) # Test should fail because of crash + expect(output).to match(/Test Executable Crashed/i) expect(output).to match(/Unit test failures./) - expect(File.exist?('./build/gcov/results/test_example_file_sigsegv.fail')) - output_rd = File.read('./build/gcov/results/test_example_file_sigsegv.fail') - expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ ) + expect(File.exist?('./build/gcov/results/test_example_file_crash.fail')) + output_rd = File.read('./build/gcov/results/test_example_file_crash.fail') + expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ ) expect(output).to match(/TESTED:\s+1/) expect(output).to match(/PASSED:\s+0/) expect(output).to match(/FAILED:\s+1/) @@ -269,7 +269,7 @@ def can_create_gcov_html_report_from_test_runner_with_enabled_debug_and_cmd_args Dir.chdir @proj_name do FileUtils.cp test_asset_path("example_file.h"), 'src/' FileUtils.cp test_asset_path("example_file.c"), 'src/' - FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' + FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/' FileUtils.cp test_asset_path("project_with_guts_gcov.yml"), 'project.yml' @c.merge_project_yml_for_test({:test_runner => { :cmdline_args => true }}) @@ -279,13 +279,13 @@ def can_create_gcov_html_report_from_test_runner_with_enabled_debug_and_cmd_args " TEST_ASSERT_EQUAL_INT(0, difference_between_numbers(1,1));\n" \ "}\n" - updated_test_file = File.read('test/test_example_file_sigsegv.c').split("\n") + updated_test_file = File.read('test/test_example_file_crash.c').split("\n") updated_test_file.insert(updated_test_file.length(), add_test_case) - File.write('test/test_example_file_sigsegv.c', updated_test_file.join("\n"), mode: 'w') + File.write('test/test_example_file_crash.c', updated_test_file.join("\n"), mode: 'w') output = `bundle exec ruby -S ceedling gcov:all --exclude_test_case=test_add_numbers_will_fail 2>&1` expect($?.exitstatus).to match(0) - expect(File.exist?('./build/gcov/results/test_example_file_sigsegv.pass')) + expect(File.exist?('./build/gcov/results/test_example_file_crash.pass')) expect(output).to match(/TESTED:\s+2/) expect(output).to match(/PASSED:\s+2/) expect(output).to match(/FAILED:\s+0/) diff --git a/spec/spec_system_helper.rb b/spec/spec_system_helper.rb index 202bb3cf..3e2c5a0d 100644 --- a/spec/spec_system_helper.rb +++ b/spec/spec_system_helper.rb @@ -701,38 +701,38 @@ def run_all_test_when_test_case_name_is_passed_it_will_autoset_cmdline_args end - def test_run_of_projects_fail_because_of_sigsegv_without_report + def test_run_of_projects_fail_because_of_crash_without_report @c.with_context do Dir.chdir @proj_name do FileUtils.cp test_asset_path("example_file.h"), 'src/' FileUtils.cp test_asset_path("example_file.c"), 'src/' - FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' + FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/' output = `bundle exec ruby -S ceedling test:all 2>&1` - expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called - expect(output).to match(/Segmentation Fault/i) + expect($?.exitstatus).to match(1) # Test should fail because of crash + expect(output).to match(/Test Executable Crashed/i) expect(output).to match(/Unit test failures./) expect(!File.exist?('./build/test/results/test_add.fail')) end end end - def test_run_of_projects_fail_because_of_sigsegv_with_report + def test_run_of_projects_fail_because_of_crash_with_report @c.with_context do Dir.chdir @proj_name do FileUtils.cp test_asset_path("example_file.h"), 'src/' FileUtils.cp test_asset_path("example_file.c"), 'src/' - FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' + FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/' @c.merge_project_yml_for_test({:project => { :use_backtrace => true }}) output = `bundle exec ruby -S ceedling test:all 2>&1` - expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called - expect(output).to match(/Segmentation Fault/i) + expect($?.exitstatus).to match(1) # Test should fail because of crash + expect(output).to match(/Test Executable Crashed/i) expect(output).to match(/Unit test failures./) - expect(File.exist?('./build/test/results/test_example_file_sigsegv.fail')) - output_rd = File.read('./build/test/results/test_example_file_sigsegv.fail') - expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ ) + expect(File.exist?('./build/test/results/test_example_file_crash.fail')) + output_rd = File.read('./build/test/results/test_example_file_crash.fail') + expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ ) end end end @@ -742,18 +742,18 @@ def execute_all_test_cases_from_crashing_test_runner_and_return_test_report_with Dir.chdir @proj_name do FileUtils.cp test_asset_path("example_file.h"), 'src/' FileUtils.cp test_asset_path("example_file.c"), 'src/' - FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' + FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/' @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling test:all 2>&1` - expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called - expect(output).to match(/Segmentation fault/i) + expect($?.exitstatus).to match(1) # Test should fail because of crash + expect(output).to match(/Test Executable Crashed/i) expect(output).to match(/Unit test failures./) - expect(File.exist?('./build/test/results/test_example_file_sigsegv.fail')) - output_rd = File.read('./build/test/results/test_example_file_sigsegv.fail') - expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ ) + expect(File.exist?('./build/test/results/test_example_file_crash.fail')) + output_rd = File.read('./build/test/results/test_example_file_crash.fail') + expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ ) expect(output).to match(/TESTED:\s+2/) expect(output).to match(/PASSED:\s+(?:0|1)/) expect(output).to match(/FAILED:\s+(?:1|2)/) @@ -767,18 +767,18 @@ def execute_and_collect_debug_logs_from_crashing_test_case_defined_by_test_case_ Dir.chdir @proj_name do FileUtils.cp test_asset_path("example_file.h"), 'src/' FileUtils.cp test_asset_path("example_file.c"), 'src/' - FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' + FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/' @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling test:all --test_case=test_add_numbers_will_fail 2>&1` - expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called - expect(output).to match(/Segmentation fault/i) + expect($?.exitstatus).to match(1) # Test should fail because of crash + expect(output).to match(/Test Executable Crashed/i) expect(output).to match(/Unit test failures./) - expect(File.exist?('./build/test/results/test_example_file_sigsegv.fail')) - output_rd = File.read('./build/test/results/test_example_file_sigsegv.fail') - expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ ) + expect(File.exist?('./build/test/results/test_example_file_crash.fail')) + output_rd = File.read('./build/test/results/test_example_file_crash.fail') + expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ ) expect(output).to match(/TESTED:\s+1/) expect(output).to match(/PASSED:\s+(?:0|1)/) expect(output).to match(/FAILED:\s+(?:1|2)/) @@ -792,18 +792,18 @@ def execute_and_collect_debug_logs_from_crashing_test_case_defined_by_exclude_te Dir.chdir @proj_name do FileUtils.cp test_asset_path("example_file.h"), 'src/' FileUtils.cp test_asset_path("example_file.c"), 'src/' - FileUtils.cp test_asset_path("test_example_file_sigsegv.c"), 'test/' + FileUtils.cp test_asset_path("test_example_file_crash.c"), 'test/' @c.merge_project_yml_for_test({:project => { :use_backtrace => true }, :test_runner => { :cmdline_args => true }}) output = `bundle exec ruby -S ceedling test:all --exclude_test_case=add_numbers_adds_numbers 2>&1` - expect($?.exitstatus).to match(1) # Test should fail as sigsegv is called - expect(output).to match(/Segmentation fault/i) + expect($?.exitstatus).to match(1) # Test should fail because of crash + expect(output).to match(/Test Executable Crashed/i) expect(output).to match(/Unit test failures./) - expect(File.exist?('./build/test/results/test_example_file_sigsegv.fail')) - output_rd = File.read('./build/test/results/test_example_file_sigsegv.fail') - expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_sigsegv.c\:14/ ) + expect(File.exist?('./build/test/results/test_example_file_crash.fail')) + output_rd = File.read('./build/test/results/test_example_file_crash.fail') + expect(output_rd =~ /test_add_numbers_will_fail \(\) at test\/test_example_file_crash.c\:14/ ) expect(output).to match(/TESTED:\s+1/) expect(output).to match(/PASSED:\s+(?:0|1)/) expect(output).to match(/FAILED:\s+(?:1|2)/)