From 7ab02ecee52b040ac6fbfd6733f8dba5cb75f2d1 Mon Sep 17 00:00:00 2001 From: Bhargava Shastry Date: Wed, 9 Nov 2016 14:57:26 +0100 Subject: [PATCH] Tests added and dead code removed; afl-sancov will no longer support vanilla coverage of AFL queue input (use afl-cov) --- afl-sancov.py | 439 +----------------- tests/afl-sancov-generator.sh | 4 +- tests/aflsancov.py | 1 - ...0000,sig:06,src:000003,op:havoc,rep:2.json | 13 + ...0000,sig:06,src:000003,op:havoc,rep:4.json | 13 + ...0000,sig:06,src:000003,op:havoc,rep:2.json | 0 ...0000,sig:06,src:000003,op:havoc,rep:4.json | 0 ...0000,sig:06,src:000003,op:havoc,rep:2.json | 12 + ...0000,sig:06,src:000003,op:havoc,rep:4.json | 12 + ...0000,sig:06,src:000003,op:havoc,rep:2.json | 0 ...0000,sig:06,src:000003,op:havoc,rep:4.json | 0 tests/run.py | 138 ------ tests/test-afl-sancov.py | 402 ++++++++-------- ...id:000000,sig:06,src:000003,op:havoc,rep:2 | 0 ...id:000000,sig:06,src:000003,op:havoc,rep:4 | 0 15 files changed, 253 insertions(+), 781 deletions(-) delete mode 120000 tests/aflsancov.py create mode 100644 tests/expects/ddmode/asan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json create mode 100644 tests/expects/ddmode/asan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json rename tests/expects/ddmode/{ => ubsan}/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json (100%) rename tests/expects/ddmode/{ => ubsan}/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json (100%) create mode 100644 tests/expects/ddnum/asan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json create mode 100644 tests/expects/ddnum/asan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json rename tests/expects/ddnum/{ => ubsan}/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json (100%) rename tests/expects/ddnum/{ => ubsan}/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json (100%) delete mode 100755 tests/run.py rename tests/{afl-out => }/unique/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2 (100%) rename tests/{afl-out => }/unique/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4 (100%) diff --git a/afl-sancov.py b/afl-sancov.py index e6e37bb..fb43f5a 100755 --- a/afl-sancov.py +++ b/afl-sancov.py @@ -2,7 +2,7 @@ # # File: afl-sancov # -# Version: 0.1 +# Version: 1.1 # # Purpose: Leverage sancov towards coverage consolidation, program spec analysis etc. # @@ -46,8 +46,8 @@ class AFLSancovReporter: """Base class for the AFL Sancov reporter""" - Version = '1.0' - Description = 'A tool for coverage consolidation and program spectrum analysis' + Version = '1.1' + Description = 'A tool for spectrum based fault localization' Want_Output = True No_Output = False @@ -115,13 +115,10 @@ def run(self): self.setup_parsing() - if not self.args.dd_mode: - rv = self.process_afl_queue() + if self.args.dd_num == 1: + rv = self.process_afl_crashes() else: - if self.args.dd_num == 1: - rv = self.process_afl_crashes() - else: - rv = self.process_afl_crashes_deep() + rv = self.process_afl_crashes_deep() return not rv @@ -305,7 +302,7 @@ def process_afl_crashes_deep(self): :return: ''' - crash_files = self.import_unique_crashes(self.unique_crash_path) + crash_files = self.import_unique_crashes(self.args.crash_dir) num_crash_files = len(crash_files) self.logr("\n*** Imported %d new crash files from: %s\n" \ @@ -391,7 +388,7 @@ def process_afl_crashes(self): :return: ''' - crash_files = self.import_unique_crashes(self.unique_crash_path) + crash_files = self.import_unique_crashes(self.args.crash_dir) num_crash_files = len(crash_files) self.logr("\n*** Imported %d new crash files from: %s\n" \ @@ -436,31 +433,6 @@ def process_afl_crashes(self): self.write_result_as_json(cbasename, pbasename) - ## Legacy code - # gp = self.linecov_report_to_str(self.crashdd_pos_report) - # - # # Extend the global list with current crash delta diff - # self.crashdd_pos_list.extend(self.crashdd_pos_report) - # - # crashdd_outfile = self.cov_paths['delta_diff_dir'] + '/' + cbasename + '.dd' - # - # header = "diff crash ({}) -> parent ({})".format(cbasename, pbasename) - # self.write_file(header, crashdd_outfile) - # self.write_strlist_to_file(gp, crashdd_outfile) - # - # ### Delete later - # parentdd_outfile = self.cov_paths['delta_diff_dir'] + '/' + pbasename + '.dd' - # header = "diff parent ({}) -> crash ({})".format(pbasename, cbasename) - # self.write_file(header, parentdd_outfile) - # - # self.parentdd_pos_report = self.prev_pos_report.difference(self.curr_pos_report) - # self.parentdd_pos_report = sorted(self.parentdd_pos_report, key=lambda cov_entry: cov_entry[0]) - # parentgp = self.linecov_report_to_str(self.parentdd_pos_report) - # - # self.write_strlist_to_file(parentgp, parentdd_outfile) - ## Legacy code - - # self.dd_obtain_stats() self.cleanup() return True @@ -516,161 +488,6 @@ def find_queue_parent(self, queue_fname): def find_parent_crashing(self, crash_fname): return self.get_parent(crash_fname) - def process_afl_queue(self): - - rv = True - has_run_once = False - - # Current AFL (queue) input filename - last_processed_file = '' - last_sancov_raw = '' - - # Used to track progress across multiple fuzzing dirs - num_processed_files = 0 - num_imported_files = 0 - prev_num_queue_files = 0 - dir_ctr = 0 - - # Legacy of --live mode - while True: - - if not self.import_afl_dirs(): - rv = False - break - - for fuzz_dir in self.cov_paths['dirs']: - - queue_id = 0 - new_files = self.import_test_cases(fuzz_dir + '/queue') - - # Record old queue length for test case count - if dir_ctr > 0: - prev_num_queue_files = num_queue_files - - num_queue_files = len(new_files) - num_imported_files += num_queue_files - dir_ctr += 1 - - if new_files: - self.logr("\n*** Imported %d new test cases from: %s\n" \ - % (num_queue_files, (fuzz_dir + '/queue'))) - - for f in new_files: - - out_lines = [] - # Since new_files is a sorted list of queue ids that is most likely - # named sequentially, `queue_id` is a good heuristic - # for guessing id contained in plot data - curr_cycle = self.get_cycle_num(fuzz_dir, queue_id) - queue_id += 1 - - self.logr("[+] AFL test case: %s (%d / %d), cycle: %d" \ - % (os.path.basename(f), (queue_id + prev_num_queue_files), num_imported_files, - curr_cycle)) - - self.gen_paths_queue(fuzz_dir, f) - - if dir_ctr > 1 and not self.cov_paths['dirs'][fuzz_dir]['prev_file']: - assert last_processed_file, "Last processed file is empty!" - self.cov_paths['dirs'][fuzz_dir]['prev_file'] = last_processed_file - self.cov_paths['dirs'][fuzz_dir]['prev_sancov_raw'] = last_sancov_raw - - assert self.args.coverage_cmd, "Missing coverage cmd! We shouldn't be here!" - - ### Dry run to eliminate queue inputs that crash program - cov_cmd = self.args.coverage_cmd.replace('AFL_FILE', f) - if self.does_dry_run_throw_error(cov_cmd): - self.logr("Queue file ({}) crashes the program! Ignoring.".format(f)) - continue - - ### execute the command to generate code coverage stats - ### for the current AFL test case file - sancov_env = self.get_sancov_env_for_afl_input(fuzz_dir, os.path.basename(f)) - if has_run_once: - self.run_cmd(cov_cmd, self.No_Output, sancov_env) - else: - out_lines = self.run_cmd(cov_cmd, self.Want_Output, sancov_env) - has_run_once = True - - ### Extract sancov stats for this test case - ### This writes pos and zero line cov reports to two distinct - ### reports and appends them to self.curr_reports. - self.extract_line_cov(fuzz_dir) - - ### Convert to gcov style report? - - ### Diff against global report and update global report with - ### new coverage info - if self.cov_paths['dirs'][fuzz_dir]['prev_file']: - self.coverage_diff(fuzz_dir, f) - else: - # Bootstrap global dict - self.global_pos_report = self.curr_pos_report - self.global_zero_report = self.curr_zero_report - - # if not cargs.disable_lcov_web and cargs.lcov_web_all: - # gen_web_cov_report(fuzz_dir, cov_paths, cargs) - - ### log the output of the very first coverage command to - ### assist in troubleshooting - if len(out_lines): - self.logr("\n\n++++++ BEGIN - first exec output for CMD: %s" % \ - (self.args.coverage_cmd.replace('AFL_FILE', f))) - for line in out_lines: - self.logr(" %s" % (line)) - self.logr("++++++ END\n") - - num_processed_files += 1 - - # We need last_processed files to link last processed file of one fuzz dir - # to first processed file of the next fuzz dir - last_processed_file = f - last_sancov_raw = self.cov_paths['dirs'][fuzz_dir]['sancov_raw'] - - # Back-up current queue file coverage info for diff against next queue file - # coverage. Note: This is a shallow copy because curr* reports are newly - # constructed each time. - self.prev_pos_report = self.curr_pos_report - self.prev_zero_report = self.curr_zero_report - - self.cov_paths['dirs'][fuzz_dir]['prev_file'] = f - - if self.args.afl_queue_id_limit \ - and queue_id > self.args.afl_queue_id_limit: - self.logr("[+] queue/ id limit of %d reached..." \ - % self.args.afl_queue_id_limit) - break - - break - - if num_processed_files > 0: - self.logr("[+] Processed %d / %d test cases.\n" \ - % (num_processed_files, num_imported_files)) - else: - if rv: - self.logr("[*] Did not find any AFL test cases, exiting.\n") - rv = False - - ### write out the final zero coverage and positive coverage reports - self.write_global_cov() - - # if not cargs.disable_lcov_web: - # gen_web_cov_report(fuzz_dir, cov_paths, cargs) - - return rv - - def write_global_cov(self): - self.global_pos_report = sorted(self.global_pos_report, \ - key=lambda cov_entry: cov_entry[0]) - self.global_zero_report = sorted(self.global_zero_report, \ - key=lambda cov_entry: cov_entry[0]) - gp = self.linecov_report_to_str(self.global_pos_report) - gz = self.linecov_report_to_str(self.global_zero_report) - - self.write_strlist_to_file(gp, self.cov_paths['pos_cov']) - self.write_strlist_to_file(gz, self.cov_paths['zero_cov']) - return - def init_tracking(self): self.cov_paths['top_dir'] = self.args.afl_fuzzing_dir + '/sancov' @@ -736,94 +553,6 @@ def import_afl_dirs(self): return True - def gen_paths_queue(self, fuzz_dir, afl_file): - - basename = os.path.basename(afl_file) - basedir = os.path.basename(fuzz_dir) - - cp = self.cov_paths['dirs'][fuzz_dir] - - # Create subdirs inside diff,web, and cons using basedir as folder name - for k in ['diff_dir', 'web_dir', 'cons_dir']: - if not self.is_dir(self.cov_paths[k] + '/' + basedir): - os.mkdir(self.cov_paths[k] + '/' + basedir) - - ### coverage diffs from one ID file to the next - cp['diff'] = self.cov_paths['diff_dir'] + '/' + basedir + '/' + basename - - ### current id:NNNNNN* test case file - cp['id_file'] = basedir + '/' + basename - - ### web files - cp['web_dir'] = self.cov_paths['web_dir'] + \ - '/' + basedir + '/' + basename - - ### raw sancov file - cp['sancov_raw'] = self.cov_paths['cons_dir'] + \ - '/' + basedir + '/' + basename + '.sancov' - - ### pos line cov - # cp['pos_line_cov'] = self.cov_paths['cons_dir'] + \ - # '/' + basedir + '/' + basename + '.pos_line_cov' - # - # ### zero line cov - # cp['zero_line_cov'] = self.cov_paths['cons_dir'] + \ - # '/' + basedir + '/' + basename + '.zero_line_cov' - # - # ### pos func cov - # cp['pos_func_cov'] = self.cov_paths['cons_dir'] + \ - # '/' + basedir + '/' + basename + '.pos_func_cov' - # - # ### zero func cov - # cp['zero_func_cov'] = self.cov_paths['cons_dir'] + \ - # '/' + basedir + '/' + basename + '.zero_func_cov' - - if cp['prev_file']: - cp['prev_sancov_raw'] = self.cov_paths['cons_dir'] + '/' \ - + basedir + '/' + os.path.basename(cp['prev_file']) \ - + '.sancov' - # cp['prev_pos_line_cov'] = self.cov_paths['cons_dir'] + '/' \ - # + basedir + '/' + os.path.basename(cp['prev_file']) \ - # + '.pos_line_cov' - # cp['prev_zero_line_cov'] = self.cov_paths['cons_dir'] + '/' \ - # + basedir + '/' + os.path.basename(cp['prev_file']) \ - # + '.zero_line_cov' - # cp['prev_pos_func_cov'] = self.cov_paths['cons_dir'] + '/' \ - # + basedir + '/' + os.path.basename(cp['prev_file']) \ - # + '.pos_func_cov' - # cp['prev_zero_func_cov'] = self.cov_paths['cons_dir'] + '/' \ - # + basedir + '/' + os.path.basename(cp['prev_file']) \ - # + '.zero_func_cov' - - return - - # def gen_paths_ddmode(self, afl_file): - # - # basename = os.path.basename(afl_file) - # - # cp = self.cov_paths - # - # ### raw sancov file - # cp['sancov_raw'] = self.cov_paths['delta_diff_dir'] + \ - # '/' + basename + '.sancov' - # - # ### For a single delta-diff run, prev_file should point to - # ### non-crashing input (parent of crash file) - # if cp['prev_file']: - # cp['prev_sancov_raw'] = self.cov_paths['delta_diff_dir'] + '/' \ - # + os.path.basename(cp['prev_file']) \ - # + '.sancov' - # - # return - - def get_sancov_env_for_afl_input(self, fuzz_dir, afl_input): - - cp = self.cov_paths['dirs'][fuzz_dir] - assert cp['sancov_raw'], "Attempting to write to non-existent " \ - "sancov raw file" - - return self.get_sancov_env(cp['sancov_raw'], afl_input) - def get_sancov_env(self, sancov_output, afl_input): fpath, fname = os.path.split(sancov_output) @@ -856,71 +585,6 @@ def get_sancov_env(self, sancov_output, afl_input): return sancov_env - def coverage_diff(self, fuzz_dir, afl_file): - - log_lines = [] - delta_log_lines = [] - - cp = self.cov_paths['dirs'][fuzz_dir] - - # Previous queue file and cons-cov dir - a_file = os.path.basename(cp['prev_file']) - a_dir = os.path.basename(os.path.dirname(cp['prev_sancov_raw'])) - - # Current queue file and cons-cov dir - b_file = os.path.basename(afl_file) - b_dir = os.path.basename(fuzz_dir) - - ### with the coverage from the previous sancov results extracted - ### the previous time we went through this function, we remove - ### associated files unless instructed to keep them - if not self.args.preserve_all_sancov_files: - self.rm_prev_cov_files(cp) - - ### We aren't interested in the number of times AFL has executed - ### a line or function (since we can't really get this anyway because - ### gcov stats aren't influenced by AFL directly) - what we want is - ### simply whether a new line or function has been executed by this - ### test case. So, we look for new positive coverage. - - # Log lines contain pos difference in coverage information between a_file (prev queue input) - # and b_file (curr queue input) - log_lines.append("diff %s/%s -> %s/%s" % (a_dir, a_file, b_dir, b_file)) - inter_queue_pos_diff = sorted(self.curr_pos_report.difference(self.prev_pos_report), \ - key=lambda cov_entry: cov_entry[0]) - log_lines += self.linecov_report_to_str(inter_queue_pos_diff) - - # Delta log lines contain pos difference in cov between b_file (curr queue input) - # and global cov info - accu_queue_pos_diff = sorted(self.curr_pos_report.difference(self.global_pos_report), \ - key=lambda cov_entry: cov_entry[0]) - delta_log_lines += self.linecov_report_to_str(accu_queue_pos_diff) - - # Update global cov - self.global_pos_report = self.global_pos_report.union(self.curr_pos_report) - self.global_zero_report = self.global_zero_report.intersection(self.curr_zero_report) - - if len(log_lines): - self.logr("\n Coverage diff %s/%s %s/%s" \ - % (a_dir, a_file, b_dir, b_file)) - for l in log_lines: - self.logr(l) - self.append_file(l, cp['diff']) - self.logr("") - - if len(delta_log_lines): - for l in delta_log_lines: - self.append_file(l, self.cov_paths['id_delta_cov']) - - return - - def extract_line_cov(self, fuzz_dir): - cp = self.cov_paths['dirs'][fuzz_dir] - # Extract coverage - if not self.rename_and_extract_linecov(cp['sancov_raw']): - return False - return True - # Rename ..sancov to user-supplied `sancov_fname` # Extract linecov info into self.curr* report def rename_and_extract_linecov(self, sancov_fname): @@ -986,14 +650,6 @@ def linecov_report(self, repstr): # Don't do this if you want to keep sets # return sorted(s, key=lambda cov_entry: cov_entry[0]) - def linecov_report_to_str(self, lcreport): - tempstr = [] - for (fp, func, ln, col) in lcreport: - tempstr.append("File: {}".format(fp)) - tempstr.append("Func: {}".format(func)) - tempstr.append("Line: {} Col: {}\n".format(ln, col)) - return tempstr - def find_sancov_file_and_rename(self, searchdir, newname): for filename in os.listdir(searchdir): @@ -1009,13 +665,6 @@ def find_sancov_file_and_rename(self, searchdir, newname): self.logr("Could not generate coverage info for parent {}. Bailing out!".format(newname)) return False - @staticmethod - def rm_prev_cov_files(ct): - prev_raw_filepath = ct['prev_sancov_raw'] - if os.path.exists(prev_raw_filepath): - os.remove(prev_raw_filepath) - return - # Credit: http://stackoverflow.com/a/1104641/4712439 def does_dry_run_throw_error(self, cmd): @@ -1055,26 +704,6 @@ def run_cmd(self, cmd, collect, env=None): return out - def get_cycle_num(self, fuzz_dir, id_num): - - ### default cycle - cycle_num = 0 - - if not os.path.isfile(fuzz_dir + '/plot_data'): - return cycle_num - - with open(fuzz_dir + '/plot_data') as f: - for line in f: - ### unix_time, cycles_done, cur_path, paths_total, pending_total,... - ### 1427742641, 11, 54, 419, 45, 0, 2.70%, 0, 0, 9, 1645.47 - vals = line.split(', ') - ### test the id number against the current path - if vals[2] == str(id_num): - cycle_num = int(vals[1]) - break - - return cycle_num - @staticmethod def import_test_cases(qdir): return sorted(glob.glob(qdir + "/id:*")) @@ -1095,27 +724,12 @@ def parse_cmdline(self, args): p.add_argument("--disable-cmd-redirection", action='store_true', help="Disable redirection of command results to /dev/null", default=False) - # p.add_argument("--disable-lcov-web", action='store_true', - # help="Disable generation of all lcov web code coverage reports", - # default=False) p.add_argument("--coverage-include-lines", action='store_true', help="Include lines in zero-coverage status files", default=False) - # p.add_argument("--enable-branch-coverage", action='store_true', - # help="Include branch coverage in code coverage reports (may be slow)", - # default=False) p.add_argument("--preserve-all-sancov-files", action='store_true', help="Keep all sancov files (not usually necessary)", default=False) - # p.add_argument("--disable-lcov-exclude-pattern", action='store_true', - # help="Allow default /usr/include/* pattern to be included in lcov results", - # default=False) - # p.add_argument("--lcov-exclude-pattern", type=str, - # help="Set exclude pattern for lcov results", - # default="/usr/include/\*") - p.add_argument("--afl-queue-id-limit", type=int, - help="Limit the number of id:NNNNNN* files processed in the AFL queue/ directory", - default=0) p.add_argument("-v", "--verbose", action='store_true', help="Verbose mode", default=False) p.add_argument("-V", "--version", action='store_true', @@ -1135,20 +749,14 @@ def parse_cmdline(self, args): help="Path to llvm-symbolizer", default="llvm-symbolizer") p.add_argument("--bin-path", type=str, help="Path to coverage instrumented binary") - p.add_argument("--dd-mode", action='store_true', - help="Experimental! Enables delta debugging mode. In this mode, coverage traces of crashing input\n" - "and it's non-crashing parent are diff'ed.", - default=False) + p.add_argument("--crash-dir", type=str, + help="Path to unique AFL crashes post triage") p.add_argument("--dd-num", type=int, help="Experimental! Perform more compute intensive analysis of crashing input by comparing its" "path profile with aggregated path profiles of N=dd-num randomly selected non-crashing inputs", default=1) p.add_argument("--sancov-bug", action='store_true', help="Sancov bug that occurs for certain coverage_dir env vars", default=False) - # p.add_argument("--dd-raw-queue-path", type=str, - # help="Path to raw queue files (used in --dd-mode)") - # p.add_argument("--dd-crash-file", type=str, - # help="Path to crashing input for deep analysis (used in --dd-mode)") return p.parse_args(args) @@ -1165,11 +773,8 @@ def validate_args(self): print "[*] --afl-fuzzing-dir missing" return False - # TODO: Hard-coded path. Move to cmdline arg. - self.unique_crash_path = self.args.afl_fuzzing_dir + '/unique' - - if not os.path.isdir(self.unique_crash_path): - print "[*] There is no directory called 'unique' in --afl-fuzzing-dir" + if not self.args.crash_dir or not os.path.isdir(self.args.crash_dir): + print "[*] --crash-dir missing or not a dir" return False if not self.args.bin_path: @@ -1243,9 +848,8 @@ def init_mkdirs(self): if create_cov_dirs: for k in ['top_dir', 'web_dir', 'cons_dir', 'diff_dir']: os.mkdir(self.cov_paths[k]) - if self.args.dd_mode: - for k in ['delta_diff_dir', 'dd_stash_dir', 'dd_filter_dir']: - os.mkdir(self.cov_paths[k]) + for k in ['delta_diff_dir', 'dd_stash_dir', 'dd_filter_dir']: + os.mkdir(self.cov_paths[k]) ### write coverage results in the following format cfile = open(self.cov_paths['id_delta_cov'], 'w') @@ -1271,19 +875,6 @@ def append_file(pstr, path): f.close() return - @staticmethod - def write_file(str, file): - f = open(file, 'w') - f.write("%s\n" % str) - f.close() - return - - @staticmethod - def write_strlist_to_file(strlist, file): - with open(file, 'a') as thefile: - for item in strlist: - print>> thefile, item - @classmethod def write_status(cls, status_file): f = open(status_file, 'w') @@ -1296,4 +887,4 @@ def write_status(cls, status_file): if __name__ == "__main__": reporter = AFLSancovReporter(sys.argv[1:]) - sys.exit(reporter.run()) + sys.exit(reporter.run()) \ No newline at end of file diff --git a/tests/afl-sancov-generator.sh b/tests/afl-sancov-generator.sh index 0452c75..89b4cd8 100755 --- a/tests/afl-sancov-generator.sh +++ b/tests/afl-sancov-generator.sh @@ -4,6 +4,8 @@ echo -e "\t[+] Generating coverage information for test-sancov.c" rm -f test-sancov clang-3.8 -O0 -g -fsanitize=undefined -fsanitize-coverage=edge \ - test-sancov.c -o test-sancov + test-sancov.c -o test-sancov-ubsan +clang-3.8 -O0 -g -fsanitize=address -fsanitize-coverage=edge \ + test-sancov.c -o test-sancov-asan exit 0 diff --git a/tests/aflsancov.py b/tests/aflsancov.py deleted file mode 120000 index ae6e1d0..0000000 --- a/tests/aflsancov.py +++ /dev/null @@ -1 +0,0 @@ -../afl-sancov.py \ No newline at end of file diff --git a/tests/expects/ddmode/asan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json b/tests/expects/ddmode/asan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json new file mode 100644 index 0000000..29658ca --- /dev/null +++ b/tests/expects/ddmode/asan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json @@ -0,0 +1,13 @@ +{ + "shrink-percent": 80.0, + "dice-linecount": 1, + "slice-linecount": 5, + "diff-node-spec": [ + { + "count": 1, + "line": "/home/bhargava/work/gitlab/afl-sancov/tests/test-sancov.c:main:25:3" + } + ], + "crashing-input": "HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2", + "parent-input": "id:000003,src:000001,op:havoc,rep:4,+cov" +} \ No newline at end of file diff --git a/tests/expects/ddmode/asan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json b/tests/expects/ddmode/asan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json new file mode 100644 index 0000000..cc6a6d9 --- /dev/null +++ b/tests/expects/ddmode/asan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json @@ -0,0 +1,13 @@ +{ + "shrink-percent": 80.0, + "dice-linecount": 1, + "slice-linecount": 5, + "diff-node-spec": [ + { + "count": 1, + "line": "/home/bhargava/work/gitlab/afl-sancov/tests/test-sancov.c:main:25:3" + } + ], + "crashing-input": "HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4", + "parent-input": "id:000003,sync:SESSION000,src:000003,+cov" +} \ No newline at end of file diff --git a/tests/expects/ddmode/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json b/tests/expects/ddmode/ubsan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json similarity index 100% rename from tests/expects/ddmode/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json rename to tests/expects/ddmode/ubsan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json diff --git a/tests/expects/ddmode/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json b/tests/expects/ddmode/ubsan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json similarity index 100% rename from tests/expects/ddmode/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json rename to tests/expects/ddmode/ubsan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json diff --git a/tests/expects/ddnum/asan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json b/tests/expects/ddnum/asan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json new file mode 100644 index 0000000..f3f7488 --- /dev/null +++ b/tests/expects/ddnum/asan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json @@ -0,0 +1,12 @@ +{ + "shrink-percent": 66.66666666666667, + "crashing-input": "HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2", + "dice-linecount": 1, + "diff-node-spec": [ + { + "count": 3, + "line": "/home/bhargava/work/gitlab/afl-sancov/tests/test-sancov.c:main:25:3" + } + ], + "slice-linecount": 3 +} \ No newline at end of file diff --git a/tests/expects/ddnum/asan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json b/tests/expects/ddnum/asan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json new file mode 100644 index 0000000..7cc370b --- /dev/null +++ b/tests/expects/ddnum/asan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json @@ -0,0 +1,12 @@ +{ + "shrink-percent": 66.66666666666667, + "crashing-input": "HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4", + "dice-linecount": 1, + "diff-node-spec": [ + { + "count": 3, + "line": "/home/bhargava/work/gitlab/afl-sancov/tests/test-sancov.c:main:25:3" + } + ], + "slice-linecount": 3 +} \ No newline at end of file diff --git a/tests/expects/ddnum/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json b/tests/expects/ddnum/ubsan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json similarity index 100% rename from tests/expects/ddnum/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json rename to tests/expects/ddnum/ubsan/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json diff --git a/tests/expects/ddnum/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json b/tests/expects/ddnum/ubsan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json similarity index 100% rename from tests/expects/ddnum/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json rename to tests/expects/ddnum/ubsan/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json diff --git a/tests/run.py b/tests/run.py deleted file mode 100755 index 46bf32b..0000000 --- a/tests/run.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python2 - -from aflsancov import * -import argparse -import sys, os -try: - import subprocess32 as subprocess -except ImportError: - import subprocess - -def main(): - - reporter = AFLSancovReporter() - # cargs = reporter.parse_cmdline() - - ### config - tmp_file = './tmp_cmd.out' - test_cmd = './test-afl-sancov.py' - - ### the AFL test cases in the test suite are built against this - ### commit in the fwknop code base - # fwknop_commit = 'e3ae6747' - - # fwknop_codecov_dir = 'fwknop-codecov.git' - # fwknop_codecov_compile = './compile/afl-compile-code-coverage.sh' - - # fwknop_afl_dir = 'fwknop-afl.git' - # fwknop_afl_compile = './compile/afl-compile.sh' - - ### system commands that we require - cmds = { -# 'git':'', - 'pysancov':'', - 'sancov-3.8':'', - 'llvm-symbolizer-3.8':'', - 'python':'', - } - - print "[+] Starting up afl-sancov test suite..." - - ### make sure required system binaries are installed - for cmd in cmds: - cmds[cmd] = reporter.which(cmd) - if not cmds[cmd]: - print "[*] Could not find command '%s', exiting." % (cmd) - return False - print "[+] Required binaries exist." - - ### clone the fwknop repository since the test suite operates - ### against fwknop code - # if not is_dir(fwknop_codecov_dir): - # print "[+] (Code cov) cloning fwknop repo: %s" % (cargs.fwknop_git) - # do_cmd("%s clone %s %s" % (cmds['git'], - # cargs.fwknop_git, fwknop_codecov_dir), None, cargs) - # - # if not is_dir(fwknop_codecov_dir): - # print "[*] Could not clone %s, set a different --fwknop-git path?" - # return - # - # if not is_dir(fwknop_afl_dir): - # print "[+] (AFL support) Cloning fwknop repo: %s" % (cargs.fwknop_git) - # do_cmd("%s clone %s %s" % (cmds['git'], - # fwknop_codecov_dir, fwknop_afl_dir), None, cargs) - # - # ### build both fwknop repositories under the specified commit - # build_fwknop(fwknop_codecov_dir, fwknop_commit, - # fwknop_codecov_compile, cmds, cargs) - # build_fwknop(fwknop_afl_dir, fwknop_commit, - # fwknop_afl_compile, cmds, cargs) - - ### run the actual tests - print "[+] Running afl-sancov tests (ignore 'Terminated' messages)..." - rv = subprocess.call("%s %s" % (cmds['python'], test_cmd), - stdin=None, shell=True) - - return rv - -# def build_fwknop(cdir, commit, compile_cmd, cmds, cargs): -# -# curr_dir = os.getcwd() -# os.chdir(cdir) -# if os.path.exists('./server/.libs/fwknopd'): -# do_cmd("%s clean" % (cmds['make']), None, cargs) -# do_cmd("%s checkout %s" % (cmds['git'], commit), None, cargs) -# do_cmd("./autogen.sh", None, cargs) -# -# print "[+] Compiling %s with test/afl/%s..." % (cdir, compile_cmd) -# os.chdir('./test/afl') -# do_cmd("%s" % (compile_cmd), None, cargs) -# os.chdir(curr_dir) -# -# return -# -# def do_cmd(cmd, tmp_file, cargs): -# -# out = [] -# -# if cargs.verbose: -# print " CMD: %s" % cmd -# -# fh = None -# if tmp_file: -# fh = open(tmp_file, 'w') -# else: -# fh = open(os.devnull, 'w') -# -# if cargs.verbose and not tmp_file: -# subprocess.call(cmd, stdin=None, shell=True) -# else: -# subprocess.call(cmd, stdin=None, -# stdout=fh, stderr=subprocess.STDOUT, shell=True) -# -# fh.close() -# -# if tmp_file: -# with open(tmp_file, 'r') as f: -# for line in f: -# out.append(line.rstrip('\n')) -# -# return out - -# def parse_cmdline(): -# -# p = argparse.ArgumentParser() -# -# p.add_argument("--fwknop-git", type=str, -# help="Location of fwknop git repository", -# default="https://github.com/mrash/fwknop.git") -# p.add_argument("--ignore-core-pattern", action='store_true', -# help="Ignore the /proc/sys/kernel/core_pattern setting in --live mode", -# default=False) -# p.add_argument("-v", "--verbose", action='store_true', -# help="Verbose mode", default=False) -# -# return p.parse_args() - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test-afl-sancov.py b/tests/test-afl-sancov.py index 03e01db..d9f5055 100755 --- a/tests/test-afl-sancov.py +++ b/tests/test-afl-sancov.py @@ -36,152 +36,193 @@ class TestAflSanCov(unittest.TestCase): ### set a few paths tmp_file = './tmp_cmd.out' - # version_file = '../VERSION' - # afl_cov_cmd = './aflsancov.py' - # single_generator = './afl-sancov-generator.sh' - # parallel_generator = './afl/afl-cov-generator-parallel.sh' - # afl_cov_live = './afl/afl-cov-generator-live.sh' top_out_dir = './afl-out' sancov_dir = top_out_dir + '/sancov' dd_dir = sancov_dir + '/delta-diff' expects_dir = './expects' - expects_ddmode_dir = expects_dir + '/ddmode' - expects_ddnum_dir = expects_dir + '/ddnum' + expects_ddmode_ubsan_dir = expects_dir + '/ddmode/ubsan' + expects_ddnum_ubsan_dir = expects_dir + '/ddnum/ubsan' + expects_ddmode_asan_dir = expects_dir + '/ddmode/asan' + expects_ddnum_asan_dir = expects_dir + '/ddnum/asan' dd_filename1 = '/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2.json' dd_filename2 = '/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4.json' dd_file1 = dd_dir + dd_filename1 dd_file2 = dd_dir + dd_filename2 - expects_ddmode_file1 = expects_ddmode_dir + dd_filename1 - expects_ddmode_file2 = expects_ddmode_dir + dd_filename2 - expects_ddnum_file1 = expects_ddnum_dir + dd_filename1 - expects_ddnum_file2 = expects_ddnum_dir + dd_filename2 + ## UBSAN + expects_ddmode_ubsan_file1 = expects_ddmode_ubsan_dir + dd_filename1 + expects_ddmode_ubsan_file2 = expects_ddmode_ubsan_dir + dd_filename2 + expects_ddnum_ubsan_file1 = expects_ddnum_ubsan_dir + dd_filename1 + expects_ddnum_ubsan_file2 = expects_ddnum_ubsan_dir + dd_filename2 + ## ASAN + expects_ddmode_asan_file1 = expects_ddmode_asan_dir + dd_filename1 + expects_ddmode_asan_file2 = expects_ddmode_asan_dir + dd_filename2 + expects_ddnum_asan_file1 = expects_ddnum_asan_dir + dd_filename1 + expects_ddnum_asan_file2 = expects_ddnum_asan_dir + dd_filename2 expected_line_substring = 'afl-sancov/tests/test-sancov.c:main:25:3' -# live_afl_cmd = './fuzzing-wrappers/server-access-redir.sh' -# live_parallel_afl_cmd = './fuzzing-wrappers/server-access-parallel-redir.sh' - - # def do_cmd(self, cmd): - # out = [] - # fh = open(self.tmp_file, 'w') - # subprocess.call(cmd, stdin=None, - # stdout=fh, stderr=subprocess.STDOUT, shell=True) - # fh.close() - # with open(self.tmp_file, 'r') as f: - # for line in f: - # out.append(line.rstrip('\n')) - # return out - def compare_json(self, file1, file2): + with open(file1) as data_file1: data1 = json.load(data_file1) with open(file2) as data_file2: data2 = json.load(data_file2) - if data1["shrink-percent"] != data2["shrink-percent"]: - return False - if data1["dice-linecount"] != data2["dice-linecount"]: - return False - if data1["slice-linecount"] != data2["slice-linecount"]: - return False - if data1["diff-node-spec"][0]["count"] != data2["diff-node-spec"][0]["count"]: - return False - if self.expected_line_substring not in data1["diff-node-spec"][0]["line"]: - return False - if data1["crashing-input"] != data2["crashing-input"]: - return False + + self.assertEqual(data1["shrink-percent"], data2["shrink-percent"], 'Shrink percent did not match') + self.assertEqual(data1["dice-linecount"], data2["dice-linecount"], 'Dice line count did not match') + self.assertEqual(data1["slice-linecount"], data2["slice-linecount"], 'Slice line count did not match') + self.assertEqual(data1["diff-node-spec"][0]["count"], data2["diff-node-spec"][0]["count"], + 'Dice frequency did not match') + self.assertTrue(self.expected_line_substring in data1["diff-node-spec"][0]["line"], + 'Dice line did not match') + self.assertEqual(data1["crashing-input"], data2["crashing-input"], 'Crashing input did not match') if 'parent-input' in data1 and 'parent-input' in data2: - if data1["parent-input"] != data2["parent-input"]: - return False + self.assertEqual(data1["parent-input"], data2["parent-input"], 'Parent input did not match') return True - ### start afl-cov in --live mode - this is for both single and - ### parallel instance testing - # def live_init(self): - # if is_dir(os.path.dirname(self.top_out_dir)): - # if is_dir(self.top_out_dir): - # rmtree(self.top_out_dir) - # else: - # if not is_dir(os.path.dirname(self.top_out_dir)): - # os.mkdir(os.path.dirname(self.top_out_dir)) - # - # ### start up afl-cov in the background before AFL is running - # try: - # subprocess.Popen([self.afl_cov_live]) - # except OSError: - # return False - # time.sleep(2) - # return True - - # def afl_stop(self): - # - # ### stop any afl-fuzz instances - # self.do_cmd("%s --stop-afl --afl-fuzzing-dir %s" \ - # % (self.afl_cov_cmd, self.top_out_dir)) - # - # ### now stop afl-cov - # afl_cov_pid = get_running_pid( - # self.top_out_dir + '/cov/afl-cov-status', - # 'afl_cov_pid\s+\:\s+(\d+)') - # if afl_cov_pid: - # os.kill(afl_cov_pid, signal.SIGTERM) - # - # return - def test_version(self): self.assertFalse(AFLSancovReporter(['--version']).run()) - # with open(self.version_file, 'r') as f: - # version = f.readline().rstrip() - # self.assertTrue(version - # in ''.join(self.do_cmd("%s --version" % (self.afl_cov_cmd))), - # "afl-sancov --version does not match VERSION file") - - # def test_help(self): - # self.assertTrue('--verbose' - # in ''.join(self.do_cmd("%s -h" % (self.afl_cov_cmd))), - # "--verbose not in -h output") def test_overwrite_dir(self): - args = ['-d', './afl-out', '-e "cat AFL_FILE | ./test-sancov"', '--bin-path={}/test-sancov'.format(os.getcwd()), + args = ['-d', './afl-out', '-e "cat AFL_FILE | ./test-sancov-ubsan"', '--bin-path={}/test-sancov-ubsan'.format(os.getcwd()), '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', - '--pysancov-path=/usr/local/bin/pysancov'] + '--pysancov-path=/usr/local/bin/pysancov', '--crash-dir={}/unique'.format(os.getcwd())] reporter = AFLSancovReporter(args) self.assertTrue(reporter.run()) + def test_ddmode_ubsan(self): + args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov-ubsan', '--bin-path={}/test-sancov-ubsan'.format(os.getcwd()), + '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', + '--pysancov-path=/usr/local/bin/pysancov', '--crash-dir={}/unique'.format(os.getcwd()),'--overwrite'] + reporter = AFLSancovReporter(args) + self.assertFalse(reporter.run()) + self.assertTrue(os.path.exists(self.dd_dir), + "No delta-diff dir generated during dd-mode invocation") + self.assertTrue((os.path.exists(self.dd_file1) and os.path.exists(self.dd_file2)), + "Missing delta-diff file(s) during dd-mode invocation") + + self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddmode_ubsan_file1), + "Delta-diff file {} does not match".format(self.dd_filename1)) + self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddmode_ubsan_file2), + "Delta-diff file {} does not match".format(self.dd_filename2)) + + def test_ddmode_ubsan_sancov_bug(self): + args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov-ubsan', '--bin-path={}/test-sancov-ubsan'.format(os.getcwd()), + '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', + '--pysancov-path=/usr/local/bin/pysancov', '--crash-dir={}/unique'.format(os.getcwd()),'--overwrite', + '--sancov-bug'] + reporter = AFLSancovReporter(args) + self.assertFalse(reporter.run()) + self.assertTrue(os.path.exists(self.dd_dir), + "No delta-diff dir generated during dd-mode invocation") + self.assertTrue((os.path.exists(self.dd_file1) and os.path.exists(self.dd_file2)), + "Missing delta-diff file(s) during dd-mode invocation") + + self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddmode_ubsan_file1), + "Delta-diff file {} does not match".format(self.dd_filename1)) + self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddmode_ubsan_file2), + "Delta-diff file {} does not match".format(self.dd_filename2)) + + def test_ddnum_ubsan(self): + args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov-ubsan', '--bin-path={}/test-sancov-ubsan'.format(os.getcwd()), + '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', + '--pysancov-path=/usr/local/bin/pysancov', '--overwrite', '--dd-num=3', + '--crash-dir={}/unique'.format(os.getcwd())] + reporter = AFLSancovReporter(args) + self.assertFalse(reporter.run()) + self.assertTrue(os.path.exists(self.dd_dir), + "No delta-diff dir generated during dd-num invocation") + self.assertTrue((os.path.exists(self.dd_file1) and os.path.exists(self.dd_file2)), + "Missing delta-diff file(s) during dd-num invocation") + + self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddnum_ubsan_file1), + "Delta-diff file {} does not match".format(self.dd_filename1)) + self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddnum_ubsan_file2), + "Delta-diff file {} does not match".format(self.dd_filename2)) + + def test_ddnum_ubsan_sancov_bug(self): + args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov-ubsan', + '--bin-path={}/test-sancov-ubsan'.format(os.getcwd()), + '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', + '--pysancov-path=/usr/local/bin/pysancov', '--overwrite', '--dd-num=3', + '--crash-dir={}/unique'.format(os.getcwd()), '--sancov-bug'] + reporter = AFLSancovReporter(args) + self.assertFalse(reporter.run()) + self.assertTrue(os.path.exists(self.dd_dir), + "No delta-diff dir generated during dd-num invocation") + self.assertTrue((os.path.exists(self.dd_file1) and os.path.exists(self.dd_file2)), + "Missing delta-diff file(s) during dd-num invocation") + + self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddnum_ubsan_file1), + "Delta-diff file {} does not match".format(self.dd_filename1)) + self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddnum_ubsan_file2), + "Delta-diff file {} does not match".format(self.dd_filename2)) - ### generate coverage, and then try to regenerate without --overwrite - # self.do_cmd("%s --afl-queue-id-limit 1 --overwrite" \ - # % (self.single_generator)) - # - # self.assertTrue(os.path.exists(self.sancov_dir), - # "No sancov dir generated during invocation") - # out_str = ''.join(self.do_cmd("%s --afl-queue-id-limit 1" \ - # % (self.single_generator))) - # self.assertTrue("use --overwrite" in out_str, - # "Missing --overwrite not caught") - def test_ddmode(self): - args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov', '--bin-path={}/test-sancov'.format(os.getcwd()), + def test_ddmode_asan(self): + args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov-asan', + '--bin-path={}/test-sancov-asan'.format(os.getcwd()), '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', - '--pysancov-path=/usr/local/bin/pysancov', '--overwrite', '--dd-mode'] + '--pysancov-path=/usr/local/bin/pysancov', '--crash-dir={}/unique'.format(os.getcwd()), '--overwrite', + '--sanitizer=asan'] reporter = AFLSancovReporter(args) self.assertFalse(reporter.run()) - # self.do_cmd("{} --overwrite --dd-mode".format(self.single_generator)) self.assertTrue(os.path.exists(self.dd_dir), "No delta-diff dir generated during dd-mode invocation") self.assertTrue((os.path.exists(self.dd_file1) and os.path.exists(self.dd_file2)), "Missing delta-diff file(s) during dd-mode invocation") - self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddmode_file1), + self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddmode_asan_file1), + "Delta-diff file {} does not match".format(self.dd_filename1)) + self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddmode_asan_file2), + "Delta-diff file {} does not match".format(self.dd_filename2)) + + + def test_ddmode_asan_sancov_bug(self): + args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov-asan', + '--bin-path={}/test-sancov-asan'.format(os.getcwd()), + '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', + '--pysancov-path=/usr/local/bin/pysancov', '--crash-dir={}/unique'.format(os.getcwd()), '--overwrite', + '--sancov-bug', '--sanitizer=asan'] + reporter = AFLSancovReporter(args) + self.assertFalse(reporter.run()) + self.assertTrue(os.path.exists(self.dd_dir), + "No delta-diff dir generated during dd-mode invocation") + self.assertTrue((os.path.exists(self.dd_file1) and os.path.exists(self.dd_file2)), + "Missing delta-diff file(s) during dd-mode invocation") + + self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddmode_asan_file1), + "Delta-diff file {} does not match".format(self.dd_filename1)) + self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddmode_asan_file2), + "Delta-diff file {} does not match".format(self.dd_filename2)) + + + def test_ddnum_asan(self): + args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov-asan', + '--bin-path={}/test-sancov-asan'.format(os.getcwd()), + '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', + '--pysancov-path=/usr/local/bin/pysancov', '--overwrite', '--dd-num=3', + '--crash-dir={}/unique'.format(os.getcwd()), '--sanitizer=asan'] + reporter = AFLSancovReporter(args) + self.assertFalse(reporter.run()) + self.assertTrue(os.path.exists(self.dd_dir), + "No delta-diff dir generated during dd-num invocation") + self.assertTrue((os.path.exists(self.dd_file1) and os.path.exists(self.dd_file2)), + "Missing delta-diff file(s) during dd-num invocation") + + self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddnum_asan_file1), "Delta-diff file {} does not match".format(self.dd_filename1)) - self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddmode_file2), + self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddnum_asan_file2), "Delta-diff file {} does not match".format(self.dd_filename2)) - def test_ddnum(self): - args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov', '--bin-path={}/test-sancov'.format(os.getcwd()), + + def test_ddnum_asan_sancov_bug(self): + args = ['-d', './afl-out', '-e', 'cat AFL_FILE | ./test-sancov-asan', + '--bin-path={}/test-sancov-asan'.format(os.getcwd()), '--sancov-path=/usr/bin/sancov-3.8', '--llvm-sym-path=/usr/bin/llvm-symbolizer-3.8', - '--pysancov-path=/usr/local/bin/pysancov', '--overwrite', '--dd-mode', '--dd-num=3'] - # self.do_cmd("{} --overwrite --dd-mode --dd-num 5".format(self.single_generator)) + '--pysancov-path=/usr/local/bin/pysancov', '--overwrite', '--dd-num=3', + '--crash-dir={}/unique'.format(os.getcwd()), '--sancov-bug', '--sanitizer=asan'] reporter = AFLSancovReporter(args) self.assertFalse(reporter.run()) self.assertTrue(os.path.exists(self.dd_dir), @@ -189,122 +230,49 @@ def test_ddnum(self): self.assertTrue((os.path.exists(self.dd_file1) and os.path.exists(self.dd_file2)), "Missing delta-diff file(s) during dd-num invocation") - self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddnum_file1), + self.assertTrue(self.compare_json(self.dd_file1, self.expects_ddnum_asan_file1), "Delta-diff file {} does not match".format(self.dd_filename1)) - self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddnum_file2), + self.assertTrue(self.compare_json(self.dd_file2, self.expects_ddnum_asan_file2), "Delta-diff file {} does not match".format(self.dd_filename2)) - # def test_stop_requires_fuzz_dir(self): - # self.assertTrue('Must set' - # in ''.join(self.do_cmd("%s --stop-afl" % (self.afl_cov_cmd))), - # "--afl-fuzzing-dir missing from --stop-afl mode") - - # def test_func_search_requires_fuzz_dir(self): - # self.assertTrue('Must set' - # in ''.join(self.do_cmd("%s --func-search test" % (self.afl_cov_cmd))), - # "--afl-fuzzing-dir missing from --func-search mode") - - # def test_line_search_requires_fuzz_dir(self): - # self.assertTrue('Must set' - # in ''.join(self.do_cmd("%s --line-search 1234" % (self.afl_cov_cmd))), - # "--afl-fuzzing-dir missing from --line-search mode") - - # def test_live_parallel(self): - # - # if not self.live_init(): - # return self.assertTrue(False, "Could not run generator cmd: %s" \ - # % (self.afl_cov_live)) - # - # ### put the wrapper in place - # wrapper ='fwknop-afl.git/test/afl/fuzzing-wrappers' + \ - # '/server-access-parallel-redir.sh' - # if os.path.exists(wrapper): - # os.remove(wrapper) - # copy('afl/server-access-parallel-redir.sh', wrapper) - # curr_dir = os.getcwd() - # os.chdir('./fwknop-afl.git/test/afl') - # - # ### now start two copies of AFL - # try: - # subprocess.Popen([self.live_parallel_afl_cmd, "-M", "fuzzer01"]) - # except OSError: - # os.chdir(curr_dir) - # return self.assertTrue(False, - # "Could not run live_parallel_afl_cmd: %s -M fuzzer01" \ - # % (self.live_parallel_afl_cmd)) - # - # try: - # subprocess.Popen([self.live_parallel_afl_cmd, "-S", "fuzzer02"]) - # except OSError: - # os.chdir(curr_dir) - # return self.assertTrue(False, - # "Could not run live_parallel_afl_cmd: %s -S fuzzer02" \ - # % (self.live_parallel_afl_cmd)) - # - # os.chdir(curr_dir) - # - # time.sleep(3) - # - # self.afl_stop() - # - # if not (is_dir(self.top_out_dir + '/fuzzer01') - # and is_dir(self.top_out_dir + '/fuzzer02')): - # return self.assertTrue(False, - # "fuzzer01 or fuzzer02 directory missing") - # - # ### check for the coverage directory since afl-cov should have - # ### seen the running AFL instance by now - # return self.assertTrue(is_dir(self.top_out_dir + '/cov'), - # "live coverage directory '%s' does not exist" \ - # % (self.top_out_dir + '/cov')) - # - # def test_live(self): - # - # if not self.live_init(): - # return self.assertTrue(False, "Could not run generator cmd: %s" \ - # % (self.afl_cov_live)) - # - # ### put the wrapper in place - # wrapper = 'fwknop-afl.git/test/afl/fuzzing-wrappers/server-access-redir.sh' - # if os.path.exists(wrapper): - # os.remove(wrapper) - # copy('afl/server-access-redir.sh', wrapper) - # curr_dir = os.getcwd() - # os.chdir('./fwknop-afl.git/test/afl') - # - # ### now start AFL and let it run for longer than --sleep in the - # ### generator script - then look for the coverage directory - # try: - # subprocess.Popen([self.live_afl_cmd]) - # except OSError: - # os.chdir(curr_dir) - # return self.assertTrue(False, - # "Could not run live_afl_cmd: %s" % (self.live_afl_cmd)) - # os.chdir(curr_dir) - # - # time.sleep(3) - # - # self.afl_stop() - # - # ### check for the coverage directory since afl-cov should have - # ### seen the running AFL instance by now - # return self.assertTrue(is_dir(self.top_out_dir + '/cov'), - # "live coverage directory '%s' does not exist" \ - # % (self.top_out_dir + '/cov')) - - # def test_queue_limit_5(self): - # out_str = ''.join(self.do_cmd("%s --afl-queue-id-limit 5 --overwrite" \ - # % (self.single_generator))) - # self.assertTrue('Final lcov web report' in out_str - # and "New 'line' coverage: 1571" in out_str) - # - # def test_queue_limit_5_parallel(self): - # out_str = ''.join(self.do_cmd("%s --afl-queue-id-limit 5 --overwrite" \ - # % (self.parallel_generator))) - # self.assertTrue('Final lcov web report' in out_str - # and "New 'line' coverage: 1571" in out_str - # and "Imported 145 new test cases" in out_str - # and "Imported 212 new test cases" in out_str) + def test_validate_cov_cmd(self): + args = [] + reporter = AFLSancovReporter(args) + # Checks no cov cmd + self.assertTrue(reporter.run()) + args = ['-e', 'cat FILE | ./test-sancov-asan'] + reporter = AFLSancovReporter(args) + # Checks incorrect cov cmd + self.assertTrue(reporter.run()) + args = ['-e', 'cat AFL_FILE | ./test-sancov-asan'] + reporter = AFLSancovReporter(args) + # Checks no afl fuzz dir + self.assertTrue(reporter.run()) + args.extend(['-d', './afl-out']) + reporter = AFLSancovReporter(args) + # Checks no crash dir + self.assertTrue(reporter.run()) + args.extend(['--crash-dir={}/unique'.format(os.getcwd())]) + reporter = AFLSancovReporter(args) + # Checks no bin path + self.assertTrue(reporter.run()) + args.extend(['--bin-path={}/test-sancov-noexist'.format(os.getcwd())]) + reporter = AFLSancovReporter(args) + # Checks incorrect bin path + self.assertTrue(reporter.run()) + args[len(args)-1] = '--bin-path={}/test-sancov-ubsan'.format(os.getcwd()) + args.extend(['--sancov-path=sancov-noexist']) + reporter = AFLSancovReporter(args) + # Checks incorrect sancov path + self.assertTrue(reporter.run()) + args[len(args)-1] = '--pysancov-path=pysancov-noexist' + reporter = AFLSancovReporter(args) + # Checks incorrect pysancov path + self.assertTrue(reporter.run()) + args[len(args)-1] = '--llvm-sym-path=llvm-symbolizer-noexist' + reporter = AFLSancovReporter(args) + # Checks incorrect llvm-sym path + self.assertTrue(reporter.run()) if __name__ == "__main__": unittest.main() diff --git a/tests/afl-out/unique/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2 b/tests/unique/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2 similarity index 100% rename from tests/afl-out/unique/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2 rename to tests/unique/HARDEN:0001,SESSION000:id:000000,sig:06,src:000003,op:havoc,rep:2 diff --git a/tests/afl-out/unique/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4 b/tests/unique/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4 similarity index 100% rename from tests/afl-out/unique/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4 rename to tests/unique/HARDEN:0001,SESSION001:id:000000,sig:06,src:000003,op:havoc,rep:4