Capture crash dumps from libfuzzer, when provided (#2793)

Enables capturing crashdumps generated by ASAN at point of failure.

This helps in several ways:
- provides a crash dump in the case that we cannot repro a failure later
- some people have stated that crash dumps would be more useful to their team than the repro VM
- we should be able to use these for automated submission to Watson or similar

---

Crash dumps are automatically collected (for libfuzzer) jobs, if we find any. They should be activated by enabling crash dumps in ASAN, via:

- On Linux: `ASAN_OPTIONS=disable_coredump=0:abort_on_error=1:unmap_shadow_on_exit=1`
  - OneFuzz will override the (Ubuntu) crash dump options to generate core dumps instead and then upload them.
- On Windows: `ASAN_SAVE_DUMPS=my_dump.dmp`
  - OneFuzz will look for any `*.dmp` files in the working directory and then upload them.

In both cases, the crash dump will be renamed to match the crashing input, if possible, and uploaded to a new `crashdumps` container.

---

Also updated: the “simple” LibFuzzer test has been updated to be compiled with `cl.exe` instead of `clang` on Windows, so that we are exercising the MSVC implementation of ASAN/LibFuzzer, and the CI image has been updated to `windows-2022`. The restriction to an old version of the Windows SDK has been removed.

.github/workflows/ci.yml

@@ -527,7 +527,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: src/integration-tests/artifacts
-          key: integration-tests|linux|${{ hashFiles('src/integration-tests/**/*') }}
+          key: integration-tests|linux|${{ env.ACTIONS_CACHE_KEY_DATE }}|${{ hashFiles('src/integration-tests/**/*') }}
       - name: Build integration tests
         if: steps.cache-integration-tests.outputs.cache-hit != 'true'
         run: |
@@ -590,15 +590,17 @@ jobs:
           name: artifact-integration-tests-linux
           path: src/integration-tests/artifacts
   build-integration-tests-windows:
-    runs-on: windows-2019
+    runs-on: windows-2022
     steps:
       - uses: actions/checkout@v3
       - name: Cache integration tests
         id: cache-integration-tests
         uses: actions/cache@v3
         with:
           path: src/integration-tests/artifacts
-          key: integration-tests|windows|${{ hashFiles('src/integration-tests/**/*') }}
+          key: integration-tests|windows|${{ env.ACTIONS_CACHE_KEY_DATE }}|${{ hashFiles('src/integration-tests/**/*') }}
+      - name: Setup C/C++ environment
+        uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # pinned to v1.12.1
       - name: Build integration tests
         if: steps.cache-integration-tests.outputs.cache-hit != 'true'
         run: |
@@ -609,33 +611,6 @@ jobs:
           choco install make
           $env:Path += ";C:\Program Files\LLVM\bin;C:\ProgramData\chocolatey\bin"
 
-          # WORKAROUND: effectively downgrade the default Windows 10 SDK version.
-          #
-          # This ensures we link against a version of the SDK which won't trigger a
-          # startup bug in the LLVM-shipped ASAN runtime.
-
-          # Assume a default MSVC 2019 install path.
-          $MsvcDir = 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC'
-
-          # Assume that `$MsvcDir` only contains version-named subdirectories.
-          $MsvcVersion = ((Get-ChildItem $MsvcDir).name | Sort-Object -Descending)[0]
-          $MsvcLib = "${MsvcDir}/${MsvcVersion}/lib/x64"
-
-          # Known "good" (non-bug-surfacing) version.
-          $WindowsSdkVersion = '10.0.18362.0'
-
-          # Assume default install path.
-          $WindowsSdkDir = 'C:/Program Files (x86)/Windows Kits/10'
-          $WindowsSdkLib = "${WindowsSdkDir}/Lib/${WindowsSdkVersion}"
-          $WindowsSdkInclude = "${WindowsSdkDir}/Include/${WindowsSdkVersion}"
-
-          # Used by `clang.exe`.
-          $env:CPATH = $WindowsSdkInclude
-          $env:LIBRARY_PATH = "${MsvcLib};${WindowsSdkLib}/ucrt/x64;${WindowsSdkLib}/um/x64"
-
-          # Used by `link.exe`.
-          $env:LIB = $env:LIBRARY_PATH
-
           cd src/integration-tests
 
           mkdir artifacts/windows-libfuzzer

docs/webhook_events.md

@@ -143,6 +143,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "coverage",
                 "crashes",
                 "inputs",
+                "crashdumps",
                 "no_repro",
                 "readonly_inputs",
                 "reports",
@@ -1956,6 +1957,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "coverage",
                 "crashes",
                 "inputs",
+                "crashdumps",
                 "no_repro",
                 "readonly_inputs",
                 "reports",
@@ -2867,6 +2869,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "coverage",
                 "crashes",
                 "inputs",
+                "crashdumps",
                 "no_repro",
                 "readonly_inputs",
                 "reports",
@@ -3358,6 +3361,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "coverage",
                 "crashes",
                 "inputs",
+                "crashdumps",
                 "no_repro",
                 "readonly_inputs",
                 "reports",
@@ -3867,6 +3871,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "coverage",
                 "crashes",
                 "inputs",
+                "crashdumps",
                 "no_repro",
                 "readonly_inputs",
                 "reports",
@@ -4324,6 +4329,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "coverage",
                 "crashes",
                 "inputs",
+                "crashdumps",
                 "no_repro",
                 "readonly_inputs",
                 "reports",
@@ -4808,6 +4814,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "coverage",
                 "crashes",
                 "inputs",
+                "crashdumps",
                 "no_repro",
                 "readonly_inputs",
                 "reports",
@@ -5422,6 +5429,7 @@ If webhook is set to have Event Grid message format then the payload will look a
                 "coverage",
                 "crashes",
                 "inputs",
+                "crashdumps",
                 "no_repro",
                 "readonly_inputs",
                 "reports",

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -106,6 +106,7 @@ public enum ContainerType {
     Analysis,
     Coverage,
     Crashes,
+    Crashdumps,
     Inputs,
     NoRepro,
     ReadonlyInputs,

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -1103,6 +1103,7 @@ Dictionary<string, string> Tags
     public IContainerDef? Analysis { get; set; }
     public IContainerDef? Coverage { get; set; }
     public IContainerDef? Crashes { get; set; }
+    public IContainerDef? Crashdumps { get; set; }
     public IContainerDef? Inputs { get; set; }
     public IContainerDef? NoRepro { get; set; }
     public IContainerDef? ReadonlyInputs { get; set; }

src/ApiService/ApiService/onefuzzlib/Config.cs

@@ -118,6 +118,9 @@ await _containers.GetContainerSasUrl(container.Name, StorageType.Corpus, Convert
                     case ContainerType.Crashes:
                         config.Crashes = def;
                         break;
+                    case ContainerType.Crashdumps:
+                        config.Crashdumps = def;
+                        break;
                     case ContainerType.Inputs:
                         config.Inputs = def;
                         break;

src/ApiService/ApiService/onefuzzlib/Defs.cs

@@ -190,6 +190,12 @@ public static class Defs {
                             Value: 1,
                             Permissions: ContainerPermission.Write
                         ),
+                        new ContainerDefinition(
+                            Type:ContainerType.Crashdumps,
+                            Compare: Compare.Equal,
+                            Value:1,
+                            Permissions: ContainerPermission.Write
+                            ),
                         new ContainerDefinition(
                             Type: ContainerType.Inputs,
                             Compare: Compare.Equal,
@@ -279,6 +285,12 @@ public static class Defs {
                             Value: 1,
                             Permissions: ContainerPermission.Write
                         ),
+                        new ContainerDefinition(
+                            Type:ContainerType.Crashdumps,
+                            Compare: Compare.Equal,
+                            Value:1,
+                            Permissions: ContainerPermission.Write
+                            ),
                         new ContainerDefinition(
                             Type: ContainerType.Inputs,
                             Compare: Compare.Equal,

src/agent/onefuzz-task/src/local/common.rs

@@ -24,6 +24,7 @@ use crate::tasks::utils::parse_key_value;
 pub const SETUP_DIR: &str = "setup_dir";
 pub const INPUTS_DIR: &str = "inputs_dir";
 pub const CRASHES_DIR: &str = "crashes_dir";
+pub const CRASHDUMPS_DIR: &str = "crashdumps_dir";
 pub const TARGET_WORKERS: &str = "target_workers";
 pub const REPORTS_DIR: &str = "reports_dir";
 pub const NO_REPRO_DIR: &str = "no_repro_dir";

src/agent/onefuzz-task/src/local/libfuzzer_fuzz.rs

@@ -6,8 +6,8 @@ use std::path::PathBuf;
 use crate::{
     local::common::{
         build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, CmdType,
-        SyncCountDirMonitor, UiEvent, CHECK_FUZZER_HELP, CRASHES_DIR, INPUTS_DIR, TARGET_ENV,
-        TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS,
+        SyncCountDirMonitor, UiEvent, CHECK_FUZZER_HELP, CRASHDUMPS_DIR, CRASHES_DIR, INPUTS_DIR,
+        TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_WORKERS,
     },
     tasks::{
         config::CommonConfig,
@@ -27,6 +27,8 @@ pub fn build_fuzz_config(
 ) -> Result<Config> {
     let crashes = get_synced_dir(CRASHES_DIR, common.job_id, common.task_id, args)?
         .monitor_count(&event_sender)?;
+    let crashdumps = get_synced_dir(CRASHDUMPS_DIR, common.job_id, common.task_id, args)?
+        .monitor_count(&event_sender)?;
     let inputs = get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)?
         .monitor_count(&event_sender)?;
 
@@ -49,6 +51,7 @@ pub fn build_fuzz_config(
         inputs,
         readonly_inputs,
         crashes,
+        crashdumps,
         target_exe,
         target_env,
         target_options,
@@ -85,6 +88,10 @@ pub fn build_shared_args() -> Vec<Arg> {
             .long(CRASHES_DIR)
             .required(true)
             .value_parser(value_parser!(PathBuf)),
+        Arg::new(CRASHDUMPS_DIR)
+            .long(CRASHDUMPS_DIR)
+            .required(true)
+            .value_parser(value_parser!(PathBuf)),
         Arg::new(TARGET_WORKERS)
             .long(TARGET_WORKERS)
             .value_parser(value_parser!(u64)),

src/agent/onefuzz-task/src/local/template.rs

@@ -51,6 +51,7 @@ struct LibFuzzer {
     inputs: PathBuf,
     readonly_inputs: Vec<PathBuf>,
     crashes: PathBuf,
+    crashdumps: PathBuf,
     target_exe: PathBuf,
     target_env: HashMap<String, String>,
     target_options: Vec<String>,
@@ -143,6 +144,7 @@ impl TaskConfig {
                     inputs: context.to_monitored_sync_dir("inputs", &config.inputs)?,
                     readonly_inputs: Some(ri?),
                     crashes: context.to_monitored_sync_dir("crashes", &config.crashes)?,
+                    crashdumps: context.to_monitored_sync_dir("crashdumps", &config.crashdumps)?,
                     target_exe: config.target_exe.clone(),
                     target_env: config.target_env.clone(),
                     target_options: config.target_options.clone(),