Bump flume from 0.10.14 to 0.11.0 in /src/agent

.github/workflows/ci.yml

@@ -79,16 +79,24 @@ jobs:
           key: ${{env.ACTIONS_CACHE_KEY_DATE}} # additional key for cache-busting
           workspaces: src/agent
       - name: Linux Prereqs
-        if: runner.os == 'Linux' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
+        if: runner.os == 'Linux'
         run: |
           sudo apt-get -y update
-          sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config
+          sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config clang
+      - name: Clone onefuzz-samples
+        run: git clone https://github.com/microsoft/onefuzz-samples
+      - name: Prepare for agent integration tests
+        shell: bash
+        working-directory: ./onefuzz-samples/examples/simple-libfuzzer
+        run: |
+          make
+          mkdir -p ../../../src/agent/onefuzz-task/tests/targets/simple
+          cp fuzz.exe ../../../src/agent/onefuzz-task/tests/targets/simple/fuzz.exe
+          cp *.pdb ../../../src/agent/onefuzz-task/tests/targets/simple/ 2>/dev/null || :
       - name: Install Rust Prereqs
-        if: steps.rust-build-cache.outputs.cache-hit != 'true' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
         shell: bash
         run: src/ci/rust-prereqs.sh
       - run: src/ci/agent.sh
-        if: steps.cache-agent-artifacts.outputs.cache-hit != 'true'
         shell: bash
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

CHANGELOG.md

@@ -7,6 +7,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 8.8.0
+
+### Added
+
+* Agent: Added Mariner Linux support for agent VMs [#3306](https://github.com/microsoft/onefuzz/pull/3306)
+* Service: Added support for custom ado fields that mark work items as duplicate [#3467](https://github.com/microsoft/onefuzz/pull/3467)
+* Service: Permanently store OneFuzz job result data - # crashing input, # regression crashing input, etc. - in Azure storage [#3380](https://github.com/microsoft/onefuzz/pull/3380), [#3439](https://github.com/microsoft/onefuzz/pull/3439)
+* Service: Added validation for Iteration/AreaPath on notifications when a job is submitted with a notification config and for `onefuzz debug notification test_template` [#3386](https://github.com/microsoft/onefuzz/pull/3386)
+
+### Changed
+
+* Agent: Updated libfuzzer-fuzz basic template to include required args and make it match cli [#3429](https://github.com/microsoft/onefuzz/pull/3429)
+* Agent: Downgraded some debug logs from warn to debug [#3450](https://github.com/microsoft/onefuzz/pull/3450)
+* CLI: Removed CLI commands from the local fuzzing tasks as they can now be described via yaml template [#3428](https://github.com/microsoft/onefuzz/pull/3428)
+* Service: AutoScale table entries are now deleted on VMSS shutdown [#3455](https://github.com/microsoft/onefuzz/pull/3455)
+
+### Fixed
+
+* Agent: Fixed local path generation [#3432](https://github.com/microsoft/onefuzz/pull/3432), [#3460](https://github.com/microsoft/onefuzz/pull/3460)
+
+## 8.7.1
+
+### Fixed
+
+* Service: Removed deprecated Azure retention policy setting that was causing scaleset deployment errors [#3452](https://github.com/microsoft/onefuzz/pull/3452)
+
 ## 8.7.0
 
 ### Added

CURRENT_VERSION

@@ -1 +1 @@
-8.7.0
+8.8.0

README.md

@@ -1,5 +1,20 @@
 # <img src="docs/onefuzz_text.svg" height="120" alt="OneFuzz" />
 
+# IMPORTANT NOTICE
+
+**_Since September 2020 when OneFuzz was first open sourced, we’ve been on a journey to create a best-in-class orchestrator for running fuzzers, driving security and quality into our products._**
+
+
+**_Initially launched by a small group in MSR, OneFuzz has now become a significant internal platform within Microsoft. As such, we are regretfully archiving the project to focus our attention on becoming a more deeply integrated service within the company. Unfortunately, we aren’t a large enough team to live in both the open-source world and the internal Microsoft world with its own unique set of requirements._**
+
+**_Our current plan is to archive the project in the next few months. That means we’ll still be making updates for a little while. Of course, even after it’s archived, you’ll still be able to fork it and make the changes you need. Once we’ve decided on a specific date for archiving, we’ll update this readme._**
+
+**_Thanks for taking the journey with us._**
+
+**_The OneFuzz team._**
+
+---
+
 [![Onefuzz build status](https://github.com/microsoft/onefuzz/workflows/Build/badge.svg?branch=main)](https://github.com/microsoft/onefuzz/actions/workflows/ci.yml?query=branch%3Amain)
 
 ## A self-hosted Fuzzing-As-A-Service platform

contrib/onefuzz-job-azure-devops-pipeline/ado-work-items.json

@@ -13,6 +13,10 @@
       "System.AreaPath": "OneFuzz-Ado-Integration",
       "System.Title": "{{report.task_id}}"
     },
+    "ado_duplicate_fields": {
+      "System.Reason": "My custom value that means a work item is a duplicate",
+      "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate"
+    },
     "on_duplicate": {
       "increment": [],
       "comment": "DUP {{report.input_sha256}} <br> Repro Command: <br> <pre> {{ repro_cmd }} </pre> ",

docs/notifications/ado.md

@@ -51,6 +51,13 @@ clickable, make it a link.
       "System.Title": "{{ report.crash_site }} - {{ report.executable }}",
       "Microsoft.VSTS.TCM.ReproSteps": "This is my call stack: <ul> {{ for item in report.call_stack }} <li> {{ item }} </li> {{ end }} </ul>"
     },
+    "ado_duplicate_fields": {
+      "System.Reason": "My custom value that means a work item is a duplicate",
+      "Custom.Work.Item.Field": "My custom value that means a work item is a duplicate"
+      // note: the fields and values below are checked by default and don't need to be specified
+      // "System.Reason": "Duplicate"
+      // "Microsoft.VSTS.Common.ResolvedReason": "Duplicate"
+    },
     "comment": "This is my comment. {{ report.input_sha256 }} {{ input_url }} <br> <pre>{{ repro_cmd }}</pre>",
     "unique_fields": ["System.Title", "System.AreaPath"],
     "on_duplicate": {

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -49,6 +49,7 @@ public enum ErrorCode {
     ADO_VALIDATION_MISSING_PAT_SCOPES = 492,
     ADO_WORKITEM_PROCESSING_DISABLED = 494,
     ADO_VALIDATION_INVALID_PATH = 495,
+    ADO_VALIDATION_INVALID_PROJECT = 496,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -689,6 +689,7 @@ public record AdoTemplate(
     List<string> UniqueFields,
     Dictionary<string, string> AdoFields,
     ADODuplicateTemplate OnDuplicate,
+    Dictionary<string, string>? AdoDuplicateFields = null,
     string? Comment = null
     ) : NotificationTemplate {
     public async Task<OneFuzzResultVoid> Validate() {
@@ -704,8 +705,9 @@ public record RenderedAdoTemplate(
     List<string> UniqueFields,
     Dictionary<string, string> AdoFields,
     ADODuplicateTemplate OnDuplicate,
+    Dictionary<string, string>? AdoDuplicateFields = null,
     string? Comment = null
-    ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, Comment);
+    ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, AdoDuplicateFields, Comment);
 
 public record TeamsTemplate(SecretData<string> Url) : NotificationTemplate {
     public Task<OneFuzzResultVoid> Validate() {

src/ApiService/ApiService/onefuzzlib/notifications/Ado.cs

@@ -89,30 +89,97 @@ private static bool IsTransient(Exception e) {
         return errorCodes.Any(errorStr.Contains);
     }
 
-    private static async Async.Task<OneFuzzResultVoid> ValidatePath(string project, string path, TreeStructureGroup structureGroup, WorkItemTrackingHttpClient client) {
-        var pathType = (structureGroup == TreeStructureGroup.Areas) ? "Area" : "Iteration";
-        var pathParts = path.Split('\\');
-        if (!string.Equals(pathParts[0], project, StringComparison.OrdinalIgnoreCase)) {
+    public static OneFuzzResultVoid ValidateTreePath(IEnumerable<string> path, WorkItemClassificationNode? root) {
+        if (root is null) {
+            return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PROJECT, new string[] {
+                $"Path \"{string.Join('\\', path)}\" is invalid. The specified ADO project doesn't exist.",
+                "Double check the 'project' field in your ADO config.",
+            });
+        }
+
+        string treeNodeTypeName;
+        switch (root.StructureType) {
+            case TreeNodeStructureType.Area:
+                treeNodeTypeName = "Area";
+                break;
+            case TreeNodeStructureType.Iteration:
+                treeNodeTypeName = "Iteration";
+                break;
+            default:
+                return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
+                    $"Path root \"{root.Name}\" is an unsupported type. Expected Area or Iteration but got {root.StructureType}.",
+                });
+        }
+
+        // Validate path based on
+        // https://learn.microsoft.com/en-us/azure/devops/organizations/settings/about-areas-iterations?view=azure-devops#naming-restrictions
+        var maxNodeLength = 255;
+        var maxDepth = 13;
+        // Invalid characters from the link above plus the escape sequences (since they have backslashes and produce confusingly formatted errors if not caught here)
+        var invalidChars = new char[] { '/', ':', '*', '?', '"', '<', '>', '|', ';', '#', '$', '*', '{', '}', ',', '+', '=', '[', ']' };
+
+        // Ensure that none of the path parts are too long
+        var erroneous = path.FirstOrDefault(part => part.Length > maxNodeLength);
+        if (erroneous != null) {
+            return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
+                $"{treeNodeTypeName} Path \"{string.Join('\\', path)}\" is invalid. \"{erroneous}\" is too long. It must be less than {maxNodeLength} characters.",
+                "Learn more about naming restrictions here: https://learn.microsoft.com/en-us/azure/devops/organizations/settings/about-areas-iterations?view=azure-devops#naming-restrictions"
+            });
+        }
+
+        // Ensure that none of the path parts contain invalid characters
+        erroneous = path.FirstOrDefault(part => invalidChars.Any(part.Contains));
+        if (erroneous != null) {
             return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
-                $"Path \"{path}\" is invalid. It must start with the project name, \"{project}\".",
-                $"Example: \"{project}\\{path}\".",
+                $"{treeNodeTypeName} Path \"{string.Join('\\', path)}\" is invalid. \"{erroneous}\" contains an invalid character ({string.Join(" ", invalidChars)}).",
+                "Make sure that the path is separated by backslashes (\\) and not forward slashes (/).",
+                "Learn more about naming restrictions here: https://learn.microsoft.com/en-us/azure/devops/organizations/settings/about-areas-iterations?view=azure-devops#naming-restrictions"
             });
         }
 
-        var current = await client.GetClassificationNodeAsync(project, structureGroup, depth: pathParts.Length - 1);
-        if (current == null) {
+        // Ensure no unicode control characters
+        erroneous = path.FirstOrDefault(part => part.Any(ch => char.IsControl(ch)));
+        if (erroneous != null) {
             return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
-                $"{pathType} Path \"{path}\" is invalid. \"{project}\" is not a valid project.",
+                // More about control codes and their range here: https://en.wikipedia.org/wiki/Unicode_control_characters
+                $"{treeNodeTypeName} Path \"{string.Join('\\', path)}\" is invalid. \"{erroneous}\" contains a unicode control character (\\u0000 - \\u001F or \\u007F - \\u009F).",
+                "Make sure that you're path doesn't contain any escape characters (\\0 \\a \\b \\f \\n \\r \\t \\v).",
+                "Learn more about naming restrictions here: https://learn.microsoft.com/en-us/azure/devops/organizations/settings/about-areas-iterations?view=azure-devops#naming-restrictions"
             });
         }
 
-        foreach (var part in pathParts.Skip(1)) {
+        // Ensure that there aren't too many path parts
+        if (path.Count() > maxDepth) {
+            return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
+                $"{treeNodeTypeName} Path \"{string.Join('\\', path)}\" is invalid. It must be less than {maxDepth} levels deep.",
+                "Learn more about naming restrictions here: https://learn.microsoft.com/en-us/azure/devops/organizations/settings/about-areas-iterations?view=azure-devops#naming-restrictions"
+            });
+        }
+
+
+        // Path should always start with the project name ADO expects an absolute path
+        if (!string.Equals(path.First(), root.Name, StringComparison.OrdinalIgnoreCase)) {
+            return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
+                $"{treeNodeTypeName} Path \"{string.Join('\\', path)}\" is invalid. It must start with the project name, \"{root.Name}\".",
+                $"Example: \"{root.Name}\\{path}\".",
+            });
+        }
+
+        // Validate that each part of the path is a valid child of the previous part
+        var current = root;
+        foreach (var part in path.Skip(1)) {
             var child = current.Children?.FirstOrDefault(x => string.Equals(x.Name, part, StringComparison.OrdinalIgnoreCase));
             if (child == null) {
-                return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
-                    $"{pathType} Path \"{path}\" is invalid. \"{part}\" is not a valid child of \"{current.Name}\".",
-                    $"Valid children of \"{current.Name}\" are: [{string.Join(',', current.Children?.Select(x => $"\"{x.Name}\"") ?? new List<string>())}].",
-                });
+                if (current.Children is null || !current.Children.Any()) {
+                    return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
+                        $"{treeNodeTypeName} Path \"{string.Join('\\', path)}\" is invalid. \"{current.Name}\" has no children.",
+                    });
+                } else {
+                    return OneFuzzResultVoid.Error(ErrorCode.ADO_VALIDATION_INVALID_PATH, new string[] {
+                        $"{treeNodeTypeName} Path \"{string.Join('\\', path)}\" is invalid. \"{part}\" is not a valid child of \"{current.Name}\".",
+                        $"Valid children of \"{current.Name}\" are: [{string.Join(',', current.Children?.Select(x => $"\"{x.Name}\"") ?? new List<string>())}].",
+                    });
+                }
             }
 
             current = child;
@@ -195,14 +262,19 @@ await policy.ExecuteAsync(async () => {
 
         try {
             // Validate AreaPath and IterationPath exist
+            // This also validates that the config.Project exists
             if (config.AdoFields.TryGetValue("System.AreaPath", out var areaPathString)) {
-                var validateAreaPath = await ValidatePath(config.Project, areaPathString, TreeStructureGroup.Areas, witClient);
+                var path = areaPathString.Split('\\');
+                var root = await witClient.GetClassificationNodeAsync(config.Project, TreeStructureGroup.Areas, depth: path.Length - 1);
+                var validateAreaPath = ValidateTreePath(path, root);
                 if (!validateAreaPath.IsOk) {
                     return validateAreaPath;
                 }
             }
             if (config.AdoFields.TryGetValue("System.IterationPath", out var iterationPathString)) {
-                var validateIterationPath = await ValidatePath(config.Project, iterationPathString, TreeStructureGroup.Iterations, witClient);
+                var path = iterationPathString.Split('\\');
+                var root = await witClient.GetClassificationNodeAsync(config.Project, TreeStructureGroup.Iterations, depth: path.Length - 1);
+                var validateIterationPath = ValidateTreePath(path, root);
                 if (!validateIterationPath.IsOk) {
                     return validateIterationPath;
                 }
@@ -291,6 +363,7 @@ public static RenderedAdoTemplate RenderAdoTemplate(ILogger logTracer, Renderer
             original.UniqueFields,
             adoFields,
             onDuplicate,
+            original.AdoDuplicateFields,
             original.Comment != null ? Render(renderer, original.Comment, instanceUrl, logTracer) : null
         );
     }
@@ -535,7 +608,7 @@ public async Async.Task Process(IList<(string, string)> notificationInfo) {
                     _logTracer.AddTags(new List<(string, string)> { ("MatchingWorkItemIds", $"{workItem.Id}") });
                     _logTracer.LogInformation("Found matching work item");
                 }
-                if (IsADODuplicateWorkItem(workItem)) {
+                if (IsADODuplicateWorkItem(workItem, _config.AdoDuplicateFields)) {
                     continue;
                 }
 
@@ -575,13 +648,17 @@ public async Async.Task Process(IList<(string, string)> notificationInfo) {
             }
         }
 
-        private static bool IsADODuplicateWorkItem(WorkItem wi) {
+        private static bool IsADODuplicateWorkItem(WorkItem wi, Dictionary<string, string>? duplicateFields) {
             // A work item could have System.State == Resolve && System.Reason == Duplicate
             // OR it could have System.State == Closed && System.Reason == Duplicate
             // I haven't found any other combinations where System.Reason could be duplicate but just to be safe
             // we're explicitly _not_ checking the state of the work item to determine if it's duplicate
             return wi.Fields.ContainsKey("System.Reason") && string.Equals(wi.Fields["System.Reason"].ToString(), "Duplicate", StringComparison.OrdinalIgnoreCase)
             || wi.Fields.ContainsKey("Microsoft.VSTS.Common.ResolvedReason") && string.Equals(wi.Fields["Microsoft.VSTS.Common.ResolvedReason"].ToString(), "Duplicate", StringComparison.OrdinalIgnoreCase)
+            || duplicateFields?.Any(fieldPair => {
+                var (field, value) = fieldPair;
+                return wi.Fields.ContainsKey(field) && string.Equals(wi.Fields[field].ToString(), value, StringComparison.OrdinalIgnoreCase);
+            }) == true
             // Alternatively, the work item can also specify a 'relation' to another work item.
             // This is typically used to create parent/child relationships between work items but can also
             // Be used to mark duplicates so we should check this as well.

src/ApiService/IntegrationTests/JinjaToScribanMigrationTests.cs

@@ -111,6 +111,7 @@ public async Async.Task OptionalFieldsAreSupported() {
                 },
                 "{{ if org }} blah {{ end }}"
             ),
+            null,
             "{{ if org }} blah {{ end }}"
         );
 
@@ -137,6 +138,7 @@ public async Async.Task All_ADO_Fields_Are_Migrated() {
                 },
                 "{% if org %} comment {% endif %}"
             ),
+            null,
             "{% if org %} comment {% endif %}"
         );
 

src/ApiService/Tests/OrmModelsTest.cs

@@ -232,6 +232,7 @@ from authToken in Arb.Generate<SecretData<string>>()
                         from str in Arb.Generate<NonEmptyString>()
                         from fields in Arb.Generate<List<string>>()
                         from adoFields in Arb.Generate<Dictionary<string, string>>()
+                        from adoDuplicateFields in Arb.Generate<Dictionary<string, string>>()
                         from dupeTemplate in Arb.Generate<ADODuplicateTemplate>()
                         select new AdoTemplate(
                             baseUrl,
@@ -241,6 +242,7 @@ from dupeTemplate in Arb.Generate<ADODuplicateTemplate>()
                             fields,
                             adoFields,
                             dupeTemplate,
+                            adoDuplicateFields,
                             str.Get));
 
         public static Arbitrary<TeamsTemplate> ArbTeamsTemplate()