From 0390822144b6fde4190ef1b4cf0c8f00fb17dcb6 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:52:40 -0700 Subject: [PATCH 1/3] Increasing Azure App Function Timeout to 12 Hours (#3599) * add * Set function timeout to 12 hours. --- src/ApiService/ApiService/host.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ApiService/ApiService/host.json b/src/ApiService/ApiService/host.json index beb2e4020b..7b83b41ae4 100644 --- a/src/ApiService/ApiService/host.json +++ b/src/ApiService/ApiService/host.json @@ -1,5 +1,6 @@ { "version": "2.0", + "functionTimeout": "12:00:00", "logging": { "applicationInsights": { "samplingSettings": { From 909b1309d8b855763248d4faaad6fbe9dc8b7c63 Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:30:42 -0700 Subject: [PATCH 2/3] JobResult Table Re-design - Improved task- and machine- granularity (#3539) * Release 8.7.1 (hotfix) (#3459) * Remove the retention policy setting (#3452) --------- Co-authored-by: Cheick Keita * Revert "Release 8.7.1 (hotfix) (#3459)" (#3468) This reverts commit c69deed50e81cc1805f6f82ebb10513a211cbbe2. * Redo 8.7.1 (#3469) * Redo-8.7.1-hotfix --------- Co-authored-by: Cheick Keita * Support custom ado fields that mark work items as duplicate (#3467) * Add field to ado config for checking duplicate work items * Make duplicate fields nullable and add it to python models * Update broken tests * Update docs to include new ado_duplicate_fields property * Update readme with archive message (#3408) Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> * Bump tokio from 1.30.0 to 1.32.0 in /src/proxy-manager (#3425) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.30.0 to 1.32.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.30.0...tokio-1.32.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tokio from 1.30.0 to 1.32.0 in /src/agent (#3424) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.30.0 to 1.32.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.30.0...tokio-1.32.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Remove unnecessary method argument (#3473) * Bump elsa from 1.8.1 to 1.9.0 in /src/agent (#3411) Bumps [elsa](https://github.com/manishearth/elsa) from 1.8.1 to 1.9.0. - [Commits](https://github.com/manishearth/elsa/compare/v1.8.1...v1.9.0) --- updated-dependencies: - dependency-name: elsa dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tempfile from 3.7.1 to 3.8.0 in /src/agent (#3437) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.7.1 to 3.8.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.7.1...v3.8.0) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tempfile from 3.7.1 to 3.8.0 in /src/proxy-manager (#3436) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.7.1 to 3.8.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.7.1...v3.8.0) --- updated-dependencies: - dependency-name: tempfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Updating requirements.txt to accept >= onefuzztypes. (#3477) * Updating requirements.txt to accept >= onefuzztypes. * Trying to loosen restriction. * Bump notify from 6.0.1 to 6.1.1 in /src/agent (#3435) Bumps [notify](https://github.com/notify-rs/notify) from 6.0.1 to 6.1.1. - [Release notes](https://github.com/notify-rs/notify/releases) - [Changelog](https://github.com/notify-rs/notify/blob/main/CHANGELOG.md) - [Commits](https://github.com/notify-rs/notify/compare/notify-6.0.1...notify-6.1.1) --- updated-dependencies: - dependency-name: notify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump azure_* crates (#3478) * Release 8.8.0 (#3466) * Release 8.8.0 * Bump clap from 4.3.21 to 4.4.2 in /src/agent (#3484) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.21 to 4.4.2. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.21...v4.4.2) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump gimli from 0.27.3 to 0.28.0 in /src/agent (#3414) Bumps [gimli](https://github.com/gimli-rs/gimli) from 0.27.3 to 0.28.0. - [Changelog](https://github.com/gimli-rs/gimli/blob/master/CHANGELOG.md) - [Commits](https://github.com/gimli-rs/gimli/compare/0.27.3...0.28.0) --- updated-dependencies: - dependency-name: gimli dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump clap from 4.3.21 to 4.4.2 in /src/proxy-manager (#3474) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.21 to 4.4.2. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.21...v4.4.2) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump winreg from 0.50.0 to 0.51.0 in /src/agent (#3434) Bumps [winreg](https://github.com/gentoo90/winreg-rs) from 0.50.0 to 0.51.0. - [Release notes](https://github.com/gentoo90/winreg-rs/releases) - [Changelog](https://github.com/gentoo90/winreg-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/gentoo90/winreg-rs/compare/v0.50.0...v0.51.0) --- updated-dependencies: - dependency-name: winreg dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> * Starting integration tests (#3438) * Starting integration tests * Ready to test the test * Parametrize test * checkpoint * Test works * Run integration tests in pipeline * fmt * . * -p * Install clang * quotes not required in yaml? * Hopefully fixed windows? * Try without killondrop * lint * small test * another test * Reuse core name * Wrong step * bump tokio? * Try with rust * make build happy * Bump pete and small clean up * Clean up and make the test pass regularly * fix broken ci * Lower the poll timeout * Set the timeout in a nicer way * fix windows * fmt * Include and copy pdbs * Ignore if pdb is missing on linux * It takes too long for coverage to be generated * lint * Only warn on missing coverage since it's flaky * Fix windows build * Small clean up * Try lowering the poll delay * fix coverage * PR comments * . * Apparently make is missing? * Remove aggressive step skipping in CI * Fix sed checks for CLI versioning (#3486) * Fix sed checks for CLI versioning * Fix. * Fix. * Changing build_cli * Trying greater than * Tring once more. * Trying major minor * trying to replace major minor * Using major minor * Bump bytes from 1.4.0 to 1.5.0 in /src/agent (#3488) Bumps [bytes](https://github.com/tokio-rs/bytes) from 1.4.0 to 1.5.0. - [Release notes](https://github.com/tokio-rs/bytes/releases) - [Changelog](https://github.com/tokio-rs/bytes/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/bytes/compare/v1.4.0...v1.5.0) --- updated-dependencies: - dependency-name: bytes dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Improve area/iteration path validation (#3489) * Add more comprehensive checks and better error messages to area/iteration path validation * Join invalid chars with space instead of comma * Make tree path validation more testable * Add error code for invalid ADO project in config * Write unit tests for tree path validation * Format tree path unit tests * Merge escape character and control character checks and clarify error message * Improve handling of unexpected breakpoints (#3493) * Improve handling of unexpected breakpoints * fmt * Update azure_* crates (#3503) * Fuzz coverage recording (#3322) * Fuzz coverage recording * Update cargo.toml * Update src/agent/coverage/fuzz/fuzz_targets/fuzz_target_record_coverage.rs Co-authored-by: George Pollard * Fix fuzz --------- Co-authored-by: George Pollard * Reporting coverage on task start up (#3502) * Reporting coverage on task start up * Moving metric up. * Remove feature flag from heartbeat metrics. (#3505) * Update archive notice. (#3507) * Add onefuzz service version to job created events (#3504) * Tevoinea/add version checking in local tasks (#3517) * Compare task version to service version * Swallow output when looking for appropriate name * Create directories if they don't exist in the template (#3522) * Create directories if they don't exist in the template * fmt * Support for retention policies on containers (#3501) - [x] ability to specify a retention period on a container, which applies to newly-created blobs - [x] specify default retention periods in templates from CLI side There's a small breaking change to the Python JobHelper class. * Bump rayon from 1.7.0 to 1.8.0 in /src/agent (#3520) Bumps [rayon](https://github.com/rayon-rs/rayon) from 1.7.0 to 1.8.0. - [Changelog](https://github.com/rayon-rs/rayon/blob/master/RELEASES.md) - [Commits](https://github.com/rayon-rs/rayon/compare/rayon-core-v1.7.0...rayon-core-v1.8.0) --- updated-dependencies: - dependency-name: rayon dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump insta from 1.31.0 to 1.32.0 in /src/agent (#3521) Bumps [insta](https://github.com/mitsuhiko/insta) from 1.31.0 to 1.32.0. - [Changelog](https://github.com/mitsuhiko/insta/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/insta/compare/1.31.0...1.32.0) --- updated-dependencies: - dependency-name: insta dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Disable `repro` and `debug` VM CLI commands. (#3494) * Disable and VM CLI commands. * Formatting. * More formatting. * More formatting. * Removing Repro check. * Make modules case insenstive on windows (#3527) * Make modules and coverage allowlist case insensitive on Windows * Tests and fmt * PR comments * fmt * Debugging missing file coverage * fmt * Broken linux test * Add a case insensitive transformer for better perf * cargo fix * Update windows interceptor list (#3528) * Template creation command (#3531) * Tasks are selectable * Almost there * It works * fmt * remove dead code * Remove unnecessary comments * Improve instructions * fix bug * Add some dummy values for paths * Terminate process on timeout in windows for the coverage task (#3529) * Terminate process on timeout in windows for the coverage task * set the timeout before we start the debugger * split the target launch from the debugger initialization wait for the process to finish on a separate thread * fix build * move comments * Ignore regression update when the work item is in some states (#3532) * Ignore regression update when the work item is in some states * format * formatting * don't hide messages in the poison queue * fix typo * update regression logic update test_template to support regression * build fix * mypy fix * build fix * move regression ignore state under ADODuplicateTemplate * replace extend with append * update set_tcp_keepalive * mke mypy happy * copy ADODuplicateTemplate.OnDuplicate.RegressionIgnoreStates * Updating IterationCount to be Task-based. * Changing to machine_id based * Fixing repro event name. * Updating iteration pr. * Single entry results. * Retry. * trying with unique guid. * Generic string type. * putting it back. * removing old update code. * removing comment. * Attempting to partition on task_id, machine_id, and event type. * Using replace and update. * Add logging statement. * UPdating such that we query. * attempting to try update. * Trying different update mechanism. * Checking previous value. * cleanup. * Removing old model. * Case guard for better readability. * Fix import ordering. * Removing duplicate code. * accidentally didn't include update. * Constructing the wrong way. * Moving back to standard switch. * Removing old code. * Removing more old code. * Using constants. * Addressing comments. * Corrected. * Updating. * Comment. * Replacing with Update to deal with edge cases. * Using timestamp. * Adding CreatedAt field. * Comparing timestamps. * Setting propery. * Adding check for CreatedAt. * Making created_at optional. * Trying again. * Remove. * Add log statement. * Remove function handle.: * Updating * Concat names * Set version. --------- Signed-off-by: dependabot[bot] Co-authored-by: Adam <103067949+AdamL-Microsoft@users.noreply.github.com> Co-authored-by: Cheick Keita Co-authored-by: Kanan B <32438208+kananb@users.noreply.github.com> Co-authored-by: Marc Greisen Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: George Pollard Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com> Co-authored-by: George Pollard --- .../ApiService/Functions/QueueJobResult.cs | 9 +- .../ApiService/OneFuzzTypes/Model.cs | 49 +++----- .../onefuzzlib/JobResultOperations.cs | 118 +++++++----------- src/agent/Cargo.lock | 2 + src/agent/onefuzz-result/Cargo.toml | 6 +- src/agent/onefuzz-result/src/job_result.rs | 9 +- 6 files changed, 87 insertions(+), 106 deletions(-) diff --git a/src/ApiService/ApiService/Functions/QueueJobResult.cs b/src/ApiService/ApiService/Functions/QueueJobResult.cs index 5e3bec0048..31b39802d6 100644 --- a/src/ApiService/ApiService/Functions/QueueJobResult.cs +++ b/src/ApiService/ApiService/Functions/QueueJobResult.cs @@ -31,7 +31,12 @@ public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJo var job = await _jobs.Get(task.JobId); if (job == null) { - _log.LogWarning("invalid {JobId}", task.JobId); + _log.LogWarning("invalid message {JobId}", task.JobId); + return; + } + + if (jr.CreatedAt == null) { + _log.LogWarning("invalid message, no created_at field {JobId}", task.JobId); return; } @@ -52,7 +57,7 @@ public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJo return; } - var jobResult = await _context.JobResultOperations.CreateOrUpdate(job.JobId, jobResultType, value); + var jobResult = await _context.JobResultOperations.CreateOrUpdate(job.JobId, jr.TaskId, jr.MachineId, jr.CreatedAt.Value, jr.Version, jobResultType, value); if (!jobResult.IsOk) { _log.LogError("failed to create or update with job result {JobId}", job.JobId); } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index ab41853a74..4dd4000283 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -34,19 +34,6 @@ public enum HeartbeatType { TaskAlive, } -[SkipRename] -public enum JobResultType { - NewCrashingInput, - NoReproCrashingInput, - NewReport, - NewUniqueReport, - NewRegressionReport, - NewCoverage, - NewCrashDump, - CoverageData, - RuntimeStats, -} - public record HeartbeatData(HeartbeatType Type); public record TaskHeartbeatEntry( @@ -55,12 +42,14 @@ public record TaskHeartbeatEntry( [property: Required] Guid MachineId, HeartbeatData[] Data); -public record JobResultData(JobResultType Type); +public record JobResultData(string Type); public record TaskJobResultEntry( Guid TaskId, Guid? JobId, Guid MachineId, + DateTime? CreatedAt, + double Version, JobResultData Data, Dictionary Value ); @@ -921,26 +910,24 @@ public record SecretAddress(Uri Url) : ISecret { public record SecretData(ISecret Secret) { } +[SkipRename] +public enum JobResultType { + CoverageData, + RuntimeStats, +} + public record JobResult( - [PartitionKey][RowKey] Guid JobId, + [PartitionKey] Guid JobId, + [RowKey] string TaskIdMachineIdMetric, + Guid TaskId, + Guid MachineId, + DateTime CreatedAt, string Project, string Name, - double NewCrashingInput = 0, - double NoReproCrashingInput = 0, - double NewReport = 0, - double NewUniqueReport = 0, - double NewRegressionReport = 0, - double NewCrashDump = 0, - double InstructionsCovered = 0, - double TotalInstructions = 0, - double CoverageRate = 0, - double IterationCount = 0 -) : EntityBase() { - public JobResult(Guid JobId, string Project, string Name) : this( - JobId: JobId, - Project: Project, - Name: Name, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) { } -} + string Type, + double Version, + Dictionary MetricValue +) : EntityBase(); public record JobConfig( string Project, diff --git a/src/ApiService/ApiService/onefuzzlib/JobResultOperations.cs b/src/ApiService/ApiService/onefuzzlib/JobResultOperations.cs index 1166cf91d4..b39c654642 100644 --- a/src/ApiService/ApiService/onefuzzlib/JobResultOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/JobResultOperations.cs @@ -2,99 +2,75 @@ using Microsoft.Extensions.Logging; using Polly; namespace Microsoft.OneFuzz.Service; +using System.Net; public interface IJobResultOperations : IOrm { - Async.Task GetJobResult(Guid jobId); - Async.Task CreateOrUpdate(Guid jobId, JobResultType resultType, Dictionary resultValue); + Async.Task GetJobResult(Guid jobId, Guid taskId, Guid machineId, string metricType); + Async.Task CreateOrUpdate(Guid jobId, Guid taskId, Guid machineId, DateTime createdAt, double version, string resultType, Dictionary resultValue); } public class JobResultOperations : Orm, IJobResultOperations { + const string COVERAGE_DATA = "CoverageData"; + const string RUNTIME_STATS = "RuntimeStats"; + public JobResultOperations(ILogger log, IOnefuzzContext context) : base(log, context) { } - public async Async.Task GetJobResult(Guid jobId) { - return await SearchByPartitionKeys(new[] { jobId.ToString() }).SingleOrDefaultAsync(); + public async Async.Task GetJobResult(Guid jobId, Guid taskId, Guid machineId, string metricType) { + var data = QueryAsync(Query.SingleEntity(jobId.ToString(), string.Concat(taskId, "-", machineId, "-", metricType))); + return await data.FirstOrDefaultAsync(); } - private JobResult UpdateResult(JobResult result, JobResultType type, Dictionary resultValue) { - - var newResult = result; - double newValue; - switch (type) { - case JobResultType.NewCrashingInput: - newValue = result.NewCrashingInput + resultValue["count"]; - newResult = result with { NewCrashingInput = newValue }; - break; - case JobResultType.NewReport: - newValue = result.NewReport + resultValue["count"]; - newResult = result with { NewReport = newValue }; - break; - case JobResultType.NewUniqueReport: - newValue = result.NewUniqueReport + resultValue["count"]; - newResult = result with { NewUniqueReport = newValue }; - break; - case JobResultType.NewRegressionReport: - newValue = result.NewRegressionReport + resultValue["count"]; - newResult = result with { NewRegressionReport = newValue }; - break; - case JobResultType.NewCrashDump: - newValue = result.NewCrashDump + resultValue["count"]; - newResult = result with { NewCrashDump = newValue }; - break; - case JobResultType.CoverageData: - double newCovered = resultValue["covered"]; - double newTotalCovered = resultValue["features"]; - double newCoverageRate = resultValue["rate"]; - newResult = result with { InstructionsCovered = newCovered, TotalInstructions = newTotalCovered, CoverageRate = newCoverageRate }; - break; - case JobResultType.RuntimeStats: - double newTotalIterations = resultValue["total_count"]; - newResult = result with { IterationCount = newTotalIterations }; - break; - default: - _logTracer.LogWarning($"Invalid Field {type}."); - break; - } - _logTracer.LogInformation($"Attempting to log new result: {newResult}"); - return newResult; - } - - private async Async.Task TryUpdate(Job job, JobResultType resultType, Dictionary resultValue) { + private async Async.Task TryUpdate(Job job, Guid taskId, Guid machineId, DateTime createdAt, double version, string resultType, Dictionary resultValue) { var jobId = job.JobId; + var taskIdMachineIdMetric = string.Concat(taskId, "-", machineId, "-", resultType); - var jobResult = await GetJobResult(jobId); - - if (jobResult == null) { - _logTracer.LogInformation("Creating new JobResult for Job {JobId}", jobId); - - var entry = new JobResult(JobId: jobId, Project: job.Config.Project, Name: job.Config.Name); + var oldEntry = await GetJobResult(jobId, taskId, machineId, resultType); - jobResult = UpdateResult(entry, resultType, resultValue); - - var r = await Insert(jobResult); - if (!r.IsOk) { - throw new InvalidOperationException($"failed to insert job result {jobResult.JobId}"); + if (oldEntry == null) { + _logTracer.LogInformation($"attempt to insert new job result {taskId} and taskId+machineId+metricType {taskIdMachineIdMetric}"); + var newEntry = new JobResult(JobId: jobId, TaskIdMachineIdMetric: taskIdMachineIdMetric, TaskId: taskId, MachineId: machineId, CreatedAt: createdAt, Project: job.Config.Project, Name: job.Config.Name, resultType, Version: version, resultValue); + var result = await Insert(newEntry); + if (!result.IsOk) { + throw new InvalidOperationException($"failed to insert job result with taskId {taskId} and taskId+machineId+metricType {taskIdMachineIdMetric}"); } - _logTracer.LogInformation("created job result {JobId}", jobResult.JobId); - } else { - _logTracer.LogInformation("Updating existing JobResult entry for Job {JobId}", jobId); - - jobResult = UpdateResult(jobResult, resultType, resultValue); + return true; + } - var r = await Update(jobResult); - if (!r.IsOk) { - throw new InvalidOperationException($"failed to insert job result {jobResult.JobId}"); - } - _logTracer.LogInformation("updated job result {JobId}", jobResult.JobId); + ResultVoid<(HttpStatusCode Status, string Reason)> r; + switch (resultType) { + case COVERAGE_DATA: + case RUNTIME_STATS: + if (oldEntry.CreatedAt < createdAt) { + oldEntry = oldEntry with { CreatedAt = createdAt, MetricValue = resultValue }; + r = await Update(oldEntry); + if (!r.IsOk) { + throw new InvalidOperationException($"failed to replace job result with taskId {taskId} and machineId+metricType {taskIdMachineIdMetric}"); + } + } else { + _logTracer.LogInformation($"received an out-of-date metric. skipping."); + } + break; + default: + _logTracer.LogInformation($"attempt to update job result {taskId} and taskId+machineId+metricType {taskIdMachineIdMetric}"); + oldEntry.MetricValue["count"]++; + oldEntry = oldEntry with { MetricValue = oldEntry.MetricValue }; + r = await Update(oldEntry); + if (!r.IsOk) { + throw new InvalidOperationException($"failed to update job result with taskId {taskId} and machineId+metricType {taskIdMachineIdMetric}"); + } + break; } + return true; + } - public async Async.Task CreateOrUpdate(Guid jobId, JobResultType resultType, Dictionary resultValue) { + public async Async.Task CreateOrUpdate(Guid jobId, Guid taskId, Guid machineId, DateTime createdAt, double version, string resultType, Dictionary resultValue) { var job = await _context.JobOperations.Get(jobId); if (job == null) { @@ -106,7 +82,7 @@ public async Async.Task CreateOrUpdate(Guid jobId, JobResultT _logTracer.LogInformation("attempt to update job result {JobId}", job.JobId); var policy = Policy.Handle().WaitAndRetryAsync(50, _ => new TimeSpan(0, 0, 5)); await policy.ExecuteAsync(async () => { - success = await TryUpdate(job, resultType, resultValue); + success = await TryUpdate(job, taskId, machineId, createdAt, version, resultType, resultValue); _logTracer.LogInformation("attempt {success}", success); }); return OneFuzzResultVoid.Ok; diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 5393e9b767..e4143473e3 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -463,6 +463,7 @@ dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", + "serde", "winapi 0.3.9", ] @@ -2241,6 +2242,7 @@ version = "0.2.0" dependencies = [ "anyhow", "async-trait", + "chrono", "log", "onefuzz-telemetry", "reqwest", diff --git a/src/agent/onefuzz-result/Cargo.toml b/src/agent/onefuzz-result/Cargo.toml index 7c7de6615c..7e156ac91d 100644 --- a/src/agent/onefuzz-result/Cargo.toml +++ b/src/agent/onefuzz-result/Cargo.toml @@ -9,10 +9,14 @@ license = "MIT" [dependencies] anyhow = { version = "1.0", features = ["backtrace"] } async-trait = "0.1" +chrono = { version = "0.4", default-features = false, features = [ + "clock", + "std", + "serde" +] } reqwest = "0.11" serde = "1.0" storage-queue = { path = "../storage-queue" } uuid = { version = "1.4", features = ["serde", "v4"] } onefuzz-telemetry = { path = "../onefuzz-telemetry" } log = "0.4" - diff --git a/src/agent/onefuzz-result/src/job_result.rs b/src/agent/onefuzz-result/src/job_result.rs index 08f7bbc1ee..e6b4f50377 100644 --- a/src/agent/onefuzz-result/src/job_result.rs +++ b/src/agent/onefuzz-result/src/job_result.rs @@ -3,6 +3,8 @@ use anyhow::Result; use async_trait::async_trait; +use chrono::DateTime; +pub use chrono::Utc; use onefuzz_telemetry::warn; use reqwest::Url; use serde::{self, Deserialize, Serialize}; @@ -32,6 +34,8 @@ struct JobResult { job_id: Uuid, machine_id: Uuid, machine_name: String, + created_at: DateTime, + version: f64, data: JobResultData, value: HashMap, } @@ -103,7 +107,8 @@ impl JobResultSender for TaskJobResultClient { let job_id = self.context.state.job_id; let machine_id = self.context.state.machine_id; let machine_name = self.context.state.machine_name.clone(); - + let created_at = chrono::Utc::now(); + let version = 1.0; let _ = self .context .queue_client @@ -112,6 +117,8 @@ impl JobResultSender for TaskJobResultClient { job_id, machine_id, machine_name, + created_at, + version, data, value, }) From 5cf619ee8fac3062d63f855556015693a632bafa Mon Sep 17 00:00:00 2001 From: Noah McGregor Harper <74685766+nharper285@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:48:18 -0700 Subject: [PATCH 3/3] Move initial coverage reporting to later in startup to capture read-only inputs (#3605) * add * Move intial coverage report farther into task start. * Keeping both calls. --- src/agent/onefuzz-task/src/tasks/coverage/generic.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs index 2ebc748010..8043deee94 100644 --- a/src/agent/onefuzz-task/src/tasks/coverage/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/coverage/generic.rs @@ -146,11 +146,11 @@ impl CoverageTask { bail!("input is not specified on the command line or arguments for the target"); } - context.heartbeat.alive(); - info!("report initial coverage"); context.report_coverage_stats().await; + context.heartbeat.alive(); + for dir in &self.config.readonly_inputs { debug!("recording coverage for {}", dir.local_path.display()); @@ -174,6 +174,9 @@ impl CoverageTask { context.save_and_sync_coverage().await?; } + info!("report coverage"); + context.report_coverage_stats().await; + context.heartbeat.alive(); if let Some(queue) = &self.config.input_queue {