Merge branch 'main' into asan_dumps

CURRENT_VERSION

@@ -1 +1 @@
-8.4.0
+8.6.1

src/ApiService/ApiService/ApiService.csproj

@@ -28,7 +28,7 @@
     <PackageReference Include="Microsoft.Azure.Management.Monitor" Version="0.28.0-preview" />
 
     <PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.10.0" OutputItemType="Analyzer" />
-    <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.14.1" />
+    <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.18.0" />
     <PackageReference Include="Azure.Data.Tables" Version="12.8.0" />
     <PackageReference Include="Azure.ResourceManager.Compute" Version="1.0.0-beta.8" />
     <PackageReference Include="Azure.Identity" Version="1.8.2" />
@@ -56,7 +56,6 @@
     <PackageReference Include="Polly.Extensions.Http" Version="3.0.0" />
     <PackageReference Include="OpenTelemetry.Api" Version="1.5.0-rc.1" />
     <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="8.0.0-preview.4.23259.5" />
-    <PackageReference Include="Grpc.Net.Client" Version="2.52.0" />
   </ItemGroup>
   <ItemGroup>
     <None Update="host.json">

src/ApiService/ApiService/FeatureFlags.cs

@@ -7,4 +7,5 @@ public static class FeatureFlagConstants {
     public const string EnableCustomMetricTelemetry = "EnableCustomMetricTelemetry";
     public const string EnableBlobRetentionPolicy = "EnableBlobRetentionPolicy";
     public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention";
+    public const string EnableWorkItemCreation = "EnableWorkItemCreation";
 }

src/ApiService/ApiService/Functions/Containers.cs

@@ -41,7 +41,7 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
                     req,
                     Error.Create(
                         ErrorCode.INVALID_REQUEST,
-                        "invalid container"),
+                        $"invalid container '{get.Name}'"),
                     context: get.Name.String);
             }
 
@@ -75,13 +75,7 @@ private async Async.Task<HttpResponseData> Delete(HttpRequestData req) {
 
         var delete = request.OkV;
         _logger.LogInformation("deleting {ContainerName}", delete.Name);
-        var container = await _context.Containers.FindContainer(delete.Name, StorageType.Corpus);
-
-        var deleted = false;
-        if (container is not null) {
-            deleted = await container.DeleteIfExistsAsync();
-        }
-
+        var deleted = await _context.Containers.DeleteContainerIfExists(delete.Name, StorageType.Corpus);
         return await RequestHandling.Ok(req, deleted);
     }
 
@@ -93,7 +87,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
 
         var post = request.OkV;
         _logger.LogInformation("creating {ContainerName}", post.Name);
-        var sas = await _context.Containers.CreateContainer(
+        var sas = await _context.Containers.GetOrCreateNewContainer(
             post.Name,
             StorageType.Corpus,
             post.Metadata);

src/ApiService/ApiService/Functions/Events.cs

@@ -1,16 +1,13 @@
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Azure.Functions.Worker.Http;
-using Microsoft.Extensions.Logging;
 using Microsoft.OneFuzz.Service.Auth;
 namespace Microsoft.OneFuzz.Service.Functions;
 
 public class EventsFunction {
-    private readonly ILogger _log;
     private readonly IOnefuzzContext _context;
 
-    public EventsFunction(ILogger<EventsFunction> log, IOnefuzzContext context) {
+    public EventsFunction(IOnefuzzContext context) {
         _context = context;
-        _log = log;
     }
 
     [Function("Events")]

src/ApiService/ApiService/Functions/Jobs.cs

@@ -36,17 +36,16 @@ private async Task<HttpResponseData> Post(HttpRequestData req, FunctionContext c
         var userInfo = context.GetUserAuthInfo();
 
         var create = request.OkV;
-        var cfg = new JobConfig(
-            Build: create.Build,
-            Duration: create.Duration,
-            Logs: create.Logs,
-            Name: create.Name,
-            Project: create.Project);
 
         var job = new Job(
             JobId: Guid.NewGuid(),
             State: JobState.Init,
-            Config: cfg,
+            Config: new(
+                Build: create.Build,
+                Duration: create.Duration,
+                Logs: create.Logs,
+                Name: create.Name,
+                Project: create.Project),
             UserInfo: new(
                 ObjectId: userInfo.UserInfo.ObjectId,
                 ApplicationId: userInfo.UserInfo.ApplicationId));
@@ -56,8 +55,11 @@ private async Task<HttpResponseData> Post(HttpRequestData req, FunctionContext c
             { "container_type", "logs" }, // TODO: use ContainerType.Logs enum somehow; needs snake case name
         };
 
-        var containerName = Container.Parse($"logs-{job.JobId}");
-        var containerSas = await _context.Containers.CreateContainer(containerName, StorageType.Corpus, metadata);
+        var containerSas = await _context.Containers.CreateNewContainer(
+            Container.Parse($"logs-{job.JobId}"),
+            StorageType.Corpus,
+            metadata);
+
         if (containerSas is null) {
             return await _context.RequestHandling.NotOk(
                 req,
@@ -76,9 +78,8 @@ private async Task<HttpResponseData> Post(HttpRequestData req, FunctionContext c
             return await _context.RequestHandling.NotOk(
                 req,
                 Error.Create(
-                ErrorCode.UNABLE_TO_CREATE,
-                "unable to create job"
-                ),
+                    ErrorCode.UNABLE_TO_CREATE,
+                    "unable to create job"),
                 "job");
         }
 
@@ -134,10 +135,14 @@ private async Task<HttpResponseData> Get(HttpRequestData req) {
 
             static JobTaskInfo TaskToJobTaskInfo(Task t) => new(t.TaskId, t.Config.Task.Type, t.State);
 
-            // TODO: search.WithTasks is not checked in Python code?
-
-            var taskInfo = await _context.TaskOperations.SearchStates(jobId).Select(TaskToJobTaskInfo).ToListAsync();
-            return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo));
+            var tasks = _context.TaskOperations.SearchStates(jobId);
+            if (search.WithTasks ?? false) {
+                var ts = await tasks.ToListAsync();
+                return await RequestHandling.Ok(req, JobResponse.ForJob(job, ts));
+            } else {
+                var taskInfo = await tasks.Select(TaskToJobTaskInfo).ToListAsync();
+                return await RequestHandling.Ok(req, JobResponse.ForJob(job, taskInfo));
+            }
         }
 
         var jobs = await _context.JobOperations.SearchState(states: search.State ?? Enumerable.Empty<JobState>()).ToListAsync();

src/ApiService/ApiService/Functions/QueueFileChanges.cs

@@ -61,14 +61,17 @@ public async Async.Task Run(
             // requeuing ourselves because azure functions doesn't support retry policies
             // for queue based functions.
 
-            await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
+                await RequeueMessage(msg, TimeSpan.FromDays(1));
+            }
         } catch (Exception e) {
             _log.LogError(e, "File Added failed");
             await RequeueMessage(msg);
         }
     }
 
-    private async Async.Task FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
+    private async Async.Task<OneFuzzResultVoid> FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
         var data = fileChangeEvent.RootElement.GetProperty("data");
         var url = data.GetProperty("url").GetString()!;
         var parts = url.Split("/").Skip(3).ToList();
@@ -77,10 +80,10 @@ private async Async.Task FileAdded(JsonDocument fileChangeEvent, bool isLastRetr
         var path = string.Join('/', parts.Skip(1));
 
         _log.LogInformation("file added : {Container} - {Path}", container, path);
-        await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
+        return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
     }
 
-    private async Async.Task RequeueMessage(string msg) {
+    private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {
         var json = JsonNode.Parse(msg);
 
         // Messages that are 'manually' requeued by us as opposed to being requeued by the azure functions runtime
@@ -103,7 +106,7 @@ await _context.Queue.QueueObject(
             queueName,
             json,
             StorageType.Config,
-            CalculateExponentialBackoff(newCustomDequeueCount))
+            visibilityTimeout ?? CalculateExponentialBackoff(newCustomDequeueCount))
             .IgnoreResult();
     }
 

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -8,7 +8,7 @@ public enum ErrorCode {
     INVALID_PERMISSION = 451,
     MISSING_EULA_AGREEMENT = 452,
     INVALID_JOB = 453,
-    INVALID_TASK = INVALID_JOB,
+    INVALID_TASK = 493,
     UNABLE_TO_ADD_TASK_TO_JOB = 454,
     INVALID_CONTAINER = 455,
     UNABLE_TO_RESIZE = 456,
@@ -47,6 +47,7 @@ public enum ErrorCode {
     ADO_VALIDATION_UNEXPECTED_HTTP_EXCEPTION = 490,
     ADO_VALIDATION_UNEXPECTED_ERROR = 491,
     ADO_VALIDATION_MISSING_PAT_SCOPES = 492,
+    ADO_WORKITEM_PROCESSING_DISABLED = 494,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -173,7 +173,7 @@ public static Error Create(ErrorCode code, params string[] errors)
         => new(code, errors.ToList());
 
     public sealed override string ToString() {
-        var errorsString = Errors != null ? string.Concat("; ", Errors) : string.Empty;
+        var errorsString = Errors != null ? string.Join("; ", Errors) : string.Empty;
         return $"Error {{ Code = {Code}, Errors = {errorsString} }}";
     }
 };
@@ -282,7 +282,8 @@ public record Task(
     ISecret<Authentication>? Auth = null,
     DateTimeOffset? Heartbeat = null,
     DateTimeOffset? EndTime = null,
-    StoredUserInfo? UserInfo = null) : StatefulEntityBase<TaskState>(State) {
+    StoredUserInfo? UserInfo = null) : StatefulEntityBase<TaskState>(State), IJobTaskInfo {
+    public TaskType Type => Config.Task.Type;
 }
 
 public record TaskEvent(
@@ -672,6 +673,17 @@ public async Task<OneFuzzResultVoid> Validate() {
     }
 }
 
+public record RenderedAdoTemplate(
+    Uri BaseUrl,
+    SecretData<string> AuthToken,
+    string Project,
+    string Type,
+    List<string> UniqueFields,
+    Dictionary<string, string> AdoFields,
+    ADODuplicateTemplate OnDuplicate,
+    string? Comment = null
+    ) : AdoTemplate(BaseUrl, AuthToken, Project, Type, UniqueFields, AdoFields, OnDuplicate, Comment);
+
 public record TeamsTemplate(SecretData<string> Url) : NotificationTemplate {
     public Task<OneFuzzResultVoid> Validate() {
         // The only way we can validate in the current state is to send a test webhook
@@ -767,6 +779,11 @@ public record ClientCredentials
     string ClientSecret
 );
 
+public record ContainerInformation(
+    [PartitionKey] StorageType Type,
+    [RowKey] Container Name,
+    string ResourceId // full ARM resource ID for the container
+) : EntityBase;
 
 public record AgentConfig(
     ClientCredentials? ClientCredentials,
@@ -893,11 +910,19 @@ public JobConfig Truncate(int maxLength) {
     }
 }
 
+[JsonDerivedType(typeof(Task), typeDiscriminator: "Task")]
+[JsonDerivedType(typeof(JobTaskInfo), typeDiscriminator: "JobTaskInfo")]
+public interface IJobTaskInfo {
+    Guid TaskId { get; }
+    TaskType Type { get; }
+    TaskState State { get; }
+}
+
 public record JobTaskInfo(
     Guid TaskId,
     TaskType Type,
     TaskState State
-);
+) : IJobTaskInfo;
 
 public record Job(
     [PartitionKey][RowKey] Guid JobId,

src/ApiService/ApiService/OneFuzzTypes/Responses.cs

@@ -96,13 +96,13 @@ public record JobResponse(
     JobConfig Config,
     string? Error,
     DateTimeOffset? EndTime,
-    List<JobTaskInfo>? TaskInfo,
+    IEnumerable<IJobTaskInfo>? TaskInfo,
     StoredUserInfo? UserInfo,
     [property: JsonPropertyName("Timestamp")] // must retain capital T for backcompat
     DateTimeOffset? Timestamp
 // not including UserInfo from Job model
 ) : BaseResponse() {
-    public static JobResponse ForJob(Job j, List<JobTaskInfo>? taskInfo)
+    public static JobResponse ForJob(Job j, IEnumerable<IJobTaskInfo>? taskInfo)
         => new(
             JobId: j.JobId,
             State: j.State,

src/ApiService/ApiService/OneFuzzTypes/ReturnTypes.cs

@@ -2,6 +2,16 @@
 
 namespace Microsoft.OneFuzz.Service {
 
+    public static class Result {
+        public readonly record struct OkPlaceholder();
+        public readonly record struct OkPlaceholder<T>(T Value);
+        public readonly record struct ErrPlaceholder<T>(T Value);
+
+        public static OkPlaceholder Ok() => new();
+        public static OkPlaceholder<T> Ok<T>(T value) => new(value);
+        public static ErrPlaceholder<T> Error<T>(T value) => new(value);
+    }
+
     public readonly struct ResultVoid<T_Error> {
         public static ResultVoid<T_Error> Ok() => new();
         public static ResultVoid<T_Error> Error(T_Error err) => new(err);
@@ -13,15 +23,16 @@ public readonly struct ResultVoid<T_Error> {
         public bool IsOk { get; }
 
         public T_Error? ErrorV { get; }
-    }
 
+        public static implicit operator ResultVoid<T_Error>(Result.OkPlaceholder _ok) => Ok();
+        public static implicit operator ResultVoid<T_Error>(Result.ErrPlaceholder<T_Error> err) => Error(err.Value);
+    }
 
     public readonly struct Result<T_Ok, T_Error> {
         public static Result<T_Ok, T_Error> Ok(T_Ok ok) => new(ok);
         public static Result<T_Ok, T_Error> Error(T_Error err) => new(err);
 
         private Result(T_Ok ok) => (OkV, ErrorV, IsOk) = (ok, default, true);
-
         private Result(T_Error error) => (ErrorV, OkV, IsOk) = (error, default, false);
 
         [MemberNotNullWhen(returnValue: true, member: nameof(OkV))]
@@ -31,6 +42,9 @@ public readonly struct Result<T_Ok, T_Error> {
         public T_Error? ErrorV { get; }
 
         public T_Ok? OkV { get; }
+
+        public static implicit operator Result<T_Ok, T_Error>(Result.OkPlaceholder<T_Ok> ok) => Ok(ok.Value);
+        public static implicit operator Result<T_Ok, T_Error>(Result.ErrPlaceholder<T_Error> err) => Error(err.Value);
     }
 
     public static class OneFuzzResult {
@@ -61,6 +75,8 @@ public readonly struct OneFuzzResult<T_Ok> {
 
         // Allow simple conversion of Errors to Results.
         public static implicit operator OneFuzzResult<T_Ok>(Error err) => new(err);
+        public static implicit operator OneFuzzResult<T_Ok>(Result.OkPlaceholder<T_Ok> ok) => Ok(ok.Value);
+        public static implicit operator OneFuzzResult<T_Ok>(Result.ErrPlaceholder<Error> err) => Error(err.Value);
     }
 
     public readonly struct OneFuzzResultVoid {
@@ -83,5 +99,7 @@ public readonly struct OneFuzzResultVoid {
 
         // Allow simple conversion of Errors to Results.
         public static implicit operator OneFuzzResultVoid(Error err) => new(err);
+        public static implicit operator OneFuzzResultVoid(Result.OkPlaceholder _ok) => Ok;
+        public static implicit operator OneFuzzResultVoid(Result.ErrPlaceholder<Error> err) => Error(err.Value);
     }
 }