Skip to content

Commit

Permalink
Topic/53 cache not reloading correctly after multiple changes to temp…
Browse files Browse the repository at this point in the history
…late work items (#54)

Focus on fixing the cache issue.

1. Resolve #53
2. Bonus Resolve #50
  • Loading branch information
MrHinsh authored Jul 17, 2024
2 parents 8a172cb + 114dff5 commit 097cafb
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 67 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Microsoft.VisualStudio.Services.CircuitBreaker;
using Newtonsoft.Json.Linq;
using System.Diagnostics.Eventing.Reader;
using AzureDevOps.WorkItemClone.Repositories;

namespace AzureDevOps.WorkItemClone.ConsoleUI.Commands
{
Expand All @@ -30,8 +31,6 @@ public override async Task<int> ExecuteAsync(CommandContext context, WorkItemClo
AnsiConsole.MarkupLine($"[red]Run: [/] {config.RunName}");
string runCache = $"{config.CachePath}\\{config.RunName}";
DirectoryInfo outputPathInfo = CreateOutputPath(runCache);

AzureDevOpsApi templateApi = CreateAzureDevOpsConnection(config.templateAccessToken, config.templateOrganization, config.templateProject);
AzureDevOpsApi targetApi = CreateAzureDevOpsConnection(config.targetAccessToken, config.targetOrganization, config.targetProject);

JArray inputWorkItems = DeserializeWorkItemList(config);
Expand Down Expand Up @@ -70,8 +69,7 @@ await AnsiConsole.Progress()
.StartAsync(async ctx =>
{
// Define tasks
var task1 = ctx.AddTask("[bold]Stage 1[/]: Get Template Items", false);
var task2 = ctx.AddTask("[bold]Stage 2[/]: Load Template Items", false);
var task1 = ctx.AddTask("[bold]Stage 1+2[/]: Load Template Items", false);
var task3 = ctx.AddTask("[bold]Stage 3[/]: Get Target Project", false);
var task4 = ctx.AddTask("[bold]Stage 4[/]: Create Output Plan", false);
var task5 = ctx.AddTask("[bold]Stage 5[/]: Create Output Plan Relations ", false);
Expand All @@ -85,72 +83,21 @@ await AnsiConsole.Progress()
System.IO.File.Delete(cacheTemplateWorkItemsFile);
}
task1.MaxValue = 1;
List<WorkItemFull> templateWorkItems = null;
task1.StartTask();
task2.StartTask();
if (System.IO.File.Exists(cacheTemplateWorkItemsFile))
IWorkItemRepository templateWor = new WorkItemRepository(config.CachePath, config.templateOrganization, config.templateProject, config.templateAccessToken, (int)config.templateParentId);
await foreach (var result in templateWor.GetWorkItemsFullAsync())
{
var changedDate = System.IO.File.GetLastWriteTime(cacheTemplateWorkItemsFile).AddDays(1).Date;
//Test Cache
QueryResults fakeItemsFromTemplateQuery;
fakeItemsFromTemplateQuery = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary<string, string>() { { "@id", config.templateParentId.ToString() }, { "@changeddate", changedDate.ToString("yyyy-MM-dd") } });
if (fakeItemsFromTemplateQuery.workItems.Length == 0)
//AnsiConsole.WriteLine($"Stage 2: Processing {workItem.id}:`{workItem.fields.SystemTitle}`");
task1.MaxValue = result.total;
if (result.total == result.processed)
{
AnsiConsole.WriteLine($"Stage 1: Checked template for changes. None Detected. Loading Cache");
// Load from Cache
task1.Increment(1);
task1.Description = task1.Description + " (cache)";
task1.Increment(result.processed);
await Task.Delay(250);
task1.StopTask();
//////////////////////
templateWorkItems = JsonConvert.DeserializeObject<List<WorkItemFull>>(System.IO.File.ReadAllText(cacheTemplateWorkItemsFile));
task2.Increment(templateWorkItems.Count);
task2.Description = task2.Description + " (cache)";
AnsiConsole.WriteLine($"Stage 2: Loaded {templateWorkItems.Count()} work items from cache.");
}
}
if (templateWorkItems == null)
{
// Get From Server
// --------------------------------------------------------------
// Task 1: query for template work items
task1.StartTask();
//AnsiConsole.WriteLine("Stage 1: Executing items from Query");
QueryResults fakeItemsFromTemplateQuery;
fakeItemsFromTemplateQuery = await templateApi.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary<string, string>() { { "@id", config.templateParentId.ToString() } });
AnsiConsole.WriteLine($"Stage 1: Query returned {fakeItemsFromTemplateQuery.workItems.Count()} items id's from the template.");
task1.Description = $"[bold]Stage 1[/]: Load Template Items ({result.loadingFrom})";
task1.Increment(1);
task1.StopTask();
// --------------------------------------------------------------
// Task 2: getting work items and their full data
task2.MaxValue = fakeItemsFromTemplateQuery.workItems.Count();
task2.StartTask();
await Task.Delay(250);
//AnsiConsole.WriteLine($"Stage 2: Starting process of {task2.MaxValue} work items to get their full data ");
templateWorkItems = new List<WorkItemFull>();
//AnsiConsole.WriteLine($"Stage 2: Loading {fakeItemsFromTemplateQuery.workItems.Count()} work items from template.");
await foreach (var workItem in templateApi.GetWorkItemsFullAsync(fakeItemsFromTemplateQuery.workItems))
{
//AnsiConsole.WriteLine($"Stage 2: Processing {workItem.id}:`{workItem.fields.SystemTitle}`");
templateWorkItems.Add(workItem);
task2.Increment(1);
}
System.IO.File.WriteAllText(cacheTemplateWorkItemsFile, JsonConvert.SerializeObject(templateWorkItems, Formatting.Indented));
//AnsiConsole.WriteLine($"Stage 2: All {task2.MaxValue} work items loaded");
await Task.Delay(250);
task2.StopTask();
}
await Task.Delay(250);
task1.StopTask();
// --------------------------------------------------------------
string targetProjectRunFile = $"{runCache}\\targetProject.json";
Expand Down Expand Up @@ -206,7 +153,7 @@ await AnsiConsole.Progress()
await Task.Delay(250);
//AnsiConsole.WriteLine($"Stage 4: First Pass generation of Work Items to build will merge the provided json work items with the data from the template.");
buildItems = new List<WorkItemToBuild>();
await foreach (WorkItemToBuild witb in generateWorkItemsToBuildList(inputWorkItems, templateWorkItems, projectItem, config.targetProject))
await foreach (WorkItemToBuild witb in generateWorkItemsToBuildList(inputWorkItems, templateWor.Data.workitems, projectItem, config.targetProject))
{
// AnsiConsole.WriteLine($"Stage 4: processing {witb.guid}");
buildItems.Add(witb);
Expand All @@ -221,7 +168,7 @@ await AnsiConsole.Progress()
//AnsiConsole.WriteLine($"Stage 5: Second Pass generate relations.");
task5.StartTask();
await Task.Delay(250);
await foreach (WorkItemToBuild witb in generateWorkItemsToBuildRelations(buildItems, templateWorkItems))
await foreach (WorkItemToBuild witb in generateWorkItemsToBuildRelations(buildItems, templateWor.Data.workitems))
{
//AnsiConsole.WriteLine($"Stage 5: processing {witb.guid} for output of {witb.relations.Count-1} relations");
task5.Increment(1);
Expand Down Expand Up @@ -252,7 +199,7 @@ await AnsiConsole.Progress()
//AnsiConsole.WriteLine($"Stage 6: Processing {witb.guid} for output of {witb.relations.Count - 1} relations");
task6.Increment(1);
taskCount++;
task6.Description = $"[bold]Stage 6[/]: Create Work Items ({taskCount}/{buildItems.Count()} c:{result.created}, s:{result.skipped}, f:{result.failed})";
task6.Description = $"[bold]Stage 6[/]: Create Work Items ({taskCount-1}/{buildItems.Count()} c:{result.created}, s:{result.skipped}, f:{result.failed})";
switch (result.status)
{
case "created":
Expand Down
6 changes: 5 additions & 1 deletion AzureDevOps.WorkItemClone/DataContracts/WorkItem.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@

namespace AzureDevOps.WorkItemClone.DataContracts
{

public class CashedWorkItems
{
public List<WorkItemFull> workitems { get; set; }
public DateTime queryDatetime { get; set; }
}

public class WorkItemFull
{
Expand Down
113 changes: 113 additions & 0 deletions AzureDevOps.WorkItemClone/Repositories/WorkItemRepo.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
using AzureDevOps.WorkItemClone.DataContracts;
using Newtonsoft.Json;
using Spectre.Console;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace AzureDevOps.WorkItemClone.Repositories
{
public interface IWorkItemRepository
{
CashedWorkItems Data {get;}
IAsyncEnumerable<(int total, int processed, string loadingFrom)> GetWorkItemsFullAsync();
}
public interface IPersistantCache
{
Task SaveToCache();
Task LoadFromCache();
}

public class WorkItemRepository : IWorkItemRepository
{
public string OrganisationName { get; private set; }
public string ProjectName { get; private set; }
private string AccesToken { get; set; }
public int ParentId { get; private set; }

private AzureDevOpsApi _context;
private string cacheWorkItemsFile;
public CashedWorkItems Data { get { return cachedWorkItems; } }

CashedWorkItems cachedWorkItems = null;

public WorkItemRepository(string cachePath, string organisationName, string projectName, string accessToken, int parentId)
{
if (string.IsNullOrEmpty(organisationName))
{
throw new ArgumentNullException(nameof(organisationName));
}
this.OrganisationName = organisationName;
if (string.IsNullOrEmpty(projectName))
{
throw new ArgumentNullException(nameof(projectName));
}
this.ProjectName = projectName;
if (string.IsNullOrEmpty(accessToken))
{
throw new ArgumentNullException(nameof(accessToken));
}
this.AccesToken = accessToken;
if (parentId == 0)
{
throw new ArgumentNullException(nameof(parentId));
}
this.ParentId = parentId;
_context = new AzureDevOpsApi(accessToken, organisationName, projectName);
cacheWorkItemsFile = $"{cachePath}\\cache-{organisationName}-{projectName}-{ParentId}.json";
}


public async IAsyncEnumerable<(int total, int processed, string loadingFrom)> GetWorkItemsFullAsync()
{
if (System.IO.File.Exists(cacheWorkItemsFile))
{
// load Cache
try
{
cachedWorkItems = JsonConvert.DeserializeObject<CashedWorkItems>(System.IO.File.ReadAllText(cacheWorkItemsFile));
}
catch (Exception ex)
{
// failed to load:: do nothing we will refresh the cache.
}
if (cachedWorkItems != null)
{
//Test Cache date
QueryResults? changedWorkItems = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id AND [System.ChangedDate] > '@changeddate' order by [System.CreatedDate] desc", new Dictionary<string, string>() { { "@id", ParentId.ToString() }, { "@changeddate", cachedWorkItems.queryDatetime.AddDays(-1).ToString("yyyy-MM-dd") } });
if (changedWorkItems?.workItems.Length == 0)
{
yield return (cachedWorkItems.workitems.Count(), cachedWorkItems.workitems.Count(), "cache");
}
else
{
cachedWorkItems = null;
}
}
}
if (cachedWorkItems == null)
{

QueryResults? templateWorkItemLight;
templateWorkItemLight = await _context.GetWiqlQueryResults("Select [System.Id] From WorkItems Where [System.TeamProject] = '@project' AND [System.Parent] = @id order by [System.CreatedDate] desc", new Dictionary<string, string>() { { "@id", ParentId.ToString() } });
cachedWorkItems = new CashedWorkItems() { queryDatetime = templateWorkItemLight.asOf, workitems = new List<WorkItemFull>() };
int count = 1;
foreach (var item in templateWorkItemLight?.workItems)
{
WorkItemFull result = await _context.GetWorkItem((int)item.id);
if (result != null)
{
cachedWorkItems.workitems.Add(result);
}
yield return (templateWorkItemLight.workItems.Count(), count, "server");
count++;
}
System.IO.File.WriteAllText(cacheWorkItemsFile, JsonConvert.SerializeObject(cachedWorkItems, Formatting.Indented));
}

}

}
}

0 comments on commit 097cafb

Please sign in to comment.