From 433f23ac383748b7b4618ae8b2f796935b2b8307 Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Wed, 13 Mar 2024 21:31:14 +0000 Subject: [PATCH] [preview] Update/azure dev ops api 19 (#1979) Full update to the latest v19 version of the Azure DevOps API files as well as fix test implementation for revisions. --- ...ients.AzureDevops.ObjectModel.Tests.csproj | 2 +- .../TfsRevisionManagerTests.cs | 51 +++++++------- ...ols.Clients.AzureDevops.ObjectModel.csproj | 5 +- .../ProcessorEnrichers/TfsRevisionManager.cs | 66 ++++++++++--------- ...ationTools.Clients.AzureDevops.Rest.csproj | 2 +- .../MigrationTools.Integration.Tests.csproj | 2 +- .../MigrationTools.TestExtensions.csproj | 2 +- src/MigrationTools/MigrationTools.csproj | 4 +- .../WorkItemMigrationContext.cs | 2 +- .../VstsSyncMigrator.Core.csproj | 2 +- 10 files changed, 72 insertions(+), 66 deletions(-) diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/MigrationTools.Clients.AzureDevops.ObjectModel.Tests.csproj b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/MigrationTools.Clients.AzureDevops.ObjectModel.Tests.csproj index 0bc0b65c1..d89f6497a 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/MigrationTools.Clients.AzureDevops.ObjectModel.Tests.csproj +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/MigrationTools.Clients.AzureDevops.ObjectModel.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsRevisionManagerTests.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsRevisionManagerTests.cs index 9e8e6d04a..a31a86569 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsRevisionManagerTests.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsRevisionManagerTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.Extensions.DependencyInjection; using Microsoft.VisualStudio.TestTools.UnitTesting; using MigrationTools.DataContracts; @@ -31,18 +32,16 @@ private static TfsRevisionManagerOptions GetTfsRevisionManagerOptions() return migrationConfig; } - private static WorkItemData GetWorkItemWithRevisions(DateTime currentDateTime, int startHours = 1, int endHours = 1, bool dateIncreasing = true) + private static List GetWorkItemWithRevisions(DateTime currentDateTime, int startHours = 1, int endHours = 1, bool dateIncreasing = true) { - var fakeWorkItem = new WorkItemData(); - fakeWorkItem.Id = Guid.NewGuid().ToString(); - fakeWorkItem.Revisions = new System.Collections.Generic.SortedDictionary(); + var revisions = new System.Collections.Generic.SortedDictionary(); for (int i = startHours; i < endHours + startHours; i++) { - DateTime dateTime = dateIncreasing ? currentDateTime.AddHours(-i) : currentDateTime; - fakeWorkItem.Revisions.Add(i, new RevisionItem() { Index = i, Number = i, ChangedDate = dateTime, OriginalChangedDate = dateTime }); + DateTime dateTime = dateIncreasing ? currentDateTime.AddHours(i) : currentDateTime; + revisions.Add(i, new RevisionItem() { Index = i, Number = i, ChangedDate = dateTime, OriginalChangedDate = dateTime }); } - return fakeWorkItem; + return revisions.Values.ToList(); } @@ -54,8 +53,8 @@ public void TfsRevisionManagerInSync1() processorEnricher.Configure(peOptions); var currentDateTime = System.DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 1); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 1, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 1); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -71,12 +70,12 @@ public void TfsRevisionManagerInSync10() processorEnricher.Configure(peOptions); var currentDateTime = System.DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 10); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 1, 10); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 10); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 10); var revs = processorEnricher.GetRevisionsToMigrate(source, target); - Assert.AreEqual(revs.Count, 0); + Assert.AreEqual(0, revs.Count); } @@ -88,8 +87,8 @@ public void TfsRevisionManagerSync1() processorEnricher.Configure(peOptions); var currentDateTime = System.DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 2); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 2, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 2); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -104,8 +103,8 @@ public void TfsRevisionManagerSync10() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 11); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 11, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 11); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -120,9 +119,9 @@ public void TfsRevisionManagerReplayRevisionsOff() var processorEnricher = Services.GetRequiredService(); processorEnricher.Configure(peOptions); - var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 4); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 4, 1); + var currentDateTime = DateTime.Now.AddDays(-100); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 4); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -139,12 +138,12 @@ public void TfsRevisionManagerMaxRevision51() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 2); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 2, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 2); + List target = GetWorkItemWithRevisions(currentDateTime, 2, 2); var revs = processorEnricher.GetRevisionsToMigrate(source, target); - Assert.AreEqual(1, revs.Count); + Assert.AreEqual(0, revs.Count); } [TestMethod(), TestCategory("L0"), TestCategory("AzureDevOps.ObjectModel")] @@ -156,8 +155,8 @@ public void TfsRevisionManagerMaxRevision56() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 7); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 7, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 7); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -173,7 +172,7 @@ public void TfsRevisionManagerMaxRevision59() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 10); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 10); var revs = processorEnricher.GetRevisionsToMigrate(source, null); @@ -188,7 +187,7 @@ public void TfsRevisionManagerDatesMustBeIncreasing() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime.AddHours(-1000), 1, 10, false); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 10, false); var revs = processorEnricher.GetRevisionsToMigrate(source, null); Assert.AreEqual(true, CheckDateIncreasing(revs)); diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/MigrationTools.Clients.AzureDevops.ObjectModel.csproj b/src/MigrationTools.Clients.AzureDevops.ObjectModel/MigrationTools.Clients.AzureDevops.ObjectModel.csproj index 8063f974f..7e2cbd8fc 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/MigrationTools.Clients.AzureDevops.ObjectModel.csproj +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/MigrationTools.Clients.AzureDevops.ObjectModel.csproj @@ -12,8 +12,9 @@ - - + + + diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsRevisionManager.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsRevisionManager.cs index 631b580de..85ed3087a 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsRevisionManager.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsRevisionManager.cs @@ -74,34 +74,32 @@ protected override void RefreshForProcessorType(IProcessor processor) } } - public List GetRevisionsToMigrate(WorkItemData sourceWorkItem, WorkItemData targetWorkItem) + public List GetRevisionsToMigrate(List sourceRevisions, List targetRevisions) { - // Revisions have been sorted already on object creation. Values of the Dictionary are sorted by RevisionItem.Number - var sortedRevisions = sourceWorkItem.Revisions.Values.ToList(); + EnforceDatesMustBeIncreasing(sourceRevisions); - EnforceDatesMustBeIncreasing(sortedRevisions); + LogDebugCurrentSortedRevisions(sourceRevisions, "Source"); + LogDebugCurrentSortedRevisions(targetRevisions, "Target"); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: Raw {sourceWorkItem} Has {sortedRevisions} revisions", "Source", sourceRevisions.Count); - LogDebugCurrentSortedRevisions(sourceWorkItem, sortedRevisions); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: Raw Source {sourceWorkItem} Has {sortedRevisions} revisions", sourceWorkItem.Id, sortedRevisions.Count); + sourceRevisions = RemoveRevisionsAlreadyOnTarget(targetRevisions, sourceRevisions); - sortedRevisions = RemoveRevisionsAlreadyOnTarget(targetWorkItem, sortedRevisions); + RemoveRevisionsAllExceptLatest(sourceRevisions); - RemoveRevisionsAllExceptLatest(sortedRevisions); + RemoveRevisionsMoreThanMaxRevisions(sourceRevisions); - RemoveRevisionsMoreThanMaxRevisions(sortedRevisions); + LogDebugCurrentSortedRevisions(sourceRevisions); - LogDebugCurrentSortedRevisions(sourceWorkItem, sortedRevisions); - - return sortedRevisions; + return sourceRevisions; } - private void EnforceDatesMustBeIncreasing(List sortedRevisions) + public void EnforceDatesMustBeIncreasing(List sortedRevisions) { Log.LogDebug("TfsRevisionManager::EnforceDatesMustBeIncreasing"); DateTime lastDateTime = DateTime.MinValue; foreach (var revision in sortedRevisions) { - if (revision.OriginalChangedDate == lastDateTime || revision.OriginalChangedDate < lastDateTime) + if (revision.ChangedDate == lastDateTime || revision.OriginalChangedDate < lastDateTime) { revision.ChangedDate = lastDateTime.AddSeconds(1); Log.LogDebug("TfsRevisionManager::EnforceDatesMustBeIncreasing[{revision}]::Fix", revision.Number); @@ -110,15 +108,20 @@ private void EnforceDatesMustBeIncreasing(List sortedRevisions) } } - private void LogDebugCurrentSortedRevisions(WorkItemData sourceWorkItem, List sortedRevisions) + public void LogDebugCurrentSortedRevisions(List sortedRevisions, string designation = "Source") { - Log.LogInformation("Found {RevisionsCount} revisions to migrate on Work item:{sourceWorkItemId}", sortedRevisions.Count, sourceWorkItem.Id); - Log.LogDebug("RevisionsToMigrate:----------------------------------------------------"); + if (sortedRevisions == null) + { + Log.LogDebug("{designation}: RevisionsToMigrate: No revisions to migrate", designation); + return; + } + Log.LogInformation("{designation}: Found {RevisionsCount} revisions to migrate on Work item:{sourceWorkItemId}", designation, sortedRevisions.Count, designation); + Log.LogDebug("{designation}: RevisionsToMigrate:----------------------------------------------------", designation); foreach (RevisionItem item in sortedRevisions) { Log.LogDebug("RevisionsToMigrate: Index:{Index} - Number:{Number} - ChangedDate:{ChangedDate}", item.Index, item.Number, item.ChangedDate); } - Log.LogDebug("RevisionsToMigrate:----------------------------------------------------"); + Log.LogDebug("{designation}: RevisionsToMigrate:----------------------------------------------------", designation); } private void RemoveRevisionsMoreThanMaxRevisions(List sortedRevisions) @@ -130,7 +133,7 @@ private void RemoveRevisionsMoreThanMaxRevisions(List sortedRevisi { var revisionsToRemove = sortedRevisions.Count - Options.MaxRevisions; sortedRevisions.RemoveRange(0, revisionsToRemove); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: MaxRevisions={MaxRevisions}! There are {sortedRevisionsCount} left", Options.MaxRevisions, sortedRevisions.Count); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsMoreThanMaxRevisions MaxRevisions={MaxRevisions}! There are {sortedRevisionsCount} left", Options.MaxRevisions, sortedRevisions.Count); } } @@ -140,29 +143,32 @@ private void RemoveRevisionsAllExceptLatest(List sortedRevisions) { // Remove all but the latest revision if we are not replaying revisions sortedRevisions.RemoveRange(0, sortedRevisions.Count - 1); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: ReplayRevisions=false! There are {sortedRevisionsCount} left", sortedRevisions.Count); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAllExceptLatest ReplayRevisions=false! There are {sortedRevisionsCount} left", sortedRevisions.Count); } } - private List RemoveRevisionsAlreadyOnTarget(WorkItemData targetWorkItem, List sortedRevisions) + private List RemoveRevisionsAlreadyOnTarget(List targetRevisions, List sourceRevisions) { - if (targetWorkItem != null) + if (targetRevisions != null) { - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: Raw Target {targetWorkItemId} Has {targetWorkItemRevCount} revisions", targetWorkItem.Id, targetWorkItem.Revisions.Count); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAlreadyOnTarget Raw Target Has {targetWorkItemRevCount} revisions", targetRevisions.Count); // Target exists so remove any Changed Date matches between them - var targetChangedDates = (from RevisionItem x in targetWorkItem.Revisions.Values select x.ChangedDate).ToList(); + var targetChangedDates = (from RevisionItem x in targetRevisions select x.ChangedDate).ToList(); if (Options.ReplayRevisions) { - sortedRevisions = sortedRevisions.Where(x => !targetChangedDates.Contains(x.ChangedDate)).ToList(); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: After removing Date Matches there are {sortedRevisionsCount} left", sortedRevisions.Count); + sourceRevisions = sourceRevisions.Where(x => !targetChangedDates.Contains(x.ChangedDate)).ToList(); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAlreadyOnTarget After removing Date Matches there are {sortedRevisionsCount} left", sourceRevisions.Count); } // Find Max target date and remove all source revisions that are newer var targetLatestDate = targetChangedDates.Max(); - sortedRevisions = sortedRevisions.Where(x => x.ChangedDate > targetLatestDate).ToList(); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: After removing revisions before target latest date {targetLatestDate} there are {sortedRevisionsCount} left", targetLatestDate, sortedRevisions.Count); + sourceRevisions = sourceRevisions.Where(x => x.ChangedDate > targetLatestDate).ToList(); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAlreadyOnTarget After removing revisions before target latest date {targetLatestDate} there are {sortedRevisionsCount} left", targetLatestDate, sourceRevisions.Count); } - - return sortedRevisions; + else + { + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAlreadyOnTarget Target is null"); + } + return sourceRevisions; } public void AttachSourceRevisionHistoryJsonToTarget(WorkItemData sourceWorkItem, WorkItemData targetWorkItem) diff --git a/src/MigrationTools.Clients.AzureDevops.Rest/MigrationTools.Clients.AzureDevops.Rest.csproj b/src/MigrationTools.Clients.AzureDevops.Rest/MigrationTools.Clients.AzureDevops.Rest.csproj index c1e5d2163..529acd702 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest/MigrationTools.Clients.AzureDevops.Rest.csproj +++ b/src/MigrationTools.Clients.AzureDevops.Rest/MigrationTools.Clients.AzureDevops.Rest.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/MigrationTools.Integration.Tests/MigrationTools.Integration.Tests.csproj b/src/MigrationTools.Integration.Tests/MigrationTools.Integration.Tests.csproj index 2462d5a16..cb8c2ac40 100644 --- a/src/MigrationTools.Integration.Tests/MigrationTools.Integration.Tests.csproj +++ b/src/MigrationTools.Integration.Tests/MigrationTools.Integration.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/MigrationTools.TestExtensions/MigrationTools.TestExtensions.csproj b/src/MigrationTools.TestExtensions/MigrationTools.TestExtensions.csproj index 9c632baf6..03e9dca0f 100644 --- a/src/MigrationTools.TestExtensions/MigrationTools.TestExtensions.csproj +++ b/src/MigrationTools.TestExtensions/MigrationTools.TestExtensions.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/MigrationTools/MigrationTools.csproj b/src/MigrationTools/MigrationTools.csproj index 966ed5d21..47b578d18 100644 --- a/src/MigrationTools/MigrationTools.csproj +++ b/src/MigrationTools/MigrationTools.csproj @@ -26,8 +26,8 @@ - - + + diff --git a/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs b/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs index 10c68e5a7..a1b988954 100644 --- a/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs +++ b/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs @@ -544,7 +544,7 @@ private async Task ProcessWorkItemAsync(WorkItemData sourceWorkItem, int retryLi { "sourceWorkItemRev", sourceWorkItem.Rev }, { "ReplayRevisions", _revisionManager.Options.ReplayRevisions }} ); - List revisionsToMigrate = _revisionManager.GetRevisionsToMigrate(sourceWorkItem, targetWorkItem); + List revisionsToMigrate = _revisionManager.GetRevisionsToMigrate(sourceWorkItem.Revisions.Values.ToList(), targetWorkItem.Revisions.Values.ToList()); if (targetWorkItem == null) { targetWorkItem = ReplayRevisions(revisionsToMigrate, sourceWorkItem, null); diff --git a/src/VstsSyncMigrator.Core/VstsSyncMigrator.Core.csproj b/src/VstsSyncMigrator.Core/VstsSyncMigrator.Core.csproj index a0bd0c186..5435e237f 100644 --- a/src/VstsSyncMigrator.Core/VstsSyncMigrator.Core.csproj +++ b/src/VstsSyncMigrator.Core/VstsSyncMigrator.Core.csproj @@ -23,7 +23,7 @@ - + all none contentFiles;analyzers