From ada3f774ffbacfde67f1db9638c6753da2ac3a60 Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Fri, 15 Mar 2024 19:53:23 +0000 Subject: [PATCH 1/2] 15.0.1 - Fix pack + User Mappings (#1977) ## Consecutive Dates The system will now check that the Revision dates are consecutive. If there are a bunch of identical dates, it will push each duplicate date by 1 second to ensure that each one is consecutive. #1660 #1975 ## User Mappings There was a request to have the ability to map users to try and maintain integrity across different systems. We added a TfsUserMappingEnricher` that allows you to map users from Source to Target... this is not free and takes some work. Runnin the `ExportUsersForMappingConfig` to get the list of users will produce: ``` [ { "Source": { "FriendlyName": "Martin Hinshelwood nkdAgility.com", "AccountName": "martin@nkdagility.com" }, "target": { "FriendlyName": "Hinshelwood, Martin", "AccountName": "martin@othercompany.com" } }, { "Source": { "FriendlyName": "Rollup Bot", "AccountName": "Bot@nkdagility.com" }, "target": { "FriendlyName": "Service Account 4", "AccountName": "randoaccount@somecompany.com" } }, { "Source": { "FriendlyName": "Another non mapped Account", "AccountName": "not-mapped@nkdagility.com" }, "target": null } ] ``` ##How it works 1. Run `ExportUsersForMappingConfig` which will export all of the Users in Soruce Mapped or not to target. 2. Run `WorkItemMigrationConfig` which will run a validator by detail to warn you of missing users. If it finds a mapping it will convert the field... ##Notes - Applies to all identity fields specified in the list - It really sucks that we have to match on Display name! Email is included for internal matching - On 'ExportUsersForMappingConfig` you can set `OnlyListUsersInWorkItems` to filter the mapping based on the scope of the query. This is greater if you have many users. - Both use the `TfsUserMappingEnricherOptions` setting in `CommonEnrichersConfig` to know what to do. ``` { "ChangeSetMappingFile": null, "Source": { "$type": "TfsTeamProjectConfig", "Collection": "https://dev.azure.com/nkdagility/", "Project": "AzureDevOps-Tools", "ReflectedWorkItemIDFieldName": "nkdScrum.ReflectedWorkItemId", "AllowCrossProjectLinking": false, "AuthenticationMode": "Prompt", "PersonalAccessToken": "", "PersonalAccessTokenVariableName": "", "LanguageMaps": { "AreaPath": "Area", "IterationPath": "Iteration" } }, "Target": { "$type": "TfsTeamProjectConfig", "Collection": "https://dev.azure.com/nkdagility-preview/", "Project": "migrationTest5", "ReflectedWorkItemIDFieldName": "nkdScrum.ReflectedWorkItemId", "AllowCrossProjectLinking": false, "AuthenticationMode": "Prompt", "PersonalAccessToken": "", "PersonalAccessTokenVariableName": "", "LanguageMaps": { "AreaPath": "Area", "IterationPath": "Iteration" } }, "FieldMaps": [], "GitRepoMapping": null, "LogLevel": "Debug", "CommonEnrichersConfig": [ { "$type": "TfsUserMappingEnricherOptions", "Enabled": true, "UserMappingFile": "C:\\temp\\userExport.json", "IdentityFieldsToCheck": [ "System.AssignedTo", "System.ChangedBy", "System.CreatedBy", "Microsoft.VSTS.Common.ActivatedBy", "Microsoft.VSTS.Common.ResolvedBy", "Microsoft.VSTS.Common.ClosedBy" ] } ], "Processors": [ { "$type": "ExportUsersForMappingConfig", "Enabled": true, "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc", "OnlyListUsersInWorkItems": true } ], "Version": "15.0" } ``` Resolves #1976 Export users for mapping context (#1969) for MrHinsh --------- Co-authored-by: Nico Orschel Co-authored-by: Tom Frenzel <40773830+tomfrenzel@users.noreply.github.com> --- .gitignore | 1 + GitVersion.yml | 18 +- README.md | 47 +++- configuration.json | 36 ++- ...nTools.Clients.AzureDevops.ObjectModel.xml | 28 +++ docs/Reference/Generated/MigrationTools.xml | 23 +- .../Generated/VstsSyncMigrator.Core.xml | 8 + .../Processors/ExportUsersForMapping-notes.md | 97 ++++++-- .../WorkItemMigrationContext-notes.md | 14 +- .../TfsNodeStructure-notes.md | 21 ++ ...ocessors.exportusersformappingcontext.yaml | 34 +++ ...1.processors.workitemmigrationcontext.yaml | 15 -- ...rs.appendmigrationtoolsignaturefooter.yaml | 4 +- ...lterworkitemsthatalreadyexistintarget.yaml | 4 +- ...processorenrichers.pauseaftereachitem.yaml | 4 +- ...ichers.skiptofinalrevisedworkitemtype.yaml | 4 +- ...orenrichers.stringmanipulatorenricher.yaml | 4 +- ...cessorenrichers.tfsattachmentenricher.yaml | 38 +++ ...2.processorenrichers.tfsnodestructure.yaml | 4 +- ...processorenrichers.tfsrevisionmanager.yaml | 4 +- ...essorenrichers.tfsusermappingenricher.yaml | 47 +++- ...sorenrichers.tfsvalidaterequiredfield.yaml | 4 +- ...ssorenrichers.tfsworkitemlinkenricher.yaml | 4 +- .../_includes/sampleConfig/configuration.json | 154 ++++-------- ...processors.exportusersformappingcontext.md | 55 +++++ ....v1.processors.workitemmigrationcontext.md | 39 +-- ...hers.appendmigrationtoolsignaturefooter.md | 4 +- ...filterworkitemsthatalreadyexistintarget.md | 4 +- ...2.processorenrichers.pauseaftereachitem.md | 4 +- ...nrichers.skiptofinalrevisedworkitemtype.md | 4 +- ...ssorenrichers.stringmanipulatorenricher.md | 4 +- ...rocessorenrichers.tfsattachmentenricher.md | 59 +++++ ....v2.processorenrichers.tfsnodestructure.md | 163 ++++++++----- ...2.processorenrichers.tfsrevisionmanager.md | 4 +- ...ocessorenrichers.tfsusermappingenricher.md | 47 +++- ...essorenrichers.tfsvalidaterequiredfield.md | 4 +- ...cessorenrichers.tfsworkitemlinkenricher.md | 4 +- docs/getting-started.md | 6 +- docs/index.md | 37 ++- ...ients.AzureDevops.ObjectModel.Tests.csproj | 10 +- .../TfsNodeStructureTests.cs | 182 ++++++++++++++ .../TfsRevisionManagerTests.cs | 78 ++++-- .../Endpoints/TfsWorkItemConvertor.cs | 3 + .../Enrichers/TfsUserMappingEnricher.cs | 86 ------- ...ols.Clients.AzureDevops.ObjectModel.csproj | 5 +- .../TfsAttachmentEnricher.cs | 95 +++++--- .../TfsAttachmentEnricherOptions.cs | 48 ++++ .../ProcessorEnrichers/TfsNodeStructure.cs | 60 ++++- .../ProcessorEnrichers/TfsRevisionManager.cs | 77 ++++-- .../TfsUserMappingEnricher.cs | 224 ++++++++++++++++++ .../TfsUserMappingEnricherOptions.cs | 50 ++++ .../TfsValidateRequiredField.cs | 30 ++- .../ServiceCollectionExtensions.cs | 3 + ...ools.Clients.AzureDevops.Rest.Tests.csproj | 8 +- ...ationTools.Clients.AzureDevops.Rest.csproj | 2 +- ...ationTools.Clients.FileSystem.Tests.csproj | 8 +- ...grationTools.Clients.InMemory.Tests.csproj | 8 +- ...MigrationTools.ConsoleDataGenerator.csproj | 2 +- .../Properties/launchSettings.json | 2 +- .../MigrationTools.Host.Tests.csproj | 8 +- src/MigrationTools.Host/MigrationToolHost.cs | 10 +- .../MigrationTools.Host.csproj | 4 +- .../MigrationTools.Integration.Tests.csproj | 10 +- .../MigrationTools.TestExtensions.csproj | 2 +- .../MigrationTools.Tests.csproj | 10 +- src/MigrationTools/DataContracts/FieldItem.cs | 1 + .../DataContracts/IdentityItemData.cs | 18 ++ .../DataContracts/RevisionItem.cs | 2 + src/MigrationTools/MigrationTools.csproj | 6 +- .../ProcessorEnricherOptions.cs | 3 +- .../Services/TelemetryClientAdapter.cs | 1 + .../Configuration/EngineConfiguration.cs | 8 +- .../Processing/ExportUsersForMappingConfig.cs | 10 +- .../Processing/WorkItemMigrationConfig.cs | 23 -- .../Processors/MigrationProcessorBase.cs | 13 +- .../VstsSyncMigrator.Core.Tests.csproj | 8 +- .../WorkItemMigrationTests.cs | 188 +-------------- .../MigrationContext/ExportUsersForMapping.cs | 79 ++++-- .../TestPlansAndSuitesMigrationContext.cs | 6 +- .../WorkItemMigrationContext.cs | 142 +++++------ .../ServiceCollectionExtensions.cs | 3 +- .../VstsSyncMigrator.Core.csproj | 4 +- 82 files changed, 1742 insertions(+), 853 deletions(-) create mode 100644 docs/_data/reference.v1.processors.exportusersformappingcontext.yaml create mode 100644 docs/_data/reference.v2.processorenrichers.tfsattachmentenricher.yaml create mode 100644 docs/collections/_reference/reference.v1.processors.exportusersformappingcontext.md create mode 100644 docs/collections/_reference/reference.v2.processorenrichers.tfsattachmentenricher.md delete mode 100644 src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsUserMappingEnricher.cs rename src/MigrationTools.Clients.AzureDevops.ObjectModel/{Enrichers => ProcessorEnrichers}/TfsAttachmentEnricher.cs (55%) create mode 100644 src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricherOptions.cs create mode 100644 src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricher.cs create mode 100644 src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricherOptions.cs create mode 100644 src/MigrationTools/DataContracts/IdentityItemData.cs diff --git a/.gitignore b/.gitignore index 9d7187e17..d5c82fe0f 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,4 @@ $RECYCLE.BIN/ docs/Reference/Generated/MigrationTools.Host.xml /docs/Reference/Generated/MigrationTools.Host.xml +/docs/.obsidian diff --git a/GitVersion.yml b/GitVersion.yml index 323cc9ff9..9601b3c52 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -1,20 +1,36 @@ assembly-versioning-scheme: MajorMinorPatch mode: ContinuousDeployment continuous-delivery-fallback-tag: '' -next-version: 12.5.0 +next-version: 15.0.1 branches: main: mode: ContinuousDeployment tag: '' + increment: Patch regex: ^master$|^main$ is-release-branch: true is-mainline: true + prevent-increment-of-merged-branch-version: true + track-merge-target: false + tracks-release-branches: false preview: + mode: ContinuousDeployment tag: Preview regex: ^((?!(master)|(feature)|(pull)).) is-release-branch: false is-mainline: false source-branches: [ 'main', 'master' ] + prevent-increment-of-merged-branch-version: false + track-merge-target: false + tracks-release-branches: true + feature: + mode: ContinuousDelivery + tag: useBranchName + increment: Inherit + regex: features?[/-] + prevent-increment-of-merged-branch-version: false + track-merge-target: false + tracks-release-branches: false ignore: sha: [] merge-message-formats: {} diff --git a/README.md b/README.md index a8f51c195..ed26c6047 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,20 @@ The Azure DevOps Migration Tools allow you to bulk edit and migrate data between **Ask Questions on Github: https://github.com/nkdAgility/azure-devops-migration-tools/discussions** -## Some Data from the last 30 days (as of 06/04/2023) +## Some Data from the last 30 days (as of 05/03/2024) | Category | Metric | Notes | | ------------- | ------------- | ------------- | -| Work Item Revisions | **14m** | A single Work Item may have many revisions that we need to migrate | -| Average Work item Migration Time | **35s** | Work Item (includes all revisions, links, and attachments for the work item) | -| RelatedLinkCount | **5m** | Each work item may have many links or none. | -| Git Commit Links | **480k** | | -| Attachments | **252.37k** | Total number of attachments migrated | +| Work Items | **1m** | A single Work Item may have many revisions that we need to migrate | +| Work Item Revisions | **23m** | A single Work Item may have many revisions that we need to migrate | +| RelatedLinkCount | **11m** | Each work item may have many links or none. | +| Git Commit Links | **1.3m** | | +| Attachments | **1.2m** | Total number of attachments migrated | | Test Suits | 52k | total suits migrated | -| Test Cases Mapped | **800k** | Total test cases mapped into Suits | +| Test Cases Mapped | **1.4m** | Total test cases mapped into Suits | | Migration Run Ave | **14 minutes** | Includes dry-runs as well. | | Migration Run Total | **19bn Seconds** | Thats **316m hours** or **13m days** of run time in the last 30 days. | +| Average Work item Migration Time | **22s** | Work Item (includes all revisions, links, and attachments for the work item) | ## What can you do with this tool? @@ -42,7 +43,7 @@ The Azure DevOps Migration Tools allow you to bulk edit and migrate data between - Bulk edit of Work Items - Migration of Test Suites & Test Plans - _new_ Migration of Builds & Pipelines -- Migrate from one Language version of TFS / Azure Devops to another (*new v9.0*) +- Migrate from one Language version of TFS / Azure Devops to another (*new v9.0*)1.34 - _new_ Migration of Processes **NOTE: If you are able to migrate your entire Collection to Azure DevOps Services you should use [Azure DevOps Migration Service](https://azure.microsoft.com/services/devops/migrate/) from Microsoft. If you have a requirement to change Process Template then you will need to do that before you move to Azure DevOps Services.** @@ -56,11 +57,37 @@ The Azure DevOps Migration Tools allow you to bulk edit and migrate data between ## Installing and running the tools -We use [winget](https://learn.microsoft.com/en-us/windows/package-manager/winget/) to host the tools, and you can use the command `winget install nkdAgility.AzureDevOpsMigrationTools` to install them on Windows 10 and Windows 11. For other operating systems you can download the [latest release](https://github.com/nkdAgility/azure-devops-migration-tools/releases/latest) and unzip it to a folder of your choice. +These tools are available as a portable application and can be installed in a number of ways, including manually from a zip. +For a more detailed getting started guide please see the [documentation](https://nkdagility.com/docs/azure-devops-migration-tools/getting-started.html). + +### Option 1: Winget + +We use [winget](https://learn.microsoft.com/en-us/windows/package-manager/winget/) to host the tools, and you can use the command `winget install nkdAgility.AzureDevOpsMigrationTools` to install them on Windows 10 and Windows 11. The tools will be installed to `%Localappdata%\Microsoft\WinGet\Packages\nkdAgility.AzureDevOpsMigrationTools_Microsoft.Winget.Source_XXXXXXXXXX` and a symbolic link to `devopsmigration.exe` that lets you run it from anywhere using `devopsmigration init`. -For a more detailed getting started guide please see the [documentation](https://nkdagility.com/docs/azure-devops-migration-tools/getting-started.html). +**NOTE: Do not install using an elevated command prompt!** + +### Option 2: Chocolatey + +We also deploy to [Chocolatey](https://chocolatey.org/packages/nkdagility.azuredevopsmigrationtools) and you can use the command `choco install vsts-sync-migrator` to install them on Windows Server. + +The tools will be installed to `C:\Tools\MigrationTools\` which should be added to the path. You can run `devopsmigration.exe` + +### Option 3: Manual + +You can download the [latest release](https://github.com/nkdAgility/azure-devops-migration-tools/releases/latest) and unzip it to a folder of your choice. + +## Advanced tools + +There are additional advanced tooling available on [Azure DevOps Automation Tools](https://github.com/nkdAgility/azure-devops-automation-tools). These are a collection of Powershell scripts that can be used to; + +- Generate Migration Tools configurations across many projects on many organisations +- Export Stats on many projects on many organisations +- Publish Custom fields across many projects on many organisations +- Output the fields and other data for many projects on many organisations + +These tools are designed to help you manage migration of Work Items at scale. ## Support diff --git a/configuration.json b/configuration.json index 02975fca6..fd7b91cb1 100644 --- a/configuration.json +++ b/configuration.json @@ -36,9 +36,11 @@ "$type": "TfsNodeStructureOptions", "NodeBasePaths": [], "AreaMaps": { + "^nkdProducts([\\\\]?.*)$": "MigrationTest5$1", "^Skypoint Cloud$": "MigrationTest5" }, "IterationMaps": { + "^nkdProducts([\\\\]?.*)$": "MigrationTest5$1", "^Skypoint Cloud\\\\Sprint 1$": "MigrationTest5\\Sprint 1" }, "ShouldCreateMissingRevisionPaths": true, @@ -56,6 +58,12 @@ "ReplayRevisions": true, "MaxRevisions": 0 }, + { + "$type": "TfsAttachmentEnricherOptions", + "Enabled": true, + "ExportBasePath": "c:\\temp\\WorkItemAttachmentExport", + "MaxRevisions": 480000000 + }, { "$type": "StringManipulatorEnricherOptions", "Enabled": true, @@ -69,24 +77,34 @@ "Description": "Remove invalid characters from the end of the string" } ] + }, + { + "$type": "TfsUserMappingEnricherOptions", + "Enabled": true, + "UserMappingFile": "C:\\temp\\userExport.json", + "IdentityFieldsToCheck": [ + "System.AssignedTo", + "System.ChangedBy", + "System.CreatedBy", + "Microsoft.VSTS.Common.ActivatedBy", + "Microsoft.VSTS.Common.ResolvedBy", + "Microsoft.VSTS.Common.ClosedBy" + ] } ], "Processors": [ { "$type": "WorkItemMigrationConfig", - "Enabled": false, + "Enabled": true, "UpdateCreatedDate": true, "UpdateCreatedBy": true, - "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc", + "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Program', 'Portfolio', 'Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc", "LinkMigration": true, - "AttachmentMigration": true, - "AttachmentWorkingPath": "c:\\temp\\WorkItemAttachmentWorkingFolder\\", "FixHtmlAttachmentLinks": false, "SkipToFinalRevisedWorkItemType": false, "WorkItemCreateRetryLimit": 5, "FilterWorkItemsThatAlreadyExistInTarget": false, "PauseAfterEachWorkItem": false, - "AttachmentMaxSize": 480000000, "AttachRevisionHistory": false, "LinkMigrationSaveEachAsAdded": false, "GenerateMigrationComment": true, @@ -94,9 +112,15 @@ "MaxGracefulFailures": 0, "SkipRevisionWithInvalidIterationPath": false, "SkipRevisionWithInvalidAreaPath": false + }, + { + "$type": "ExportUsersForMappingConfig", + "Enabled": false, + "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Program', 'Portfolio', 'Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc", + "OnlyListUsersInWorkItems": true } ], - "Version": "14.2", + "Version": "15.0", "workaroundForQuerySOAPBugEnabled": false, "WorkItemTypeDefinition": { "sourceWorkItemTypeName": "targetWorkItemTypeName" diff --git a/docs/Reference/Generated/MigrationTools.Clients.AzureDevops.ObjectModel.xml b/docs/Reference/Generated/MigrationTools.Clients.AzureDevops.ObjectModel.xml index 771de80dc..45f19c8e2 100644 --- a/docs/Reference/Generated/MigrationTools.Clients.AzureDevops.ObjectModel.xml +++ b/docs/Reference/Generated/MigrationTools.Clients.AzureDevops.ObjectModel.xml @@ -13,6 +13,19 @@ from https://gist.github.com/pietergheysens/792ed505f09557e77ddfc1b83531e4fb + + + `AttachmentMigration` is set to true then you need to specify a working path for attachments to be saved locally. + + C:\temp\Migration\ + + + + `AttachmentMigration` is set to true then you need to specify a max file size for upload in bites. + For Azure DevOps Services the default is 480,000,000 bites (60mb), for TFS its 32,000,000 bites (4mb). + + 480000000 + The TfsNodeStructureEnricher is used to create missing nodes in the target project. To configure it add a `TfsNodeStructureOptions` section to `CommonEnrichersConfig` in the config file. Otherwise defaults will be applied. @@ -77,6 +90,16 @@ 0 + + + This is a list of the Identiy fields in the Source to check for user mapping purposes. You should list all identiy fields that you wan to map. + + + + + This is the file that will be used to export or import the user mappings. Use the ExportUsersForMapping processor to create the file. + + Skip validating links if the number of links in the source and the target matches! @@ -88,6 +111,11 @@ false + + + The TfsUserMappingEnricher is used to map users from the source to the target system. Run it with the ExportUsersForMappingContext to create a mapping file then with WorkItemMigrationContext to use the mapping file to update the users in the target system as you migrate the work items. + + The `TfsAreaAndIterationProcessor` migrates all of the Area nd Iteraion paths. diff --git a/docs/Reference/Generated/MigrationTools.xml b/docs/Reference/Generated/MigrationTools.xml index 232c4d987..dd598bed6 100644 --- a/docs/Reference/Generated/MigrationTools.xml +++ b/docs/Reference/Generated/MigrationTools.xml @@ -47,8 +47,9 @@ - For internal use + If enabled this will run this migrator + true @@ -241,6 +242,7 @@ + @@ -379,18 +381,6 @@ ? - - - If enabled this will migrate all of the attachments at the same time as the work item - - true - - - - `AttachmentMigration` is set to true then you need to specify a working path for attachments to be saved locally. - - C:\temp\Migration\ - **beta** If enabled this will fix any image attachments URL's, work item mention URL's or user mentions in the HTML @@ -426,13 +416,6 @@ false - - - `AttachmentMigration` is set to true then you need to specify a max file size for upload in bites. - For Azure DevOps Services the default is 480,000,000 bites (60mb), for TFS its 32,000,000 bites (4mb). - - 480000000 - This will create a json file with the revision history and attach it to the work item. Best used with `MaxRevisions` or `ReplayRevisions`. diff --git a/docs/Reference/Generated/VstsSyncMigrator.Core.xml b/docs/Reference/Generated/VstsSyncMigrator.Core.xml index 283d6b633..63c78e83e 100644 --- a/docs/Reference/Generated/VstsSyncMigrator.Core.xml +++ b/docs/Reference/Generated/VstsSyncMigrator.Core.xml @@ -144,5 +144,13 @@ Beta Work Item + + + ExportUsersForMappingContext is a tool used to create a starter mapping file for users between the source and target systems. + Use `ExportUsersForMappingConfig` to configure. + + ready + Work Items + diff --git a/docs/Reference/v1/Processors/ExportUsersForMapping-notes.md b/docs/Reference/v1/Processors/ExportUsersForMapping-notes.md index 03d36757c..288d7a8b3 100644 --- a/docs/Reference/v1/Processors/ExportUsersForMapping-notes.md +++ b/docs/Reference/v1/Processors/ExportUsersForMapping-notes.md @@ -1,18 +1,83 @@ -## Additional Samples & Info +There was a request to have the ability to map users to try and maintain integrity across different systems. We added a `TfsUserMappingEnricher` that allows you to map users from Source to Target. + +##How it works + +1. Run `ExportUsersForMappingConfig` which will export all of the Users in Source Mapped or not to target. +2. Run `WorkItemMigrationConfig` which will run a validator by detail to warn you of missing users. If it finds a mapping it will convert the field... + +## ExportUsersForMappingConfig + +Running the `ExportUsersForMappingConfig` to get the list of users will produce something like: + +``` +[ + { + "Source": { + "FriendlyName": "Martin Hinshelwood nkdAgility.com", + "AccountName": "martin@nkdagility.com" + }, + "target": { + "FriendlyName": "Hinshelwood, Martin", + "AccountName": "martin@othercompany.com" + } + }, + { + "Source": { + "FriendlyName": "Rollup Bot", + "AccountName": "Bot@nkdagility.com" + }, + "target": { + "FriendlyName": "Service Account 4", + "AccountName": "randoaccount@somecompany.com" + } + }, + { + "Source": { + "FriendlyName": "Another non mapped Account", + "AccountName": "not-mapped@nkdagility.com" + }, + "target": null + } +] +``` + +Any `null` in the target field means that the user is not mapped. You can then use this to create a mapping file will all of your users. + +IMPORTANT: The Friendly name in Azure DevOps / TFS is not nessesarily the AAD Friendly name as users can change this in the tool. We load all of the users from both systems, and match on "email" to ensure we only assume mapping for the same user. Non mapped users, or users listed as null, will not be mapped. + +### Notes + +- On `ExportUsersForMappingConfig` you can set `OnlyListUsersInWorkItems` to filter the mapping based on the scope of the query. This is greater if you have many users. +- Configured using the `TfsUserMappingEnricherOptions` setting in `CommonEnrichersConfig` + +## WorkItemMigrationConfig + +When you run the `WorkItemMigrationContext` + ``` -{ - "$type": "ExportUsersForMappingConfig", - "Enabled": false, - "LocalExportJsonFile": "c:\\temp\\ExportUsersForMappingConfig.json", - "WIQLQuery": "SELECT [System.Id], [System.Tags] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan') ORDER BY [System.ChangedDate] desc", - "IdentityFieldsToCheck": [ - "System.AssignedTo", - "System.ChangedBy", - "System.CreatedBy", - "Microsoft.VSTS.Common.ActivatedBy", - "Microsoft.VSTS.Common.ResolvedBy", - "Microsoft.VSTS.Common.ClosedBy" - ] -} -``` \ No newline at end of file +... + "LogLevel": "Debug", + "CommonEnrichersConfig": [ + { + "$type": "TfsUserMappingEnricherOptions", + "Enabled": true, + "UserMappingFile": "C:\\temp\\userExport.json", + "IdentityFieldsToCheck": [ + "System.AssignedTo", + "System.ChangedBy", + "System.CreatedBy", + "Microsoft.VSTS.Common.ActivatedBy", + "Microsoft.VSTS.Common.ResolvedBy", + "Microsoft.VSTS.Common.ClosedBy" + ] + } + ], +... +``` + + +### Notes + +- Configured using the `TfsUserMappingEnricherOptions` setting in `CommonEnrichersConfig` +- Applies to all identity fields specified in the list \ No newline at end of file diff --git a/docs/Reference/v1/Processors/WorkItemMigrationContext-notes.md b/docs/Reference/v1/Processors/WorkItemMigrationContext-notes.md index ad010b404..91cf3be26 100644 --- a/docs/Reference/v1/Processors/WorkItemMigrationContext-notes.md +++ b/docs/Reference/v1/Processors/WorkItemMigrationContext-notes.md @@ -41,11 +41,19 @@ Scope to Area Path (Team data): ## NodeBasePath Configuration +<<<<<<< HEAD +Moved to the ProcessorEnricher [TfsNodeStructure](/Reference/v2/ProcessorEnrichers/TfsNodeStructure/) + +# Iteration Maps and Area Maps + +Moved to the ProcessorEnricher [TfsNodeStructure](/Reference/v2/ProcessorEnrichers/TfsNodeStructure/) +======= Moved to the ProcessorEnricher [TfsNodeStructure](../Reference/v2/ProcessorEnrichers/TfsNodeStructure/) # Iteration Maps and Area Maps Moved to the ProcessorEnricher [TfsNodeStructure](../Reference/v2/ProcessorEnrichers/TfsNodeStructure/) +>>>>>>> origin/master @@ -78,4 +86,8 @@ A complete list of [FieldMaps](../Reference/v1/FieldMaps/index.md) are available # Removed Properties -- PrefixProjectToNodes - This option was removed in favour of the Area and Iteration Maps on [TfsNodeStructure](../Reference/v2/ProcessorEnrichers/TfsNodeStructure/) \ No newline at end of file +<<<<<<< HEAD +- PrefixProjectToNodes - This option was removed in favour of the Area and Iteration Maps on [TfsNodeStructure](/Reference/v2/ProcessorEnrichers/TfsNodeStructure/) +======= +- PrefixProjectToNodes - This option was removed in favour of the Area and Iteration Maps on [TfsNodeStructure](../Reference/v2/ProcessorEnrichers/TfsNodeStructure/) +>>>>>>> origin/master diff --git a/docs/Reference/v2/ProcessorEnrichers/TfsNodeStructure-notes.md b/docs/Reference/v2/ProcessorEnrichers/TfsNodeStructure-notes.md index 871848ab0..a689e52bd 100644 --- a/docs/Reference/v2/ProcessorEnrichers/TfsNodeStructure-notes.md +++ b/docs/Reference/v2/ProcessorEnrichers/TfsNodeStructure-notes.md @@ -7,7 +7,11 @@ You have two options to solve this problem: +<<<<<<< HEAD +1. You can manually create the mentioned work items. This is a good option if you have a small number of work items or a small number of missing nodes. This will not work if you have work items that were moved from one project to another. Those Nodes are impossible to create in the target project. +======= 1. You can manualy create the mentioned work items. This is a good option if you have a small number of work items or a small number of missing nodes. This will not work if you have work items that were moved from one project to another. Those Nodes are imposible to create in the target project. +>>>>>>> origin/master 1. You can use the `AreaMaps` and `IterationMaps` to remap the nodes to existing nodes in the target project. This is a good option if you have a large number of work items or a large number of missing nodes. ### Overview @@ -109,6 +113,23 @@ in the replacement string. `TargetProject\NewArea\ValidArea\` but `OriginalProject\DescopeThis` would not be modified by this rule. +<<<<<<< HEAD +### PrefixProjectToNodes + +The `PrefixProjectToNodes` was an option that was used to prepend the source project name to the target set of nodes. This was super valuable when the target Project already has nodes and you dont want to merge them all together. This is now replaced by the `AreaMaps` and `IterationMaps` options. + +``` +"IterationMaps": { + "^SourceServer\\\\(.*)" , "TargetServer\\SourceServer\\$1", +}, +"AreaMaps": { + "^SourceServer\\\\(.*)" , "TargetServer\\SourceServer\\$1", +} +``` + + +======= +>>>>>>> origin/master ### More Complex Regex Before your migration starts it will validate that all of the Areas and Iterations from the **Source** work items revisions exist on the **Target**. Any that do not exist will be flagged in the logs and if and the migration will stop just after it outputs a list of the missing nodes. diff --git a/docs/_data/reference.v1.processors.exportusersformappingcontext.yaml b/docs/_data/reference.v1.processors.exportusersformappingcontext.yaml new file mode 100644 index 000000000..3512037fb --- /dev/null +++ b/docs/_data/reference.v1.processors.exportusersformappingcontext.yaml @@ -0,0 +1,34 @@ +optionsClassName: ExportUsersForMappingConfig +optionsClassFullName: MigrationTools._EngineV1.Configuration.Processing.ExportUsersForMappingConfig +configurationSamples: +- name: default + description: + code: >- + { + "$type": "ExportUsersForMappingConfig", + "Enabled": false, + "WIQLQuery": null, + "OnlyListUsersInWorkItems": true + } + sampleFor: MigrationTools._EngineV1.Configuration.Processing.ExportUsersForMappingConfig +description: ExportUsersForMappingContext is a tool used to create a starter mapping file for users between the source and target systems. Use `ExportUsersForMappingConfig` to configure. +className: ExportUsersForMappingContext +typeName: Processors +architecture: v1 +options: +- parameterName: Enabled + type: Boolean + description: missng XML code comments + defaultValue: missng XML code comments +- parameterName: OnlyListUsersInWorkItems + type: Boolean + description: missng XML code comments + defaultValue: missng XML code comments +- parameterName: WIQLQuery + type: String + description: missng XML code comments + defaultValue: missng XML code comments +status: ready +processingTarget: Work Items +classFile: '' +optionsClassFile: /src/MigrationTools/_EngineV1/Configuration/Processing/ExportUsersForMappingConfig.cs diff --git a/docs/_data/reference.v1.processors.workitemmigrationcontext.yaml b/docs/_data/reference.v1.processors.workitemmigrationcontext.yaml index 2697ce57b..c343da698 100644 --- a/docs/_data/reference.v1.processors.workitemmigrationcontext.yaml +++ b/docs/_data/reference.v1.processors.workitemmigrationcontext.yaml @@ -10,14 +10,11 @@ configurationSamples: "UpdateCreatedDate": true, "UpdateCreatedBy": true, "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc", - "AttachmentMigration": true, - "AttachmentWorkingPath": "c:\\temp\\WorkItemAttachmentWorkingFolder\\", "FixHtmlAttachmentLinks": false, "SkipToFinalRevisedWorkItemType": false, "WorkItemCreateRetryLimit": 5, "FilterWorkItemsThatAlreadyExistInTarget": false, "PauseAfterEachWorkItem": false, - "AttachmentMaxSize": 480000000, "AttachRevisionHistory": false, "LinkMigrationSaveEachAsAdded": false, "GenerateMigrationComment": true, @@ -32,18 +29,6 @@ className: WorkItemMigrationContext typeName: Processors architecture: v1 options: -- parameterName: AttachmentMaxSize - type: Int32 - description: '`AttachmentMigration` is set to true then you need to specify a max file size for upload in bites. For Azure DevOps Services the default is 480,000,000 bites (60mb), for TFS its 32,000,000 bites (4mb).' - defaultValue: 480000000 -- parameterName: AttachmentMigration - type: Boolean - description: If enabled this will migrate all of the attachments at the same time as the work item - defaultValue: true -- parameterName: AttachmentWorkingPath - type: String - description: '`AttachmentMigration` is set to true then you need to specify a working path for attachments to be saved locally.' - defaultValue: C:\temp\Migration\ - parameterName: AttachRevisionHistory type: Boolean description: This will create a json file with the revision history and attach it to the work item. Best used with `MaxRevisions` or `ReplayRevisions`. diff --git a/docs/_data/reference.v2.processorenrichers.appendmigrationtoolsignaturefooter.yaml b/docs/_data/reference.v2.processorenrichers.appendmigrationtoolsignaturefooter.yaml index e0eafe56e..edc21c183 100644 --- a/docs/_data/reference.v2.processorenrichers.appendmigrationtoolsignaturefooter.yaml +++ b/docs/_data/reference.v2.processorenrichers.appendmigrationtoolsignaturefooter.yaml @@ -16,8 +16,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: RefName type: String description: For internal use diff --git a/docs/_data/reference.v2.processorenrichers.filterworkitemsthatalreadyexistintarget.yaml b/docs/_data/reference.v2.processorenrichers.filterworkitemsthatalreadyexistintarget.yaml index aeff62c9a..0b0dc58c5 100644 --- a/docs/_data/reference.v2.processorenrichers.filterworkitemsthatalreadyexistintarget.yaml +++ b/docs/_data/reference.v2.processorenrichers.filterworkitemsthatalreadyexistintarget.yaml @@ -21,8 +21,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: Query type: QueryOptions description: missng XML code comments diff --git a/docs/_data/reference.v2.processorenrichers.pauseaftereachitem.yaml b/docs/_data/reference.v2.processorenrichers.pauseaftereachitem.yaml index 4528f9a16..ebcf8781f 100644 --- a/docs/_data/reference.v2.processorenrichers.pauseaftereachitem.yaml +++ b/docs/_data/reference.v2.processorenrichers.pauseaftereachitem.yaml @@ -16,8 +16,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: RefName type: String description: For internal use diff --git a/docs/_data/reference.v2.processorenrichers.skiptofinalrevisedworkitemtype.yaml b/docs/_data/reference.v2.processorenrichers.skiptofinalrevisedworkitemtype.yaml index d11ded4d3..07e3ee1fd 100644 --- a/docs/_data/reference.v2.processorenrichers.skiptofinalrevisedworkitemtype.yaml +++ b/docs/_data/reference.v2.processorenrichers.skiptofinalrevisedworkitemtype.yaml @@ -16,8 +16,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: RefName type: String description: For internal use diff --git a/docs/_data/reference.v2.processorenrichers.stringmanipulatorenricher.yaml b/docs/_data/reference.v2.processorenrichers.stringmanipulatorenricher.yaml index 51f99a140..14a49054c 100644 --- a/docs/_data/reference.v2.processorenrichers.stringmanipulatorenricher.yaml +++ b/docs/_data/reference.v2.processorenrichers.stringmanipulatorenricher.yaml @@ -26,8 +26,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: Manipulators type: List description: List of regex based string manipulations to apply to all string fields. Each regex replacement is applied in order and can be enabled or disabled. diff --git a/docs/_data/reference.v2.processorenrichers.tfsattachmentenricher.yaml b/docs/_data/reference.v2.processorenrichers.tfsattachmentenricher.yaml new file mode 100644 index 000000000..21e6594aa --- /dev/null +++ b/docs/_data/reference.v2.processorenrichers.tfsattachmentenricher.yaml @@ -0,0 +1,38 @@ +optionsClassName: TfsAttachmentEnricherOptions +optionsClassFullName: MigrationTools.Enrichers.TfsAttachmentEnricherOptions +configurationSamples: +- name: default + description: + code: >- + { + "$type": "TfsAttachmentEnricherOptions", + "Enabled": true, + "ExportBasePath": "c:\\temp\\WorkItemAttachmentExport", + "MaxAttachmentSize": 480000000 + } + sampleFor: MigrationTools.Enrichers.TfsAttachmentEnricherOptions +description: missng XML code comments +className: TfsAttachmentEnricher +typeName: ProcessorEnrichers +architecture: v2 +options: +- parameterName: Enabled + type: Boolean + description: If enabled this will run this migrator + defaultValue: true +- parameterName: ExportBasePath + type: String + description: '`AttachmentMigration` is set to true then you need to specify a working path for attachments to be saved locally.' + defaultValue: C:\temp\Migration\ +- parameterName: MaxAttachmentSize + type: Int32 + description: '`AttachmentMigration` is set to true then you need to specify a max file size for upload in bites. For Azure DevOps Services the default is 480,000,000 bites (60mb), for TFS its 32,000,000 bites (4mb).' + defaultValue: 480000000 +- parameterName: RefName + type: String + description: For internal use + defaultValue: missng XML code comments +status: missng XML code comments +processingTarget: missng XML code comments +classFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricher.cs +optionsClassFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricherOptions.cs diff --git a/docs/_data/reference.v2.processorenrichers.tfsnodestructure.yaml b/docs/_data/reference.v2.processorenrichers.tfsnodestructure.yaml index 478d75f4f..64b4b8afc 100644 --- a/docs/_data/reference.v2.processorenrichers.tfsnodestructure.yaml +++ b/docs/_data/reference.v2.processorenrichers.tfsnodestructure.yaml @@ -29,8 +29,8 @@ options: defaultValue: '{}' - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: IterationMaps type: Dictionary description: Remapping rules for iteration paths, implemented with regular expressions. The rules apply with a higher priority than the `PrefixProjectToNodes`, that is, if no rule matches the path and the `PrefixProjectToNodes` option is enabled, then the old `PrefixProjectToNodes` behavior is applied. diff --git a/docs/_data/reference.v2.processorenrichers.tfsrevisionmanager.yaml b/docs/_data/reference.v2.processorenrichers.tfsrevisionmanager.yaml index 2a9e33561..fb88a5995 100644 --- a/docs/_data/reference.v2.processorenrichers.tfsrevisionmanager.yaml +++ b/docs/_data/reference.v2.processorenrichers.tfsrevisionmanager.yaml @@ -18,8 +18,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: MaxRevisions type: Int32 description: Sets the maximum number of revisions that will be migrated. "First + Last N = Max". If this was set to 5 and there were 10 revisions you would get the first 1 (creation) and the latest 4 migrated. diff --git a/docs/_data/reference.v2.processorenrichers.tfsusermappingenricher.yaml b/docs/_data/reference.v2.processorenrichers.tfsusermappingenricher.yaml index 40cbae0e0..beb245f4c 100644 --- a/docs/_data/reference.v2.processorenrichers.tfsusermappingenricher.yaml +++ b/docs/_data/reference.v2.processorenrichers.tfsusermappingenricher.yaml @@ -1,12 +1,45 @@ -optionsClassName: -optionsClassFullName: -configurationSamples: [] -description: missng XML code comments +optionsClassName: TfsUserMappingEnricherOptions +optionsClassFullName: MigrationTools.Enrichers.TfsUserMappingEnricherOptions +configurationSamples: +- name: default + description: + code: >- + { + "$type": "TfsUserMappingEnricherOptions", + "Enabled": false, + "IdentityFieldsToCheck": [ + "System.AssignedTo", + "System.ChangedBy", + "System.CreatedBy", + "Microsoft.VSTS.Common.ActivatedBy", + "Microsoft.VSTS.Common.ResolvedBy", + "Microsoft.VSTS.Common.ClosedBy" + ], + "UserMappingFile": "usermapping.json" + } + sampleFor: MigrationTools.Enrichers.TfsUserMappingEnricherOptions +description: The TfsUserMappingEnricher is used to map users from the source to the target system. Run it with the ExportUsersForMappingContext to create a mapping file then with WorkItemMigrationContext to use the mapping file to update the users in the target system as you migrate the work items. className: TfsUserMappingEnricher typeName: ProcessorEnrichers architecture: v2 -options: [] +options: +- parameterName: Enabled + type: Boolean + description: If enabled this will run this migrator + defaultValue: true +- parameterName: IdentityFieldsToCheck + type: List + description: This is a list of the Identiy fields in the Source to check for user mapping purposes. You should list all identiy fields that you wan to map. + defaultValue: missng XML code comments +- parameterName: RefName + type: String + description: For internal use + defaultValue: missng XML code comments +- parameterName: UserMappingFile + type: String + description: This is the file that will be used to export or import the user mappings. Use the ExportUsersForMapping processor to create the file. + defaultValue: missng XML code comments status: missng XML code comments processingTarget: missng XML code comments -classFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsUserMappingEnricher.cs -optionsClassFile: +classFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricher.cs +optionsClassFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricherOptions.cs diff --git a/docs/_data/reference.v2.processorenrichers.tfsvalidaterequiredfield.yaml b/docs/_data/reference.v2.processorenrichers.tfsvalidaterequiredfield.yaml index 0506ecf29..3bb99d4a5 100644 --- a/docs/_data/reference.v2.processorenrichers.tfsvalidaterequiredfield.yaml +++ b/docs/_data/reference.v2.processorenrichers.tfsvalidaterequiredfield.yaml @@ -16,8 +16,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: RefName type: String description: For internal use diff --git a/docs/_data/reference.v2.processorenrichers.tfsworkitemlinkenricher.yaml b/docs/_data/reference.v2.processorenrichers.tfsworkitemlinkenricher.yaml index e1f7514c5..69832c861 100644 --- a/docs/_data/reference.v2.processorenrichers.tfsworkitemlinkenricher.yaml +++ b/docs/_data/reference.v2.processorenrichers.tfsworkitemlinkenricher.yaml @@ -18,8 +18,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: FilterIfLinkCountMatches type: Boolean description: Skip validating links if the number of links in the source and the target matches! diff --git a/docs/_includes/sampleConfig/configuration.json b/docs/_includes/sampleConfig/configuration.json index 5c6e80d32..6a2f691dd 100644 --- a/docs/_includes/sampleConfig/configuration.json +++ b/docs/_includes/sampleConfig/configuration.json @@ -3,8 +3,8 @@ "Source": { "$type": "TfsTeamProjectConfig", "Collection": "https://dev.azure.com/nkdagility-preview/", - "Project": "myProjectName", - "ReflectedWorkItemIDFieldName": "Custom.ReflectedWorkItemId", + "Project": "migrationSource1", + "ReflectedWorkItemIDFieldName": "nkdScrum.ReflectedWorkItemId", "AllowCrossProjectLinking": false, "AuthenticationMode": "Prompt", "PersonalAccessToken": "", @@ -17,145 +17,89 @@ "Target": { "$type": "TfsTeamProjectConfig", "Collection": "https://dev.azure.com/nkdagility-preview/", - "Project": "myProjectName", - "ReflectedWorkItemIDFieldName": "Custom.ReflectedWorkItemId", + "Project": "migrationTest5", + "ReflectedWorkItemIDFieldName": "nkdScrum.ReflectedWorkItemId", "AllowCrossProjectLinking": false, "AuthenticationMode": "Prompt", - "PersonalAccessToken": "", + "PersonalAccessToken": "njp3kcec4nbev63fmbepvdpn35drawmonk5qf5yqsw77dgfwnjda", "PersonalAccessTokenVariableName": "", "LanguageMaps": { "AreaPath": "Area", "IterationPath": "Iteration" } }, - "FieldMaps": [ + "FieldMaps": [], + "GitRepoMapping": null, + "LogLevel": "Debug", + "CommonEnrichersConfig": [ { - "$type": "MultiValueConditionalMapConfig", - "WorkItemTypeName": "*", - "sourceFieldsAndValues": { - "Field1": "Value1", - "Field2": "Value2" + "$type": "TfsNodeStructureOptions", + "NodeBasePaths": [], + "AreaMaps": { + "^Skypoint Cloud$": "MigrationTest5" }, - "targetFieldsAndValues": { - "Field1": "Value1", - "Field2": "Value2" - } - }, - { - "$type": "FieldBlankMapConfig", - "WorkItemTypeName": "*", - "targetField": "TfsMigrationTool.ReflectedWorkItemId" - }, - { - "$type": "FieldValueMapConfig", - "WorkItemTypeName": "*", - "sourceField": "System.State", - "targetField": "System.State", - "defaultValue": "New", - "valueMapping": { - "Approved": "New", - "New": "New", - "Committed": "Active", - "In Progress": "Active", - "To Do": "New", - "Done": "Closed", - "Removed": "Removed" - } - }, - { - "$type": "FieldtoFieldMapConfig", - "WorkItemTypeName": "*", - "sourceField": "Microsoft.VSTS.Common.BacklogPriority", - "targetField": "Microsoft.VSTS.Common.StackRank", - "defaultValue": null - }, - { - "$type": "FieldtoFieldMultiMapConfig", - "WorkItemTypeName": "*", - "SourceToTargetMappings": { - "SourceField1": "TargetField1", - "SourceField2": "TargetField2" - } - }, - { - "$type": "FieldtoTagMapConfig", - "WorkItemTypeName": "*", - "sourceField": "System.State", - "formatExpression": "ScrumState:{0}" + "IterationMaps": { + "^Skypoint Cloud\\\\Sprint 1$": "MigrationTest5\\Sprint 1" + }, + "ShouldCreateMissingRevisionPaths": true, + "ReplicateAllExistingNodes": true }, { - "$type": "FieldMergeMapConfig", - "WorkItemTypeName": "*", - "sourceFields": [ - "System.Description", - "Microsoft.VSTS.Common.AcceptanceCriteria" - ], - "targetField": "System.Description", - "formatExpression": "{0}

Acceptance Criteria

{1}", - "doneMatch": "##DONE##" + "$type": "TfsWorkItemLinkEnricherOptions", + "Enabled": true, + "FilterIfLinkCountMatches": true, + "SaveAfterEachLinkIsAdded": false }, { - "$type": "RegexFieldMapConfig", - "WorkItemTypeName": "*", - "sourceField": "COMPANY.PRODUCT.Release", - "targetField": "COMPANY.DEVISION.MinorReleaseVersion", - "pattern": "PRODUCT \\d{4}.(\\d{1})", - "replacement": "$1" + "$type": "TfsRevisionManagerOptions", + "Enabled": true, + "ReplayRevisions": true, + "MaxRevisions": 0 }, { - "$type": "FieldValuetoTagMapConfig", - "WorkItemTypeName": "*", - "sourceField": "Microsoft.VSTS.CMMI.Blocked", - "pattern": "Yes", - "formatExpression": "{0}" + "$type": "TfsAttachmentEnricherOptions", + "Enabled": true, + "ExportBasePath": "c:\\temp\\WorkItemAttachmentExport", + "MaxAttachmentSize": 480000000 }, { - "$type": "TreeToTagMapConfig", - "WorkItemTypeName": "*", - "toSkip": 3, - "timeTravel": 1 + "$type": "StringManipulatorEnricherOptions", + "Enabled": true, + "MaxStringLength": 1000000, + "Manipulators": [ + { + "$type": "RegexStringManipulator", + "Enabled": true, + "Pattern": "[^( -~)\n\r\t]+", + "Replacement": "", + "Description": "Remove invalid characters from the end of the string" + } + ] } ], - "GitRepoMapping": null, - "LogLevel": "Information", - "CommonEnrichersConfig": null, "Processors": [ { "$type": "WorkItemMigrationConfig", - "Enabled": false, - "ReplayRevisions": true, - "PrefixProjectToNodes": false, + "Enabled": true, "UpdateCreatedDate": true, "UpdateCreatedBy": true, - "WIQLQueryBit": "AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request')", - "WIQLOrderBit": "[System.ChangedDate] desc", + "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc", "LinkMigration": true, - "AttachmentMigration": true, - "AttachmentWorkingPath": "c:\\temp\\WorkItemAttachmentWorkingFolder\\", "FixHtmlAttachmentLinks": false, - "SkipToFinalRevisedWorkItemType": true, + "SkipToFinalRevisedWorkItemType": false, "WorkItemCreateRetryLimit": 5, - "FilterWorkItemsThatAlreadyExistInTarget": true, + "FilterWorkItemsThatAlreadyExistInTarget": false, "PauseAfterEachWorkItem": false, - "AttachmentMaxSize": 480000000, "AttachRevisionHistory": false, "LinkMigrationSaveEachAsAdded": false, "GenerateMigrationComment": true, "WorkItemIDs": null, - "MaxRevisions": 0, - "NodeStructureEnricherEnabled": null, - "UseCommonNodeStructureEnricherConfig": false, - "NodeBasePaths": [ - "Product\\Area\\Path1", - "Product\\Area\\Path2" - ], - "AreaMaps": {}, - "IterationMaps": {}, "MaxGracefulFailures": 0, - "SkipRevisionWithInvalidIterationPath": false + "SkipRevisionWithInvalidIterationPath": false, + "SkipRevisionWithInvalidAreaPath": false } ], - "Version": "0.0", + "Version": "15.0", "workaroundForQuerySOAPBugEnabled": false, "WorkItemTypeDefinition": { "sourceWorkItemTypeName": "targetWorkItemTypeName" diff --git a/docs/collections/_reference/reference.v1.processors.exportusersformappingcontext.md b/docs/collections/_reference/reference.v1.processors.exportusersformappingcontext.md new file mode 100644 index 000000000..90c3699b9 --- /dev/null +++ b/docs/collections/_reference/reference.v1.processors.exportusersformappingcontext.md @@ -0,0 +1,55 @@ +--- +optionsClassName: ExportUsersForMappingConfig +optionsClassFullName: MigrationTools._EngineV1.Configuration.Processing.ExportUsersForMappingConfig +configurationSamples: +- name: default + description: + code: >- + { + "$type": "ExportUsersForMappingConfig", + "Enabled": false, + "WIQLQuery": null, + "OnlyListUsersInWorkItems": true + } + sampleFor: MigrationTools._EngineV1.Configuration.Processing.ExportUsersForMappingConfig +description: ExportUsersForMappingContext is a tool used to create a starter mapping file for users between the source and target systems. Use `ExportUsersForMappingConfig` to configure. +className: ExportUsersForMappingContext +typeName: Processors +architecture: v1 +options: +- parameterName: Enabled + type: Boolean + description: missng XML code comments + defaultValue: missng XML code comments +- parameterName: OnlyListUsersInWorkItems + type: Boolean + description: missng XML code comments + defaultValue: missng XML code comments +- parameterName: WIQLQuery + type: String + description: missng XML code comments + defaultValue: missng XML code comments +status: ready +processingTarget: Work Items +classFile: '' +optionsClassFile: /src/MigrationTools/_EngineV1/Configuration/Processing/ExportUsersForMappingConfig.cs + +redirectFrom: [] +layout: reference +toc: true +permalink: /Reference/v1/Processors/ExportUsersForMappingContext/ +title: ExportUsersForMappingContext +categories: +- Processors +- v1 +topics: +- topic: notes + path: /docs/Reference/v1/Processors/ExportUsersForMappingContext-notes.md + exists: false + markdown: '' +- topic: introduction + path: /docs/Reference/v1/Processors/ExportUsersForMappingContext-introduction.md + exists: false + markdown: '' + +--- \ No newline at end of file diff --git a/docs/collections/_reference/reference.v1.processors.workitemmigrationcontext.md b/docs/collections/_reference/reference.v1.processors.workitemmigrationcontext.md index f22dc098f..291fa6683 100644 --- a/docs/collections/_reference/reference.v1.processors.workitemmigrationcontext.md +++ b/docs/collections/_reference/reference.v1.processors.workitemmigrationcontext.md @@ -11,14 +11,11 @@ configurationSamples: "UpdateCreatedDate": true, "UpdateCreatedBy": true, "WIQLQuery": "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan','Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc", - "AttachmentMigration": true, - "AttachmentWorkingPath": "c:\\temp\\WorkItemAttachmentWorkingFolder\\", "FixHtmlAttachmentLinks": false, "SkipToFinalRevisedWorkItemType": false, "WorkItemCreateRetryLimit": 5, "FilterWorkItemsThatAlreadyExistInTarget": false, "PauseAfterEachWorkItem": false, - "AttachmentMaxSize": 480000000, "AttachRevisionHistory": false, "LinkMigrationSaveEachAsAdded": false, "GenerateMigrationComment": true, @@ -33,18 +30,6 @@ className: WorkItemMigrationContext typeName: Processors architecture: v1 options: -- parameterName: AttachmentMaxSize - type: Int32 - description: '`AttachmentMigration` is set to true then you need to specify a max file size for upload in bites. For Azure DevOps Services the default is 480,000,000 bites (60mb), for TFS its 32,000,000 bites (4mb).' - defaultValue: 480000000 -- parameterName: AttachmentMigration - type: Boolean - description: If enabled this will migrate all of the attachments at the same time as the work item - defaultValue: true -- parameterName: AttachmentWorkingPath - type: String - description: '`AttachmentMigration` is set to true then you need to specify a working path for attachments to be saved locally.' - defaultValue: C:\temp\Migration\ - parameterName: AttachRevisionHistory type: Boolean description: This will create a json file with the revision history and attach it to the work item. Best used with `MaxRevisions` or `ReplayRevisions`. @@ -126,7 +111,7 @@ topics: - topic: notes path: /docs/Reference/v1/Processors/WorkItemMigrationContext-notes.md exists: true - markdown: >- + markdown: >+ ## WIQL Query Bits @@ -193,6 +178,18 @@ topics: ## NodeBasePath Configuration + <<<<<<< HEAD + + Moved to the ProcessorEnricher [TfsNodeStructure](/Reference/v2/ProcessorEnrichers/TfsNodeStructure/) + + + # Iteration Maps and Area Maps + + + Moved to the ProcessorEnricher [TfsNodeStructure](/Reference/v2/ProcessorEnrichers/TfsNodeStructure/) + + ======= + Moved to the ProcessorEnricher [TfsNodeStructure](../Reference/v2/ProcessorEnrichers/TfsNodeStructure/) @@ -201,6 +198,8 @@ topics: Moved to the ProcessorEnricher [TfsNodeStructure](../Reference/v2/ProcessorEnrichers/TfsNodeStructure/) + >>>>>>> origin/master + @@ -240,7 +239,15 @@ topics: # Removed Properties + <<<<<<< HEAD + + - PrefixProjectToNodes - This option was removed in favour of the Area and Iteration Maps on [TfsNodeStructure](/Reference/v2/ProcessorEnrichers/TfsNodeStructure/) + + ======= + - PrefixProjectToNodes - This option was removed in favour of the Area and Iteration Maps on [TfsNodeStructure](../Reference/v2/ProcessorEnrichers/TfsNodeStructure/) + + >>>>>>> origin/master - topic: introduction path: /docs/Reference/v1/Processors/WorkItemMigrationContext-introduction.md exists: true diff --git a/docs/collections/_reference/reference.v2.processorenrichers.appendmigrationtoolsignaturefooter.md b/docs/collections/_reference/reference.v2.processorenrichers.appendmigrationtoolsignaturefooter.md index 37f854aca..2ca13bfc6 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.appendmigrationtoolsignaturefooter.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.appendmigrationtoolsignaturefooter.md @@ -17,8 +17,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: RefName type: String description: For internal use diff --git a/docs/collections/_reference/reference.v2.processorenrichers.filterworkitemsthatalreadyexistintarget.md b/docs/collections/_reference/reference.v2.processorenrichers.filterworkitemsthatalreadyexistintarget.md index 35539777d..6c0207513 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.filterworkitemsthatalreadyexistintarget.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.filterworkitemsthatalreadyexistintarget.md @@ -22,8 +22,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: Query type: QueryOptions description: missng XML code comments diff --git a/docs/collections/_reference/reference.v2.processorenrichers.pauseaftereachitem.md b/docs/collections/_reference/reference.v2.processorenrichers.pauseaftereachitem.md index 8c24ffc86..ad0a8d708 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.pauseaftereachitem.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.pauseaftereachitem.md @@ -17,8 +17,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: RefName type: String description: For internal use diff --git a/docs/collections/_reference/reference.v2.processorenrichers.skiptofinalrevisedworkitemtype.md b/docs/collections/_reference/reference.v2.processorenrichers.skiptofinalrevisedworkitemtype.md index 7de1ea639..617c8de68 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.skiptofinalrevisedworkitemtype.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.skiptofinalrevisedworkitemtype.md @@ -17,8 +17,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: RefName type: String description: For internal use diff --git a/docs/collections/_reference/reference.v2.processorenrichers.stringmanipulatorenricher.md b/docs/collections/_reference/reference.v2.processorenrichers.stringmanipulatorenricher.md index c3897b3ce..2f48fdaf8 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.stringmanipulatorenricher.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.stringmanipulatorenricher.md @@ -27,8 +27,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: Manipulators type: List description: List of regex based string manipulations to apply to all string fields. Each regex replacement is applied in order and can be enabled or disabled. diff --git a/docs/collections/_reference/reference.v2.processorenrichers.tfsattachmentenricher.md b/docs/collections/_reference/reference.v2.processorenrichers.tfsattachmentenricher.md new file mode 100644 index 000000000..c2759b6ce --- /dev/null +++ b/docs/collections/_reference/reference.v2.processorenrichers.tfsattachmentenricher.md @@ -0,0 +1,59 @@ +--- +optionsClassName: TfsAttachmentEnricherOptions +optionsClassFullName: MigrationTools.Enrichers.TfsAttachmentEnricherOptions +configurationSamples: +- name: default + description: + code: >- + { + "$type": "TfsAttachmentEnricherOptions", + "Enabled": true, + "ExportBasePath": "c:\\temp\\WorkItemAttachmentExport", + "MaxAttachmentSize": 480000000 + } + sampleFor: MigrationTools.Enrichers.TfsAttachmentEnricherOptions +description: missng XML code comments +className: TfsAttachmentEnricher +typeName: ProcessorEnrichers +architecture: v2 +options: +- parameterName: Enabled + type: Boolean + description: If enabled this will run this migrator + defaultValue: true +- parameterName: ExportBasePath + type: String + description: '`AttachmentMigration` is set to true then you need to specify a working path for attachments to be saved locally.' + defaultValue: C:\temp\Migration\ +- parameterName: MaxAttachmentSize + type: Int32 + description: '`AttachmentMigration` is set to true then you need to specify a max file size for upload in bites. For Azure DevOps Services the default is 480,000,000 bites (60mb), for TFS its 32,000,000 bites (4mb).' + defaultValue: 480000000 +- parameterName: RefName + type: String + description: For internal use + defaultValue: missng XML code comments +status: missng XML code comments +processingTarget: missng XML code comments +classFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricher.cs +optionsClassFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricherOptions.cs + +redirectFrom: [] +layout: reference +toc: true +permalink: /Reference/v2/ProcessorEnrichers/TfsAttachmentEnricher/ +title: TfsAttachmentEnricher +categories: +- ProcessorEnrichers +- v2 +topics: +- topic: notes + path: /docs/Reference/v2/ProcessorEnrichers/TfsAttachmentEnricher-notes.md + exists: false + markdown: '' +- topic: introduction + path: /docs/Reference/v2/ProcessorEnrichers/TfsAttachmentEnricher-introduction.md + exists: false + markdown: '' + +--- \ No newline at end of file diff --git a/docs/collections/_reference/reference.v2.processorenrichers.tfsnodestructure.md b/docs/collections/_reference/reference.v2.processorenrichers.tfsnodestructure.md index a9f897528..7434c6b23 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.tfsnodestructure.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.tfsnodestructure.md @@ -30,8 +30,8 @@ options: defaultValue: '{}' - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: IterationMaps type: Dictionary description: Remapping rules for iteration paths, implemented with regular expressions. The rules apply with a higher priority than the `PrefixProjectToNodes`, that is, if no rule matches the path and the `PrefixProjectToNodes` option is enabled, then the old `PrefixProjectToNodes` behavior is applied. @@ -69,73 +69,33 @@ topics: - topic: notes path: /docs/Reference/v2/ProcessorEnrichers/TfsNodeStructure-notes.md exists: true - markdown: >- - ## NodeBasePath Configuration ## - - The `NodeBasePaths` entry allows the filtering of the nodes to be replicated on the target projects. To try to explain the correct usage let us assume that we have a source team project `SourceProj` with the following node structures - - - - AreaPath - - SourceProj - - SourceProj\Team 1 - - SourceProj\Team 2 - - SourceProj\Team 2\Sub-Area - - SourceProj\Team 3 - - IterationPath - - SourceProj - - SourceProj\Sprint 1 - - SourceProj\Sprint 2 - - SourceProj\Sprint 2\Sub-Iteration - - SourceProj\Sprint 3 + markdown: >2- - Depending upon what node structures you wish to migrate you would need the following settings. Exclusions are also possible by prefixing a path with an exclamation mark `!`. Example are - | | | + ## Iteration Maps and Area Maps - |-|-| - | Intention | Migrate all areas and iterations and all Work Items + **NOTE: It is NOT posible to migrate a work item if the Area or Iteration path does not exist on the target project. This is because the work item will be created with the same Area and Iteration path as the source work item. If the path does not exist, the work item will not be created. _There is not way around this!_** - | NodeBasePath | `[]` - | Comment | The same AreaPath and Iteration Paths are created on the target as on the source. Hence, all migrated WI remain in their existing area and iteration paths + You have two options to solve this problem: - || - | Intention | Only migrate area path `Team 2` and it associated Work Items, but all iteration paths + <<<<<<< HEAD - | NodeBasePath | `["Team 2", "Sprint"]` + 1. You can manually create the mentioned work items. This is a good option if you have a small number of work items or a small number of missing nodes. This will not work if you have work items that were moved from one project to another. Those Nodes are impossible to create in the target project. - | Comment | Only the area path ending `Team 2` will be migrated.
The `WIQLQueryBit` should be edited to limit the WI migrated to this area path e.g. add `AND [System.AreaPath] UNDER 'SampleProject\\Team 2'` .
The migrated WI will have an area path of `TargetProj\Team 2` but retain their iteration paths matching the sprint name on the source + ======= - || + 1. You can manualy create the mentioned work items. This is a good option if you have a small number of work items or a small number of missing nodes. This will not work if you have work items that were moved from one project to another. Those Nodes are imposible to create in the target project. - | Intention | Only migrate iterations structure + >>>>>>> origin/master - | NodeBasePath | `["Sprint"]` + 1. You can use the `AreaMaps` and `IterationMaps` to remap the nodes to existing nodes in the target project. This is a good option if you have a large number of work items or a large number of missing nodes. - | Comment | Only the area path ending `Team 2` will be migrated
All the iteration paths will be migrated.
The migrated WI will have the default area path of `TargetProj` as their source area path was not migrated i.e. `TargetProj`
The migrated WI will have an iteration path match the sprint name on the source - || - - | Intention | Move all WI to the existing area and iteration paths on the targetProj - - | NodeBasePath | `["DUMMY VALUE"]` - - | Comment | As the `NodeBasePath` does not match any source area or iteration path no nodes are migrated.
Migrated WI will be assigned to any matching area or iteration paths. If no matching ones can be found they will default to the respective root values - - || - - | Intention | Move the `Team 2` area, but not its `Sub-Area` - - | NodeBasePath | `["Team 2", "!Team 2\\SubArea"]` - - | Comment | The Work Items will have to be restricted to the right areas, e.g. with `AND [System.AreaPath] UNDER 'SampleProject\\Team 2' AND [System.AreaPath] NOT UNDER 'SampleProject\\Team 2\\Sub-Area'`, otherwise their migratin will fail - - - - # Iteration Maps and Area Maps + ### Overview These two configuration elements apply after the `NodeBasePaths` selector, i.e. @@ -201,7 +161,8 @@ topics: in the replacement string. - #### Examples explained + + ### Configuration ```json @@ -267,19 +228,45 @@ topics: `TargetProject\NewArea\ValidArea\` but `OriginalProject\DescopeThis` would not be modified by this rule. + <<<<<<< HEAD + + ### PrefixProjectToNodes + + + The `PrefixProjectToNodes` was an option that was used to prepend the source project name to the target set of nodes. This was super valuable when the target Project already has nodes and you dont want to merge them all together. This is now replaced by the `AreaMaps` and `IterationMaps` options. + + + ``` + + "IterationMaps": { + "^SourceServer\\\\(.*)" , "TargetServer\\SourceServer\\$1", + }, + + "AreaMaps": { + "^SourceServer\\\\(.*)" , "TargetServer\\SourceServer\\$1", + } + + ``` + + + + ======= + + >>>>>>> origin/master + ### More Complex Regex Before your migration starts it will validate that all of the Areas and Iterations from the **Source** work items revisions exist on the **Target**. Any that do not exist will be flagged in the logs and if and the migration will stop just after it outputs a list of the missing nodes. - Our algorithm that converts the Source nodes to Target nodes processes the [mappings](https://nkdagility.com/learn/azure-devops-migration-tools/Reference/v1/Processors/WorkItemMigrationContext/#iteration-maps-and-area-maps) at that time. This means that any valid mapped nodes will never be caught by the `This path is not anchored in the source project` message as they are already altered to be valid. + Our algorithm that converts the Source nodes to Target nodes processes the mappings at that time. This means that any valid mapped nodes will never be caught by the `This path is not anchored in the source project` message as they are already altered to be valid. > We recently updated the logging for this part of the system to more easily debug both your mappings and to see what they system is doing with the nodes and their current state. You can set `"LogLevel": "Debug"` to see the details. - To add a mapping, you can follow [the documentation](https://nkdagility.com/learn/azure-devops-migration-tools/Reference/v1/Processors/WorkItemMigrationContext/#iteration-maps-and-area-maps) with this being the simplest way: + To add a mapping, you can follow the documentation with this being the simplest way: ``` @@ -381,6 +368,70 @@ topics: ], ``` + + + ## NodeBasePath Configuration + + The `NodeBasePaths` entry allows the filtering of the nodes to be replicated on the target projects. To try to explain the correct usage let us assume that we have a source team project `SourceProj` with the following node structures + + + - AreaPath + - SourceProj + - SourceProj\Team 1 + - SourceProj\Team 2 + - SourceProj\Team 2\Sub-Area + - SourceProj\Team 3 + - IterationPath + - SourceProj + - SourceProj\Sprint 1 + - SourceProj\Sprint 2 + - SourceProj\Sprint 2\Sub-Iteration + - SourceProj\Sprint 3 + + Depending upon what node structures you wish to migrate you would need the following settings. Exclusions are also possible by prefixing a path with an exclamation mark `!`. Example are + + + | | | + + |-|-| + + | Intention | Migrate all areas and iterations and all Work Items + + | NodeBasePath | `[]` + + | Comment | The same AreaPath and Iteration Paths are created on the target as on the source. Hence, all migrated WI remain in their existing area and iteration paths + + || + + | Intention | Only migrate area path `Team 2` and it associated Work Items, but all iteration paths + + | NodeBasePath | `["Team 2", "Sprint"]` + + | Comment | Only the area path ending `Team 2` will be migrated.
The `WIQLQueryBit` should be edited to limit the WI migrated to this area path e.g. add `AND [System.AreaPath] UNDER 'SampleProject\\Team 2'` .
The migrated WI will have an area path of `TargetProj\Team 2` but retain their iteration paths matching the sprint name on the source + + || + + | Intention | Only migrate iterations structure + + | NodeBasePath | `["Sprint"]` + + | Comment | Only the area path ending `Team 2` will be migrated
All the iteration paths will be migrated.
The migrated WI will have the default area path of `TargetProj` as their source area path was not migrated i.e. `TargetProj`
The migrated WI will have an iteration path match the sprint name on the source + + || + + | Intention | Move all WI to the existing area and iteration paths on the targetProj + + | NodeBasePath | `["DUMMY VALUE"]` + + | Comment | As the `NodeBasePath` does not match any source area or iteration path no nodes are migrated.
Migrated WI will be assigned to any matching area or iteration paths. If no matching ones can be found they will default to the respective root values + + || + + | Intention | Move the `Team 2` area, but not its `Sub-Area` + + | NodeBasePath | `["Team 2", "!Team 2\\SubArea"]` + + | Comment | The Work Items will have to be restricted to the right areas, e.g. with `AND [System.AreaPath] UNDER 'SampleProject\\Team 2' AND [System.AreaPath] NOT UNDER 'SampleProject\\Team 2\\Sub-Area'`, otherwise their migratin will fail - topic: introduction path: /docs/Reference/v2/ProcessorEnrichers/TfsNodeStructure-introduction.md exists: true diff --git a/docs/collections/_reference/reference.v2.processorenrichers.tfsrevisionmanager.md b/docs/collections/_reference/reference.v2.processorenrichers.tfsrevisionmanager.md index 962292fb8..5fc782201 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.tfsrevisionmanager.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.tfsrevisionmanager.md @@ -19,8 +19,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: MaxRevisions type: Int32 description: Sets the maximum number of revisions that will be migrated. "First + Last N = Max". If this was set to 5 and there were 10 revisions you would get the first 1 (creation) and the latest 4 migrated. diff --git a/docs/collections/_reference/reference.v2.processorenrichers.tfsusermappingenricher.md b/docs/collections/_reference/reference.v2.processorenrichers.tfsusermappingenricher.md index e9107d675..e1c36df86 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.tfsusermappingenricher.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.tfsusermappingenricher.md @@ -1,16 +1,49 @@ --- -optionsClassName: -optionsClassFullName: -configurationSamples: [] -description: missng XML code comments +optionsClassName: TfsUserMappingEnricherOptions +optionsClassFullName: MigrationTools.Enrichers.TfsUserMappingEnricherOptions +configurationSamples: +- name: default + description: + code: >- + { + "$type": "TfsUserMappingEnricherOptions", + "Enabled": false, + "IdentityFieldsToCheck": [ + "System.AssignedTo", + "System.ChangedBy", + "System.CreatedBy", + "Microsoft.VSTS.Common.ActivatedBy", + "Microsoft.VSTS.Common.ResolvedBy", + "Microsoft.VSTS.Common.ClosedBy" + ], + "UserMappingFile": "usermapping.json" + } + sampleFor: MigrationTools.Enrichers.TfsUserMappingEnricherOptions +description: The TfsUserMappingEnricher is used to map users from the source to the target system. Run it with the ExportUsersForMappingContext to create a mapping file then with WorkItemMigrationContext to use the mapping file to update the users in the target system as you migrate the work items. className: TfsUserMappingEnricher typeName: ProcessorEnrichers architecture: v2 -options: [] +options: +- parameterName: Enabled + type: Boolean + description: If enabled this will run this migrator + defaultValue: true +- parameterName: IdentityFieldsToCheck + type: List + description: This is a list of the Identiy fields in the Source to check for user mapping purposes. You should list all identiy fields that you wan to map. + defaultValue: missng XML code comments +- parameterName: RefName + type: String + description: For internal use + defaultValue: missng XML code comments +- parameterName: UserMappingFile + type: String + description: This is the file that will be used to export or import the user mappings. Use the ExportUsersForMapping processor to create the file. + defaultValue: missng XML code comments status: missng XML code comments processingTarget: missng XML code comments -classFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsUserMappingEnricher.cs -optionsClassFile: +classFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricher.cs +optionsClassFile: /src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricherOptions.cs redirectFrom: [] layout: reference diff --git a/docs/collections/_reference/reference.v2.processorenrichers.tfsvalidaterequiredfield.md b/docs/collections/_reference/reference.v2.processorenrichers.tfsvalidaterequiredfield.md index 93ec62034..0f8b2a64e 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.tfsvalidaterequiredfield.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.tfsvalidaterequiredfield.md @@ -17,8 +17,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: RefName type: String description: For internal use diff --git a/docs/collections/_reference/reference.v2.processorenrichers.tfsworkitemlinkenricher.md b/docs/collections/_reference/reference.v2.processorenrichers.tfsworkitemlinkenricher.md index c520382ab..4f3d2c366 100644 --- a/docs/collections/_reference/reference.v2.processorenrichers.tfsworkitemlinkenricher.md +++ b/docs/collections/_reference/reference.v2.processorenrichers.tfsworkitemlinkenricher.md @@ -19,8 +19,8 @@ architecture: v2 options: - parameterName: Enabled type: Boolean - description: For internal use - defaultValue: missng XML code comments + description: If enabled this will run this migrator + defaultValue: true - parameterName: FilterIfLinkCountMatches type: Boolean description: Skip validating links if the number of links in the source and the target matches! diff --git a/docs/getting-started.md b/docs/getting-started.md index 0521ab633..015f5b1d7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -18,9 +18,11 @@ Watch the [Video Overview](https://youtu.be/RCJsST0xBCE) to get started in 30 mi In order to run the migration you will need to install the tools first. 1. Install [winget](https://learn.microsoft.com/en-us/windows/package-manager/winget/) -1. Run `winget install nkdAgility.AzureDevOpsMigrationTools` from the [Windows Terminal](https://learn.microsoft.com/en-us/windows/terminal/) to install on Windows 10 and Windows 11. For other operating systems you can download the [latest release](https://github.com/nkdAgility/azure-devops-migration-tools/releases/latest) and unzip it to a folder of your choice. +1. Run `winget install nkdAgility.AzureDevOpsMigrationTools` from the [Windows Terminal](https://learn.microsoft.com/en-us/windows/terminal/) (Not eleveated) to install on Windows 10 and Windows 11. For other operating systems you can download the [latest release](https://github.com/nkdAgility/azure-devops-migration-tools/releases/latest) and unzip it to a folder of your choice. -The tools will be installed to `%Localappdata%\Microsoft\WinGet\Packages\nkdAgility.AzureDevOpsMigrationTools_Microsoft.Winget.Source_XXXXXXXXXX` and a symbolic link to `devopsmigration.exe` that lets you run it from anywhere using `devopsmigration init`. +Note: The tools will be installed to `%Localappdata%\Microsoft\WinGet\Packages\nkdAgility.AzureDevOpsMigrationTools_Microsoft.Winget.Source_XXXXXXXXXX` and a symbolic link to `devopsmigration.exe` that lets you run it from anywhere using `devopsmigration init`. + +**Note: There is a known issue with the winget package that it does not add the tools to the PATH if you use an elevated Terminal. You can add the tools to the PATH manually by adding `%Localappdata%\Microsoft\WinGet\Packages\nkdAgility.AzureDevOpsMigrationTools_Microsoft.Winget.Source_XXXXXXXXXX\tools` to the PATH.** ## Upgrade diff --git a/docs/index.md b/docs/index.md index a943126e0..824907305 100644 --- a/docs/index.md +++ b/docs/index.md @@ -86,12 +86,39 @@ _This is a preview version of both the documentation and the Azure DevOps Migrat * [Options migrating TFS to Azure DevOps from Richard Fennell](https://blogs.blackmarble.co.uk/blogs/rfennell/post/2017/05/10/Options-migrating-TFS-to-VSTS) * [Migrating Test artifacts and all other work item types using the Azure DevOps from Gordon Beeming](https://youtu.be/jU6E0k0eXXw) -#### Getting the Tools +#### Installing and running the tools -There are two ways to get these tools: +These tools are available as a portable application and can be installed in a number of ways, including manually from a zip. +For a more detailed getting started guide please see the [documentation](https://nkdagility.com/docs/azure-devops-migration-tools/getting-started.html). -* (recommended)[Install from Chocolatey](https://chocolatey.org/packages/vsts-sync-migrator/) -* Download the [latest release from GitHub](https://github.com/nkdAgility/azure-devops-migration-tools/releases) and unzip +##### Option 1: Winget + +We use [winget](https://learn.microsoft.com/en-us/windows/package-manager/winget/) to host the tools, and you can use the command `winget install nkdAgility.AzureDevOpsMigrationTools` to install them on Windows 10 and Windows 11. + +The tools will be installed to `%Localappdata%\Microsoft\WinGet\Packages\nkdAgility.AzureDevOpsMigrationTools_Microsoft.Winget.Source_XXXXXXXXXX` and a symbolic link to `devopsmigration.exe` that lets you run it from anywhere using `devopsmigration init`. + +**NOTE: Do not install using an elevated command prompt!** + +##### Option 2: Chocolatey + +We also deploy to [Chocolatey](https://chocolatey.org/packages/nkdagility.azuredevopsmigrationtools) and you can use the command `choco install vsts-sync-migrator` to install them on Windows Server. + +The tools will be installed to `C:\Tools\MigrationTools\` which should be added to the path. You can run `devopsmigration.exe` + +##### Option 3: Manual + +You can download the [latest release](https://github.com/nkdAgility/azure-devops-migration-tools/releases/latest) and unzip it to a folder of your choice. + +## Advanced tools + +There are additional advanced tooling available on [Azure DevOps Automation Tools](https://github.com/nkdAgility/azure-devops-automation-tools). These are a collection of Powershell scripts that can be used to; + +- Generate Migration Tools configurations across many projects on many organisations +- Export Stats on many projects on many organisations +- Publish Custom fields across many projects on many organisations +- Output the fields and other data for many projects on many organisations + +These tools are designed to help you manage migration of Work Items at scale. ## Support @@ -193,7 +220,7 @@ Check out the [FAQ pages](faq.md) ## Primary Contributors & Consultants -* **Martin Hinshelwood, naked Agility Ltd** - [@MrHinsh](https://github.com/MrHinsh) is the founder of the Azure DevOps Migration Tools is available worldwide to help organisations plan and enact their migration efforts. You can contact him through [naked Agility Ltd.](https://nkdagility.com). +* **Martin Hinshelwood, naked Agility Ltd** - [@MrHinsh](https://github.com/MrHinsh) is the founder of the Azure DevOps Migration Tools is available worldwide to help organisations plan and enact their migration efforts. You can contact him through [naked Agility Ltd.](https://nkdagility.com). * **Wes MacDonald, LIKE 10 INC.** - [@wesmacdonald](https://github.com/wesmacdonald) is a DevOps Consultant located in Toronto, Canada. You can reach out to him via [LIKE 10 INC.](http://www.like10.com). * **Ove Bastiansen** - [@ovebastiansen](https://github.com/ovebastiansen) is a DevOps consultant living in Oslo, Norway, but working worldwide in todays remote friendly world. You can reach him via his GitHub profile [https://github.com/ovebastiansen](https://github.com/ovebastiansen). * **Gordon Beeming** - [@DevStarOps](https://github.com/DevStarOps) is a DevOps Specialist in Durban, South Africa working at [Derivco](https://derivco.com). You can reach him via his [profile page](https://devstarops.com/) that links to all social media. diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/MigrationTools.Clients.AzureDevops.ObjectModel.Tests.csproj b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/MigrationTools.Clients.AzureDevops.ObjectModel.Tests.csproj index f3db041f9..d89f6497a 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/MigrationTools.Clients.AzureDevops.ObjectModel.Tests.csproj +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/MigrationTools.Clients.AzureDevops.ObjectModel.Tests.csproj @@ -11,11 +11,11 @@ - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsNodeStructureTests.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsNodeStructureTests.cs index 2c8510150..8ef065246 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsNodeStructureTests.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsNodeStructureTests.cs @@ -11,11 +11,30 @@ namespace MigrationTools.ProcessorEnrichers.Tests public class TfsNodeStructureTests { private ServiceProvider _services; + private TfsNodeStructure _structure; [TestInitialize] public void Setup() { _services = ServiceProviderHelper.GetServices(); + _structure = _services.GetRequiredService(); + _structure.ApplySettings(new TfsNodeStructureSettings + { + FoundNodes = new Dictionary(), + SourceProjectName = "SourceServer", + TargetProjectName = "TargetServer", + }); + _structure.Configure(new TfsNodeStructureOptions + { + AreaMaps = new Dictionary + { + { "SourceServer", "TargetServer" } + }, + IterationMaps = new Dictionary + { + { "SourceServer", "TargetServer" } + }, + }); } [TestMethod(), TestCategory("L0"), TestCategory("AzureDevOps.ObjectModel")] @@ -46,5 +65,168 @@ public void GetTfsNodeStructure_WithDifferentAreaPath() Assert.AreEqual(newNodeName, @"TargetProject\test\PUL"); } + + [TestMethod, TestCategory("L0")] + public void TestFixAreaPath_WhenNoAreaPathOrIterationPath_DoesntChangeQuery() + { + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + string targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); + + Assert.AreEqual(WIQLQueryBit, targetWIQLQueryBit); + } + + + [TestMethod, TestCategory("L0")] + public void TestFixAreaPath_WhenAreaPathInQuery_ChangesQuery() + { + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'SourceServer\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'TargetServer\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + string targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + } + + [TestMethod, TestCategory("L1")] + public void TestFixAreaPath_WhenAreaPathInQuery_WithPrefixProjectToNodesEnabled_ChangesQuery() + { + var nodeStructure = _services.GetRequiredService(); + + // For this test we use the prefixing of the project node and no remapping rule + + + nodeStructure.Configure(new TfsNodeStructureOptions + { + AreaMaps = new Dictionary() + { + { "^SourceServer\\\\(.*)" , "TargetServer\\SourceServer\\$1" } + }, + IterationMaps = new Dictionary(){ + { "^SourceServer\\\\(.*)" , "TargetServer\\SourceServer\\$1" } + }, + }); + + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'SourceServer\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'TargetServer\SourceServer\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + string targetWIQLQuery = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQuery); + } + + [TestMethod, TestCategory("L1")] + public void TestFixAreaPath_WhenAreaPathInQuery_WithPrefixProjectToNodesDisabled_SupportsWhitespaces() + { + var nodeStructure = _services.GetRequiredService(); + + nodeStructure.ApplySettings(new TfsNodeStructureSettings + { + FoundNodes = new Dictionary(), + SourceProjectName = "Source Project", + TargetProjectName = "Target Project", + }); + + // For this test we use no remapping rule + nodeStructure.Configure(new TfsNodeStructureOptions + { + AreaMaps = new Dictionary(), + IterationMaps = new Dictionary(), + }); + + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'Source Project\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'Target Project\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + string targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "Source Project", "Target Project", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + } + + [TestMethod, TestCategory("L1")] + public void TestFixAreaPath_WhenAreaPathInQuery_WithPrefixProjectToNodesEnabled_SupportsWhitespaces() + { + var nodeStructure = _services.GetRequiredService(); + + nodeStructure.ApplySettings(new TfsNodeStructureSettings + { + FoundNodes = new Dictionary(), + SourceProjectName = "Source Project", + TargetProjectName = "Target Project", + }); + + // For this test we use the prefixing of the project node and no remapping rules + nodeStructure.Configure(new TfsNodeStructureOptions + { + AreaMaps = new Dictionary() + { + { "^Source Project\\\\(.*)" , "Target Project\\Source Project\\$1" } + }, + IterationMaps = new Dictionary(){ + { "^Source Project\\\\(.*)" , "Target Project\\Source Project\\$1" } + }, + }); + + var WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'Source Project\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + var expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'Target Project\Source Project\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + var targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "Source Project", "Target Project", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + } + + [TestMethod, TestCategory("L0")] + public void TestFixAreaPath_WhenMultipleAreaPathInQuery_ChangesQuery() + { + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'SourceServer\Area\Path1' OR [System.AreaPath] = 'SourceServer\Area\Path2' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'TargetServer\Area\Path1' OR [System.AreaPath] = 'TargetServer\Area\Path2' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + string targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + } + + [TestMethod, TestCategory("L0")] + public void TestFixAreaPath_WhenAreaPathAtEndOfQuery_ChangesQuery() + { + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan') AND [System.AreaPath] = 'SourceServer\Area\Path1'"; + string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan') AND [System.AreaPath] = 'TargetServer\Area\Path1'"; + + string targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + } + + [TestMethod, TestCategory("L0")] + public void TestFixIterationPath_WhenInQuery_ChangesQuery() + { + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.IterationPath] = 'SourceServer\Iteration\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.IterationPath] = 'TargetServer\Iteration\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + string targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + } + + [TestMethod, TestCategory("L0")] + public void TestFixAreaPathAndIteration_WhenMultipleOccuranceInQuery_ChangesQuery() + { + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND ([System.AreaPath] = 'SourceServer\Area\Path1' OR [System.AreaPath] = 'SourceServer\Area\Path2') AND ([System.IterationPath] = 'SourceServer\Iteration\Path1' OR [System.IterationPath] = 'SourceServer\Iteration\Path2') AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND ([System.AreaPath] = 'TargetServer\Area\Path1' OR [System.AreaPath] = 'TargetServer\Area\Path2') AND ([System.IterationPath] = 'TargetServer\Iteration\Path1' OR [System.IterationPath] = 'TargetServer\Iteration\Path2') AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + string targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + } + + [TestMethod, TestCategory("L0")] + public void TestFixAreaPathAndIteration_WhenMultipleOccuranceWithMixtureOrEqualAndUnderOperatorsInQuery_ChangesQuery() + { + string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND ([System.AreaPath] = 'SourceServer\Area\Path1' OR [System.AreaPath] UNDER 'SourceServer\Area\Path2') AND ([System.IterationPath] UNDER 'SourceServer\Iteration\Path1' OR [System.IterationPath] = 'SourceServer\Iteration\Path2') AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND ([System.AreaPath] = 'TargetServer\Area\Path1' OR [System.AreaPath] UNDER 'TargetServer\Area\Path2') AND ([System.IterationPath] UNDER 'TargetServer\Iteration\Path1' OR [System.IterationPath] = 'TargetServer\Iteration\Path2') AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; + + string targetWIQLQueryBit = _structure.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); + + Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + } } } \ No newline at end of file diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsRevisionManagerTests.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsRevisionManagerTests.cs index 92c196a60..a31a86569 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsRevisionManagerTests.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel.Tests/ProcessorEnrichers/TfsRevisionManagerTests.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using Microsoft.Extensions.DependencyInjection; using Microsoft.VisualStudio.TestTools.UnitTesting; using MigrationTools.DataContracts; @@ -30,19 +32,19 @@ private static TfsRevisionManagerOptions GetTfsRevisionManagerOptions() return migrationConfig; } - private static WorkItemData GetWorkItemWithRevisions(DateTime currentDateTime, int startHours = 1, int endHours = 1) + private static List GetWorkItemWithRevisions(DateTime currentDateTime, int startHours = 1, int endHours = 1, bool dateIncreasing = true) { - var fakeWorkItem = new WorkItemData(); - fakeWorkItem.Id = Guid.NewGuid().ToString(); - fakeWorkItem.Revisions = new System.Collections.Generic.SortedDictionary(); + var revisions = new System.Collections.Generic.SortedDictionary(); for (int i = startHours; i < endHours + startHours; i++) { - fakeWorkItem.Revisions.Add(i, new RevisionItem() { Index = i, Number = i, ChangedDate = currentDateTime.AddHours(-i) }); + DateTime dateTime = dateIncreasing ? currentDateTime.AddHours(i) : currentDateTime; + revisions.Add(i, new RevisionItem() { Index = i, Number = i, ChangedDate = dateTime, OriginalChangedDate = dateTime }); } - return fakeWorkItem; + return revisions.Values.ToList(); } + [TestMethod(), TestCategory("L0"), TestCategory("AzureDevOps.ObjectModel")] public void TfsRevisionManagerInSync1() { @@ -51,8 +53,8 @@ public void TfsRevisionManagerInSync1() processorEnricher.Configure(peOptions); var currentDateTime = System.DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 1); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 1, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 1); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -68,12 +70,12 @@ public void TfsRevisionManagerInSync10() processorEnricher.Configure(peOptions); var currentDateTime = System.DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 10); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 1, 10); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 10); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 10); var revs = processorEnricher.GetRevisionsToMigrate(source, target); - Assert.AreEqual(revs.Count, 0); + Assert.AreEqual(0, revs.Count); } @@ -85,8 +87,8 @@ public void TfsRevisionManagerSync1() processorEnricher.Configure(peOptions); var currentDateTime = System.DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 2); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 2, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 2); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -101,8 +103,8 @@ public void TfsRevisionManagerSync10() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 11); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 11, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 11); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -117,9 +119,9 @@ public void TfsRevisionManagerReplayRevisionsOff() var processorEnricher = Services.GetRequiredService(); processorEnricher.Configure(peOptions); - var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 4); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 4, 1); + var currentDateTime = DateTime.Now.AddDays(-100); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 4); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -136,12 +138,12 @@ public void TfsRevisionManagerMaxRevision51() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 2); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 2, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 2); + List target = GetWorkItemWithRevisions(currentDateTime, 2, 2); var revs = processorEnricher.GetRevisionsToMigrate(source, target); - Assert.AreEqual(1, revs.Count); + Assert.AreEqual(0, revs.Count); } [TestMethod(), TestCategory("L0"), TestCategory("AzureDevOps.ObjectModel")] @@ -153,8 +155,8 @@ public void TfsRevisionManagerMaxRevision56() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 7); - WorkItemData target = GetWorkItemWithRevisions(currentDateTime, 7, 1); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 7); + List target = GetWorkItemWithRevisions(currentDateTime, 1, 1); var revs = processorEnricher.GetRevisionsToMigrate(source, target); @@ -170,12 +172,40 @@ public void TfsRevisionManagerMaxRevision59() processorEnricher.Configure(peOptions); var currentDateTime = DateTime.Now; - WorkItemData source = GetWorkItemWithRevisions(currentDateTime, 1, 10); + List source = GetWorkItemWithRevisions(currentDateTime, 1, 10); var revs = processorEnricher.GetRevisionsToMigrate(source, null); Assert.AreEqual(5, revs.Count); } + [TestMethod(), TestCategory("L0"), TestCategory("AzureDevOps.ObjectModel")] + public void TfsRevisionManagerDatesMustBeIncreasing() + { + var peOptions = GetTfsRevisionManagerOptions(); + var processorEnricher = Services.GetRequiredService(); + processorEnricher.Configure(peOptions); + + var currentDateTime = DateTime.Now; + List source = GetWorkItemWithRevisions(currentDateTime, 1, 10, false); + + var revs = processorEnricher.GetRevisionsToMigrate(source, null); + Assert.AreEqual(true, CheckDateIncreasing(revs)); + } + + private static bool CheckDateIncreasing(List revs) + { + DateTime lastDatetime = DateTime.MinValue; + bool increasing = true; + foreach (var rev in revs) + { + if (rev.ChangedDate == lastDatetime) + { + increasing = false; + } + lastDatetime = rev.ChangedDate; + } + return increasing; + } } } diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/Endpoints/TfsWorkItemConvertor.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/Endpoints/TfsWorkItemConvertor.cs index fad6bec7f..3448dfda0 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/Endpoints/TfsWorkItemConvertor.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/Endpoints/TfsWorkItemConvertor.cs @@ -26,6 +26,7 @@ public void MapWorkItemtoWorkItemData(WorkItemData context_wid, WorkItem context fieldItems[revField.Key] = new FieldItem { Name = revField.Value.Name, + FieldType = revField.Value.FieldType, ReferenceName = revField.Value.ReferenceName, Value = revField.Value.Value, internalObject = revField.Value.internalObject, @@ -47,6 +48,7 @@ private SortedDictionary GetRevisionItems(RevisionCollection Index = x.Index, Number = (int)x.Fields["System.Rev"].Value, ChangedDate = (DateTime)x.Fields["System.ChangedDate"].Value, + OriginalChangedDate = (DateTime)x.Fields["System.ChangedDate"].Value, Type = x.Fields["System.WorkItemType"].Value as string, Fields = GetFieldItems(x.Fields) }).ToList(); @@ -81,6 +83,7 @@ private Dictionary GetFieldItems(FieldCollection tfsFields) ReferenceName = x.ReferenceName, Value = x.Value, FieldType = x.FieldDefinition.FieldType.ToString(), + IsIdentity = x.FieldDefinition.IsIdentity, internalObject = x }) .ToDictionary(r => r.ReferenceName); diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsUserMappingEnricher.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsUserMappingEnricher.cs deleted file mode 100644 index 579877b8f..000000000 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsUserMappingEnricher.cs +++ /dev/null @@ -1,86 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.VisualStudio.Services.Commerce; -using MigrationTools.DataContracts; -using MigrationTools.Processors; - -namespace MigrationTools.Enrichers -{ - public class TfsUserMappingEnricher : WorkItemProcessorEnricher - { - - private readonly IMigrationEngine Engine; - - public TfsUserMappingEnricher(IServiceProvider services, ILogger logger) : base(services, logger) - { - Engine = services.GetRequiredService(); - - } - - public override void Configure(IProcessorEnricherOptions options) - { - throw new NotImplementedException(); - } - - protected override void EntryForProcessorType(IProcessor processor) - { - throw new NotImplementedException(); - } - - protected override void RefreshForProcessorType(IProcessor processor) - { - throw new NotImplementedException(); - } - - [Obsolete] - public override int Enrich(WorkItemData sourceWorkItem, WorkItemData targetWorkItem) - { - throw new NotImplementedException(); - } - - private List extractUserList(List workitems, List identityFieldsToCheck) - { - List foundUsers = new List(); - foreach (var wItem in workitems) - { - foreach (var rItem in wItem.Revisions.Values) { - foreach (var fItem in rItem.Fields.Values) - { - if (identityFieldsToCheck.Contains( fItem.ReferenceName, new CaseInsensativeStringComparer())) { - if (!foundUsers.Contains(fItem.Value)) - { - foundUsers.Add(fItem.Value.ToString()); - } - } - } - } - } - return foundUsers; - } - - public Dictionary findUsersToMap(List sourceWorkItems, List identityFieldsToCheck) - { - List sourceUsers = extractUserList(sourceWorkItems, identityFieldsToCheck); - return sourceUsers.ToDictionary(item => item, item => ""); - } - } - - public class CaseInsensativeStringComparer : IEqualityComparer - { - - public bool Equals(String x, String y) - { - return x?.IndexOf(y, StringComparison.OrdinalIgnoreCase) >= 0; - } - - public int GetHashCode(String obj) - { - return obj.GetHashCode(); - } - } -} diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/MigrationTools.Clients.AzureDevops.ObjectModel.csproj b/src/MigrationTools.Clients.AzureDevops.ObjectModel/MigrationTools.Clients.AzureDevops.ObjectModel.csproj index 8063f974f..7e2cbd8fc 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/MigrationTools.Clients.AzureDevops.ObjectModel.csproj +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/MigrationTools.Clients.AzureDevops.ObjectModel.csproj @@ -12,8 +12,9 @@ - - + + + diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsAttachmentEnricher.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricher.cs similarity index 55% rename from src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsAttachmentEnricher.cs rename to src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricher.cs index 160409dee..bc9e15419 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/Enrichers/TfsAttachmentEnricher.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricher.cs @@ -2,30 +2,40 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.TeamFoundation.Server; using Microsoft.TeamFoundation.WorkItemTracking.Client; using Microsoft.TeamFoundation.WorkItemTracking.Proxy; +using MigrationTools._EngineV1.Configuration.Processing; using MigrationTools._EngineV1.Enrichers; using MigrationTools.DataContracts; +using MigrationTools.Endpoints; +using MigrationTools.Enrichers; +using MigrationTools.Processors; using Serilog; -namespace MigrationTools.Enrichers +namespace MigrationTools.ProcessorEnrichers { - public class TfsAttachmentEnricher : IAttachmentMigrationEnricher + public class TfsAttachmentEnricher : WorkItemProcessorEnricher, IAttachmentMigrationEnricher { private WorkItemServer _server; private string _exportBasePath; private string _exportWiPath; private int _maxAttachmentSize; + private TfsAttachmentEnricherOptions _options; + private WorkItemServer _workItemServer; - public TfsAttachmentEnricher(WorkItemServer workItemServer, string exportBasePath, int maxAttachmentSize = 480000000) + public TfsAttachmentEnricherOptions Options { get { return _options; } } + + public TfsAttachmentEnricher(IServiceProvider services, ILogger logger) : base(services, logger) { - _server = workItemServer; - _exportBasePath = exportBasePath; - _maxAttachmentSize = maxAttachmentSize; + } public void ProcessAttachemnts(WorkItemData source, WorkItemData target, bool save = true) { + SetupWorkItemServer(); if (source is null) { throw new ArgumentNullException(nameof(source)); @@ -35,14 +45,13 @@ public void ProcessAttachemnts(WorkItemData source, WorkItemData target, bool sa { throw new ArgumentNullException(nameof(target)); } - - Log.Information("AttachmentMigrationEnricher: Migrating {AttachmentCount} attachemnts from {SourceWorkItemID} to {TargetWorkItemID}", source.ToWorkItem().Attachments.Count, source.Id, target.Id); + Log.LogInformation("AttachmentMigrationEnricher: Migrating {AttachmentCount} attachemnts from {SourceWorkItemID} to {TargetWorkItemID}", source.ToWorkItem().Attachments.Count, source.Id, target.Id); _exportWiPath = Path.Combine(_exportBasePath, source.ToWorkItem().Id.ToString()); - if (System.IO.Directory.Exists(_exportWiPath)) + if (Directory.Exists(_exportWiPath)) { - System.IO.Directory.Delete(_exportWiPath, true); + Directory.Delete(_exportWiPath, true); } - System.IO.Directory.CreateDirectory(_exportWiPath); + Directory.CreateDirectory(_exportWiPath); int count = 0; @@ -51,60 +60,62 @@ public void ProcessAttachemnts(WorkItemData source, WorkItemData target, bool sa count++; if (count > 100) { - break; + break; } try { string filepath = null; - System.IO.Directory.CreateDirectory(Path.Combine(_exportWiPath, wia.Id.ToString())); + Directory.CreateDirectory(Path.Combine(_exportWiPath, wia.Id.ToString())); filepath = ExportAttachment(source.ToWorkItem(), wia, _exportWiPath); - Log.Debug("AttachmentMigrationEnricher: Exported {Filename} to disk", System.IO.Path.GetFileName(filepath)); + Log.LogDebug("AttachmentMigrationEnricher: Exported {Filename} to disk", Path.GetFileName(filepath)); if (filepath != null) { ImportAttachment(target.ToWorkItem(), wia, filepath, save); - Log.Debug("AttachmentMigrationEnricher: Imported {Filename} from disk", System.IO.Path.GetFileName(filepath)); + Log.LogDebug("AttachmentMigrationEnricher: Imported {Filename} from disk", Path.GetFileName(filepath)); } } catch (Exception ex) { - Log.Error(ex, "AttachmentMigrationEnricher:Unable to process atachment from source wi {SourceWorkItemId} called {AttachmentName}", source.ToWorkItem().Id, wia.Name); + Log.LogError(ex, "AttachmentMigrationEnricher:Unable to process atachment from source wi {SourceWorkItemId} called {AttachmentName}", source.ToWorkItem().Id, wia.Name); } } if (save) { target.SaveToAzureDevOps(); - Log.Information("Work iTem now has {AttachmentCount} attachemnts", source.ToWorkItem().Attachments.Count); + Log.LogInformation("Work iTem now has {AttachmentCount} attachemnts", source.ToWorkItem().Attachments.Count); CleanUpAfterSave(); } } public void CleanUpAfterSave() { - if (_exportWiPath != null && System.IO.Directory.Exists(_exportWiPath)) + SetupWorkItemServer(); + if (_exportWiPath != null && Directory.Exists(_exportWiPath)) { try { - System.IO.Directory.Delete(_exportWiPath, true); + Directory.Delete(_exportWiPath, true); _exportWiPath = null; } catch (Exception) { - Log.Warning(" ERROR: Unable to delete folder {0}", _exportWiPath); + Log.LogWarning(" ERROR: Unable to delete folder {0}", _exportWiPath); } } } private string ExportAttachment(WorkItem wi, Attachment wia, string exportpath) { + SetupWorkItemServer(); string fname = GetSafeFilename(wia.Name); - Log.Debug(fname); - + Log.LogDebug(fname); + string fpath = Path.Combine(exportpath, wia.Id.ToString(), fname); - + if (!File.Exists(fpath)) { - Log.Debug(string.Format("...downloading {0} to {1}", fname, exportpath)); + Log.LogDebug(string.Format("...downloading {0} to {1}", fname, exportpath)); try { var fileLocation = _server.DownloadFile(wia.Id); @@ -112,20 +123,21 @@ private string ExportAttachment(WorkItem wi, Attachment wia, string exportpath) } catch (Exception ex) { - Log.Error(ex, "Exception downloading attachements"); + Log.LogError(ex, "Exception downloading attachements"); return null; } } else { - Log.Debug("...already downloaded"); + Log.LogDebug("...already downloaded"); } return fpath; } private void ImportAttachment(WorkItem targetWorkItem, Attachment wia, string filepath, bool save = true) { - var filename = System.IO.Path.GetFileName(filepath); + SetupWorkItemServer(); + var filename = Path.GetFileName(filepath); FileInfo fi = new FileInfo(filepath); if (_maxAttachmentSize > fi.Length) { @@ -152,12 +164,12 @@ private void ImportAttachment(WorkItem targetWorkItem, Attachment wia, string fi } else { - Log.Debug(" [SKIP] WorkItem {0} already contains attachment {1}", targetWorkItem.Id, filepath); + Log.LogDebug(" [SKIP] WorkItem {0} already contains attachment {1}", targetWorkItem.Id, filepath); } } else { - Log.Warning(" [SKIP] Attachment {filename} on Work Item {targetWorkItemId} is bigger than the limit of {maxAttachmentSize} bites for Azure DevOps.", filename, targetWorkItem.Id, _maxAttachmentSize); + Log.LogWarning(" [SKIP] Attachment {filename} on Work Item {targetWorkItemId} is bigger than the limit of {maxAttachmentSize} bites for Azure DevOps.", filename, targetWorkItem.Id, _maxAttachmentSize); } } @@ -165,5 +177,30 @@ public string GetSafeFilename(string filename) { return string.Join("_", filename.Split(Path.GetInvalidFileNameChars())); } + + private void SetupWorkItemServer() + { + if (_workItemServer != null) + { + IMigrationEngine engine = Services.GetRequiredService(); + _workItemServer = engine.Source.GetService(); + } + } + + protected override void RefreshForProcessorType(IProcessor processor) + { + throw new NotImplementedException(); + } + + protected override void EntryForProcessorType(IProcessor processor) + { + throw new NotImplementedException(); + } + + public override void Configure(IProcessorEnricherOptions options) + { + _options = (TfsAttachmentEnricherOptions)options; + + } } } \ No newline at end of file diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricherOptions.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricherOptions.cs new file mode 100644 index 000000000..0b99935d4 --- /dev/null +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsAttachmentEnricherOptions.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using Microsoft.TeamFoundation.Build.Client; +using MigrationTools.ProcessorEnrichers; + +namespace MigrationTools.Enrichers +{ + public class TfsAttachmentEnricherOptions : ProcessorEnricherOptions, ITfsAttachmentEnricherOptions + { + + + public override Type ToConfigure => typeof(TfsAttachmentEnricher); + + /// + /// `AttachmentMigration` is set to true then you need to specify a working path for attachments to be saved locally. + /// + /// C:\temp\Migration\ + public string ExportBasePath { get; set; } + + /// + /// `AttachmentMigration` is set to true then you need to specify a max file size for upload in bites. + /// For Azure DevOps Services the default is 480,000,000 bites (60mb), for TFS its 32,000,000 bites (4mb). + /// + /// 480000000 + public int MaxAttachmentSize { get; set; } + + public override void SetDefaults() + { + Enabled = true; + ExportBasePath = @"c:\temp\WorkItemAttachmentExport"; + MaxAttachmentSize = 480000000; + } + + static public TfsAttachmentEnricherOptions GetDefaults() + { + var result = new TfsAttachmentEnricherOptions(); + result.SetDefaults(); + return result; + } + } + + public interface ITfsAttachmentEnricherOptions + { + public string ExportBasePath { get; set; } + public int MaxAttachmentSize { get; set; } + + } +} \ No newline at end of file diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructure.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructure.cs index 39936834e..134275d1a 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructure.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsNodeStructure.cs @@ -104,13 +104,16 @@ public string GetNewNodeName(string sourceNodePath, TfsNodeStructureType nodeStr Log.LogDebug("NodeStructureEnricher.GetNewNodeName::Mappers", mappers); foreach (var mapper in mappers) { - Log.LogDebug("NodeStructureEnricher.GetNewNodeName::MapperToRun::{key}", mapper.Key); + Log.LogDebug("NodeStructureEnricher.GetNewNodeName::Mappers::{key}", mapper.Key); if (Regex.IsMatch(sourceNodePath, mapper.Key, RegexOptions.IgnoreCase)) { - Log.LogDebug("NodeStructureEnricher.GetNewNodeName::MapperMatched::{key}", mapper.Key); + Log.LogDebug("NodeStructureEnricher.GetNewNodeName::Mappers::{key}::Match", mapper.Key); string replacement = Regex.Replace(sourceNodePath, mapper.Key, mapper.Value); - Log.LogDebug("NodeStructureEnricher.GetNewNodeName::MapperMatched::{key}::replaceWith({replace})", mapper.Key, replacement); + Log.LogDebug("NodeStructureEnricher.GetNewNodeName::Mappers::{key}::replaceWith({replace})", mapper.Key, replacement); return replacement; + } else + { + Log.LogDebug("NodeStructureEnricher.GetNewNodeName::Mappers::{key}::NoMatch", mapper.Key); } } @@ -689,5 +692,56 @@ public string GetMappingForMissingItem(NodeStructureItem missingItem) return null; } + private const string RegexPatternForAreaAndIterationPathsFix = "\\[?(?System.AreaPath|System.IterationPath)+\\]?[^']*'(?[^']*(?:''.[^']*)*)'"; + + public string FixAreaPathAndIterationPathForTargetQuery(string sourceWIQLQuery, string sourceProject, string targetProject, ILogger? contextLog) + { + + string targetWIQLQuery = sourceWIQLQuery; + + if (string.IsNullOrWhiteSpace(targetWIQLQuery)) + { + return targetWIQLQuery; + } + + var matches = Regex.Matches(targetWIQLQuery, RegexPatternForAreaAndIterationPathsFix); + + + if (string.IsNullOrWhiteSpace(sourceProject) + || string.IsNullOrWhiteSpace(targetProject) + || sourceProject == targetProject) + { + return targetWIQLQuery; + } + + foreach (Match match in matches) + { + var value = match.Groups["value"].Value; + if (string.IsNullOrWhiteSpace(value) || !value.StartsWith(sourceProject)) + continue; + + var fieldType = match.Groups["key"].Value; + TfsNodeStructureType structureType; + switch (fieldType) + { + case "System.AreaPath": + structureType = TfsNodeStructureType.Area; + break; + case "System.IterationPath": + structureType = TfsNodeStructureType.Iteration; + break; + default: + throw new InvalidOperationException($"Field type {fieldType} is not supported for query remapping."); + } + + var remappedPath = GetNewNodeName(value, structureType); + targetWIQLQuery = targetWIQLQuery.Replace(value, remappedPath); + } + + contextLog?.Information("[FilterWorkItemsThatAlreadyExistInTarget] is enabled. Source project {sourceProject} is replaced with target project {targetProject} on the WIQLQueryBit which resulted into this target WIQLQueryBit \"{targetWIQLQueryBit}\" .", sourceProject, targetProject, targetWIQLQuery); + + return targetWIQLQuery; + } + } } diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsRevisionManager.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsRevisionManager.cs index 418fc243a..85ed3087a 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsRevisionManager.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsRevisionManager.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.TeamFoundation.WorkItemTracking.Client; @@ -73,33 +74,54 @@ protected override void RefreshForProcessorType(IProcessor processor) } } - public List GetRevisionsToMigrate(WorkItemData sourceWorkItem, WorkItemData targetWorkItem) + public List GetRevisionsToMigrate(List sourceRevisions, List targetRevisions) { - // Revisions have been sorted already on object creation. Values of the Dictionary are sorted by RevisionItem.Number - var sortedRevisions = sourceWorkItem.Revisions.Values.ToList(); - LogDebugCurrentSortedRevisions(sourceWorkItem, sortedRevisions); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: Raw Source {sourceWorkItem} Has {sortedRevisions} revisions", sourceWorkItem.Id, sortedRevisions.Count); + EnforceDatesMustBeIncreasing(sourceRevisions); - sortedRevisions = RemoveRevisionsAlreadyOnTarget(targetWorkItem, sortedRevisions); + LogDebugCurrentSortedRevisions(sourceRevisions, "Source"); + LogDebugCurrentSortedRevisions(targetRevisions, "Target"); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: Raw {sourceWorkItem} Has {sortedRevisions} revisions", "Source", sourceRevisions.Count); - RemoveRevisionsAllExceptLatest(sortedRevisions); + sourceRevisions = RemoveRevisionsAlreadyOnTarget(targetRevisions, sourceRevisions); - RemoveRevisionsMoreThanMaxRevisions(sortedRevisions); + RemoveRevisionsAllExceptLatest(sourceRevisions); - LogDebugCurrentSortedRevisions(sourceWorkItem, sortedRevisions); + RemoveRevisionsMoreThanMaxRevisions(sourceRevisions); - return sortedRevisions; + LogDebugCurrentSortedRevisions(sourceRevisions); + + return sourceRevisions; + } + + public void EnforceDatesMustBeIncreasing(List sortedRevisions) + { + Log.LogDebug("TfsRevisionManager::EnforceDatesMustBeIncreasing"); + DateTime lastDateTime = DateTime.MinValue; + foreach (var revision in sortedRevisions) + { + if (revision.ChangedDate == lastDateTime || revision.OriginalChangedDate < lastDateTime) + { + revision.ChangedDate = lastDateTime.AddSeconds(1); + Log.LogDebug("TfsRevisionManager::EnforceDatesMustBeIncreasing[{revision}]::Fix", revision.Number); + } + lastDateTime = revision.ChangedDate; + } } - private void LogDebugCurrentSortedRevisions(WorkItemData sourceWorkItem, List sortedRevisions) + public void LogDebugCurrentSortedRevisions(List sortedRevisions, string designation = "Source") { - Log.LogInformation("Found {RevisionsCount} revisions to migrate on Work item:{sourceWorkItemId}", sortedRevisions.Count, sourceWorkItem.Id); - Log.LogDebug("RevisionsToMigrate:----------------------------------------------------"); + if (sortedRevisions == null) + { + Log.LogDebug("{designation}: RevisionsToMigrate: No revisions to migrate", designation); + return; + } + Log.LogInformation("{designation}: Found {RevisionsCount} revisions to migrate on Work item:{sourceWorkItemId}", designation, sortedRevisions.Count, designation); + Log.LogDebug("{designation}: RevisionsToMigrate:----------------------------------------------------", designation); foreach (RevisionItem item in sortedRevisions) { Log.LogDebug("RevisionsToMigrate: Index:{Index} - Number:{Number} - ChangedDate:{ChangedDate}", item.Index, item.Number, item.ChangedDate); } - Log.LogDebug("RevisionsToMigrate:----------------------------------------------------"); + Log.LogDebug("{designation}: RevisionsToMigrate:----------------------------------------------------", designation); } private void RemoveRevisionsMoreThanMaxRevisions(List sortedRevisions) @@ -111,7 +133,7 @@ private void RemoveRevisionsMoreThanMaxRevisions(List sortedRevisi { var revisionsToRemove = sortedRevisions.Count - Options.MaxRevisions; sortedRevisions.RemoveRange(0, revisionsToRemove); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: MaxRevisions={MaxRevisions}! There are {sortedRevisionsCount} left", Options.MaxRevisions, sortedRevisions.Count); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsMoreThanMaxRevisions MaxRevisions={MaxRevisions}! There are {sortedRevisionsCount} left", Options.MaxRevisions, sortedRevisions.Count); } } @@ -121,29 +143,32 @@ private void RemoveRevisionsAllExceptLatest(List sortedRevisions) { // Remove all but the latest revision if we are not replaying revisions sortedRevisions.RemoveRange(0, sortedRevisions.Count - 1); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: ReplayRevisions=false! There are {sortedRevisionsCount} left", sortedRevisions.Count); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAllExceptLatest ReplayRevisions=false! There are {sortedRevisionsCount} left", sortedRevisions.Count); } } - private List RemoveRevisionsAlreadyOnTarget(WorkItemData targetWorkItem, List sortedRevisions) + private List RemoveRevisionsAlreadyOnTarget(List targetRevisions, List sourceRevisions) { - if (targetWorkItem != null) + if (targetRevisions != null) { - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: Raw Target {targetWorkItemId} Has {targetWorkItemRevCount} revisions", targetWorkItem.Id, targetWorkItem.Revisions.Count); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAlreadyOnTarget Raw Target Has {targetWorkItemRevCount} revisions", targetRevisions.Count); // Target exists so remove any Changed Date matches between them - var targetChangedDates = (from RevisionItem x in targetWorkItem.Revisions.Values select x.ChangedDate).ToList(); + var targetChangedDates = (from RevisionItem x in targetRevisions select x.ChangedDate).ToList(); if (Options.ReplayRevisions) { - sortedRevisions = sortedRevisions.Where(x => !targetChangedDates.Contains(x.ChangedDate)).ToList(); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: After removing Date Matches there are {sortedRevisionsCount} left", sortedRevisions.Count); + sourceRevisions = sourceRevisions.Where(x => !targetChangedDates.Contains(x.ChangedDate)).ToList(); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAlreadyOnTarget After removing Date Matches there are {sortedRevisionsCount} left", sourceRevisions.Count); } // Find Max target date and remove all source revisions that are newer var targetLatestDate = targetChangedDates.Max(); - sortedRevisions = sortedRevisions.Where(x => x.ChangedDate > targetLatestDate).ToList(); - Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate: After removing revisions before target latest date {targetLatestDate} there are {sortedRevisionsCount} left", targetLatestDate, sortedRevisions.Count); + sourceRevisions = sourceRevisions.Where(x => x.ChangedDate > targetLatestDate).ToList(); + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAlreadyOnTarget After removing revisions before target latest date {targetLatestDate} there are {sortedRevisionsCount} left", targetLatestDate, sourceRevisions.Count); } - - return sortedRevisions; + else + { + Log.LogDebug("TfsRevisionManager::GetRevisionsToMigrate::RemoveRevisionsAlreadyOnTarget Target is null"); + } + return sourceRevisions; } public void AttachSourceRevisionHistoryJsonToTarget(WorkItemData sourceWorkItem, WorkItemData targetWorkItem) diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricher.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricher.cs new file mode 100644 index 000000000..21442ee58 --- /dev/null +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricher.cs @@ -0,0 +1,224 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.TeamFoundation.Common; +using Microsoft.TeamFoundation.Server; +using Microsoft.TeamFoundation.WorkItemTracking.Client; +using Microsoft.VisualStudio.Services.Commerce; +using MigrationTools.DataContracts; +using MigrationTools.Enrichers; +using MigrationTools.Processors; + +namespace MigrationTools.ProcessorEnrichers +{ + /// + /// The TfsUserMappingEnricher is used to map users from the source to the target system. Run it with the ExportUsersForMappingContext to create a mapping file then with WorkItemMigrationContext to use the mapping file to update the users in the target system as you migrate the work items. + /// + public class TfsUserMappingEnricher : WorkItemProcessorEnricher + { + + private readonly IMigrationEngine Engine; + IGroupSecurityService _gssSourse; + private IGroupSecurityService _gssTarget; + + private IGroupSecurityService GssSource + { get { + if (_gssSourse == null) + { + _gssSourse = Engine.Source.GetService(); + } + return _gssSourse; + } } + + private IGroupSecurityService GssTarget + { + get + { + if (_gssTarget == null) + { + _gssTarget = Engine.Target.GetService(); + } + return _gssTarget; + } + } + + public TfsUserMappingEnricherOptions Options { get; private set; } + + public TfsUserMappingEnricher(IServiceProvider services, ILogger logger) : base(services, logger) + { + Engine = services.GetRequiredService(); + } + + public override void Configure(IProcessorEnricherOptions options) + { + Options = (TfsUserMappingEnricherOptions)options; + } + + protected override void EntryForProcessorType(IProcessor processor) + { + throw new NotImplementedException(); + } + + protected override void RefreshForProcessorType(IProcessor processor) + { + throw new NotImplementedException(); + } + + [Obsolete] + public override int Enrich(WorkItemData sourceWorkItem, WorkItemData targetWorkItem) + { + throw new NotImplementedException(); + } + + private List GetUsersFromWorkItems(List workitems, List identityFieldsToCheck) + { + List foundUsers = new List(); + foreach (var wItem in workitems) + { + foreach (var rItem in wItem.Revisions.Values) + { + foreach (var fItem in rItem.Fields.Values) + { + if (identityFieldsToCheck.Contains(fItem.ReferenceName, new CaseInsensativeStringComparer())) + { + if (!foundUsers.Contains(fItem.Value) && !string.IsNullOrEmpty((string)fItem.Value)) + { + foundUsers.Add(fItem.Value.ToString()); + } + } + } + } + } + return foundUsers; + } + + + + public void MapUserIdentityField(FieldItem field) + { + if (Options.Enabled && Options.IdentityFieldsToCheck.Contains(field.ReferenceName)) + { + Log.LogDebug($"TfsUserMappingEnricher::MapUserIdentityField [ReferenceName|{field.ReferenceName}]"); + var mapps = GetMappingFileData(); + if (mapps != null && mapps.ContainsKey(field.Value.ToString())) + { + field.Value = mapps[field.Value.ToString()]; + } + + } + } + + public void MapUserIdentityField(Field field) + { + if (Options.Enabled && Options.IdentityFieldsToCheck.Contains(field.ReferenceName)) + { + Log.LogDebug($"TfsUserMappingEnricher::MapUserIdentityField [ReferenceName|{field.ReferenceName}]"); + var mapps = GetMappingFileData(); + if (mapps != null && mapps.ContainsKey(field.Value.ToString())) + { + field.Value = mapps[field.Value.ToString()]; + } + + } + } + + private Dictionary _UserMappings = null; + + private Dictionary GetMappingFileData() { + if (_UserMappings == null && System.IO.File.Exists(Options.UserMappingFile)) { + var fileData = System.IO.File.ReadAllText(Options.UserMappingFile); + try + { + var fileMaps = Newtonsoft.Json.JsonConvert.DeserializeObject>(fileData); + _UserMappings = fileMaps.ToDictionary(x => x.Source.FriendlyName, x => x.target?.FriendlyName); + } + catch (Exception) + { + _UserMappings = new Dictionary(); + Log.LogError($"TfsUserMappingEnricher::GetMappingFileData [UserMappingFile|{Options.UserMappingFile}] <-- invalid - No mapping are applied!"); + } + + } else + { + Log.LogError($"TfsUserMappingEnricher::GetMappingFileData::No User Mapping file Provided! Provide file or disable TfsUserMappingEnricher"); + _UserMappings = new Dictionary(); + } + + return _UserMappings; + + } + + private List GetUsersListFromServer(IGroupSecurityService gss) + { + Identity SIDS = gss.ReadIdentity(SearchFactor.AccountName, "Project Collection Valid Users", QueryMembership.Expanded); + var people = SIDS.Members.ToList().Where(x => x.Contains("\\")).Select(x => x); + + List foundUsers = new List(); + foreach (string user in people) + { + var bits = user.Split('\\'); + Identity sids = gss.ReadIdentity(SearchFactor.AccountName, bits[1], QueryMembership.Expanded); + foundUsers.Add(new IdentityItemData() { FriendlyName = sids.DisplayName, AccountName = sids.AccountName }); + } + return foundUsers; + } + + + public List GetUsersInSourceMappedToTarget() + { + Log.LogDebug("TfsUserMappingEnricher::GetUsersInSourceMappedToTarget"); + if (Options.Enabled) + { + var sourceUsers = GetUsersListFromServer(GssSource); + Log.LogDebug($"TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [SourceUsersCount|{sourceUsers.Count}]"); + var targetUsers = GetUsersListFromServer(GssTarget); + Log.LogDebug($"TfsUserMappingEnricher::GetUsersInSourceMappedToTarget [targetUsersCount|{targetUsers.Count}]"); + return sourceUsers.Select(sUser => new IdentityMapData { Source = sUser, target = targetUsers.SingleOrDefault(tUser => tUser.FriendlyName == sUser.FriendlyName) }).ToList(); + } + else + { + Log.LogWarning("TfsUserMappingEnricher is disabled in settings. You may have users in the source that are not mapped to the target. "); + return null; + } + + } + + + public List GetUsersInSourceMappedToTargetForWorkItems(List sourceWorkItems) + { + if (Options.Enabled) + { + + Dictionary result = new Dictionary(); + List workItemUsers = GetUsersFromWorkItems(sourceWorkItems, Options.IdentityFieldsToCheck); + Log.LogDebug($"TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [workItemUsers|{workItemUsers.Count}]"); + List mappedUsers = GetUsersInSourceMappedToTarget(); + Log.LogDebug($"TfsUserMappingEnricher::GetUsersInSourceMappedToTargetForWorkItems [mappedUsers|{mappedUsers.Count}]"); + return mappedUsers.Where(x => workItemUsers.Contains(x.Source.FriendlyName)).ToList(); + } + else + { + Log.LogWarning("TfsUserMappingEnricher is disabled in settings. You may have users in the source that are not mapped to the target. "); + return null; + } + } + } + + public class CaseInsensativeStringComparer : IEqualityComparer + { + + public bool Equals(string x, string y) + { + return x?.IndexOf(y, StringComparison.OrdinalIgnoreCase) >= 0; + } + + public int GetHashCode(string obj) + { + return obj.GetHashCode(); + } + } +} diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricherOptions.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricherOptions.cs new file mode 100644 index 000000000..308b36511 --- /dev/null +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsUserMappingEnricherOptions.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using Microsoft.TeamFoundation.Build.Client; +using MigrationTools.ProcessorEnrichers; + +namespace MigrationTools.Enrichers +{ + public class TfsUserMappingEnricherOptions : ProcessorEnricherOptions, ITfsUserMappingEnricherOptions + { + + + public override Type ToConfigure => typeof(TfsUserMappingEnricher); + + /// + /// This is a list of the Identiy fields in the Source to check for user mapping purposes. You should list all identiy fields that you wan to map. + /// + public List IdentityFieldsToCheck { get; set; } + + /// + /// This is the file that will be used to export or import the user mappings. Use the ExportUsersForMapping processor to create the file. + /// + public string UserMappingFile { get; set; } + + public override void SetDefaults() + { + Enabled = false; + UserMappingFile = "usermapping.json"; + IdentityFieldsToCheck = new List { + "System.AssignedTo", + "System.ChangedBy", + "System.CreatedBy", + "Microsoft.VSTS.Common.ActivatedBy", + "Microsoft.VSTS.Common.ResolvedBy", + "Microsoft.VSTS.Common.ClosedBy" }; + } + + static public TfsUserMappingEnricherOptions GetDefaults() + { + var result = new TfsUserMappingEnricherOptions(); + result.SetDefaults(); + return result; + } + } + + public interface ITfsUserMappingEnricherOptions + { + List IdentityFieldsToCheck { get; set; } + string UserMappingFile { get; set; } + } +} \ No newline at end of file diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsValidateRequiredField.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsValidateRequiredField.cs index fa9da0919..f85ab8958 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsValidateRequiredField.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ProcessorEnrichers/TfsValidateRequiredField.cs @@ -43,22 +43,30 @@ public bool ValidatingRequiredField(string fieldToFind, List sourc var result = true; foreach (WorkItemType sourceWorkItemType in sourceWorkItemTypes) { - var workItemTypeName = sourceWorkItemType.Name; - if (Engine.TypeDefinitionMaps.Items.ContainsKey(workItemTypeName)) + try { - workItemTypeName = Engine.TypeDefinitionMaps.Items[workItemTypeName].Map(); - } - var targetType = targetTypes[workItemTypeName]; + var workItemTypeName = sourceWorkItemType.Name; + if (Engine.TypeDefinitionMaps.Items.ContainsKey(workItemTypeName)) + { + workItemTypeName = Engine.TypeDefinitionMaps.Items[workItemTypeName].Map(); + } + var targetType = targetTypes[workItemTypeName]; - if (targetType.FieldDefinitions.Contains(fieldToFind)) - { - Log.LogDebug("ValidatingRequiredField: {WorkItemTypeName} contains {fieldToFind}", targetType.Name, fieldToFind); + if (targetType.FieldDefinitions.Contains(fieldToFind)) + { + Log.LogDebug("ValidatingRequiredField: {WorkItemTypeName} contains {fieldToFind}", targetType.Name, fieldToFind); + } + else + { + Log.LogWarning("ValidatingRequiredField: {WorkItemTypeName} does not contain {fieldToFind}", targetType.Name, fieldToFind); + result = false; + } } - else + catch (WorkItemTypeDeniedOrNotExistException ex) { - Log.LogWarning("ValidatingRequiredField: {WorkItemTypeName} does not contain {fieldToFind}", targetType.Name, fieldToFind); - result = false; + Log.LogWarning(ex, "ValidatingRequiredField: Unable to validate one of the work items as its returned by TFS but has been deleted"); } + } return result; } diff --git a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ServiceCollectionExtensions.cs b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ServiceCollectionExtensions.cs index 6415887a2..da0481134 100644 --- a/src/MigrationTools.Clients.AzureDevops.ObjectModel/ServiceCollectionExtensions.cs +++ b/src/MigrationTools.Clients.AzureDevops.ObjectModel/ServiceCollectionExtensions.cs @@ -8,6 +8,7 @@ using MigrationTools.Enrichers; using MigrationTools.FieldMaps.AzureDevops.ObjectModel; using MigrationTools.ProcessorEnrichers; +using MigrationTools.ProcessorEnrichers.WorkItemProcessorEnrichers; using MigrationTools.Processors; namespace MigrationTools @@ -26,6 +27,8 @@ public static void AddMigrationToolServicesForClientAzureDevOpsObjectModel(this context.AddTransient(); // Enrichers + context.AddSingleton(); + context.AddSingleton(); context.AddSingleton(); context.AddSingleton(); context.AddSingleton(); diff --git a/src/MigrationTools.Clients.AzureDevops.Rest.Tests/MigrationTools.Clients.AzureDevops.Rest.Tests.csproj b/src/MigrationTools.Clients.AzureDevops.Rest.Tests/MigrationTools.Clients.AzureDevops.Rest.Tests.csproj index 9e157d3e0..97a410500 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest.Tests/MigrationTools.Clients.AzureDevops.Rest.Tests.csproj +++ b/src/MigrationTools.Clients.AzureDevops.Rest.Tests/MigrationTools.Clients.AzureDevops.Rest.Tests.csproj @@ -8,10 +8,10 @@ - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/MigrationTools.Clients.AzureDevops.Rest/MigrationTools.Clients.AzureDevops.Rest.csproj b/src/MigrationTools.Clients.AzureDevops.Rest/MigrationTools.Clients.AzureDevops.Rest.csproj index c1e5d2163..529acd702 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest/MigrationTools.Clients.AzureDevops.Rest.csproj +++ b/src/MigrationTools.Clients.AzureDevops.Rest/MigrationTools.Clients.AzureDevops.Rest.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/MigrationTools.Clients.FileSystem.Tests/MigrationTools.Clients.FileSystem.Tests.csproj b/src/MigrationTools.Clients.FileSystem.Tests/MigrationTools.Clients.FileSystem.Tests.csproj index 983785e3c..b3775ae29 100644 --- a/src/MigrationTools.Clients.FileSystem.Tests/MigrationTools.Clients.FileSystem.Tests.csproj +++ b/src/MigrationTools.Clients.FileSystem.Tests/MigrationTools.Clients.FileSystem.Tests.csproj @@ -11,10 +11,10 @@ - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/MigrationTools.Clients.InMemory.Tests/MigrationTools.Clients.InMemory.Tests.csproj b/src/MigrationTools.Clients.InMemory.Tests/MigrationTools.Clients.InMemory.Tests.csproj index 1b4619967..e5dea422d 100644 --- a/src/MigrationTools.Clients.InMemory.Tests/MigrationTools.Clients.InMemory.Tests.csproj +++ b/src/MigrationTools.Clients.InMemory.Tests/MigrationTools.Clients.InMemory.Tests.csproj @@ -11,10 +11,10 @@ - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/MigrationTools.ConsoleDataGenerator/MigrationTools.ConsoleDataGenerator.csproj b/src/MigrationTools.ConsoleDataGenerator/MigrationTools.ConsoleDataGenerator.csproj index d9e46dbc8..52769ef72 100644 --- a/src/MigrationTools.ConsoleDataGenerator/MigrationTools.ConsoleDataGenerator.csproj +++ b/src/MigrationTools.ConsoleDataGenerator/MigrationTools.ConsoleDataGenerator.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/MigrationTools.ConsoleFull/Properties/launchSettings.json b/src/MigrationTools.ConsoleFull/Properties/launchSettings.json index 6efaef226..1e63b28b4 100644 --- a/src/MigrationTools.ConsoleFull/Properties/launchSettings.json +++ b/src/MigrationTools.ConsoleFull/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "execute": { "commandName": "Project", - "commandLineArgs": "execute -c \"C:\\temp\\configuration.json\"" + "commandLineArgs": "execute -c \"configuration.json\"" }, "executepipe": { "commandName": "Project", diff --git a/src/MigrationTools.Host.Tests/MigrationTools.Host.Tests.csproj b/src/MigrationTools.Host.Tests/MigrationTools.Host.Tests.csproj index be4826dc5..0caa3e6e3 100644 --- a/src/MigrationTools.Host.Tests/MigrationTools.Host.Tests.csproj +++ b/src/MigrationTools.Host.Tests/MigrationTools.Host.Tests.csproj @@ -16,10 +16,10 @@ - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/MigrationTools.Host/MigrationToolHost.cs b/src/MigrationTools.Host/MigrationToolHost.cs index 3842c7dd2..514527ac9 100644 --- a/src/MigrationTools.Host/MigrationToolHost.cs +++ b/src/MigrationTools.Host/MigrationToolHost.cs @@ -49,7 +49,7 @@ public static IHostBuilder CreateDefaultBuilder(string[] args) .Enrich.WithProcessId() .WriteTo.Console(restrictedToMinimumLevel: LogEventLevel.Debug, theme: AnsiConsoleTheme.Code, outputTemplate: outputTemplate) .WriteTo.ApplicationInsights(services.GetService(), new CustomConverter(), LogEventLevel.Error) - .WriteTo.File(logPath, LogEventLevel.Verbose); + .WriteTo.File(logPath, LogEventLevel.Verbose, outputTemplate: outputTemplate); }) .ConfigureLogging((context, logBuilder) => { @@ -94,7 +94,13 @@ public static IHostBuilder CreateDefaultBuilder(string[] args) }); // Application Insights - services.AddApplicationInsightsTelemetryWorkerService(new ApplicationInsightsServiceOptions { ApplicationVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString(), ConnectionString = "InstrumentationKey=2d666f84-b3fb-4dcf-9aad-65de038d2772" }); + ApplicationInsightsServiceOptions aiso = new ApplicationInsightsServiceOptions(); + aiso.ApplicationVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString(); + aiso.ConnectionString = "InstrumentationKey=2d666f84-b3fb-4dcf-9aad-65de038d2772"; + //# if DEBUG + //aiso.DeveloperMode = true; + //#endif + services.AddApplicationInsightsTelemetryWorkerService(aiso); // Services services.AddTransient(); diff --git a/src/MigrationTools.Host/MigrationTools.Host.csproj b/src/MigrationTools.Host/MigrationTools.Host.csproj index ce61c15b7..896e00e70 100644 --- a/src/MigrationTools.Host/MigrationTools.Host.csproj +++ b/src/MigrationTools.Host/MigrationTools.Host.csproj @@ -14,7 +14,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -23,7 +23,7 @@ - + diff --git a/src/MigrationTools.Integration.Tests/MigrationTools.Integration.Tests.csproj b/src/MigrationTools.Integration.Tests/MigrationTools.Integration.Tests.csproj index da36aa230..cb8c2ac40 100644 --- a/src/MigrationTools.Integration.Tests/MigrationTools.Integration.Tests.csproj +++ b/src/MigrationTools.Integration.Tests/MigrationTools.Integration.Tests.csproj @@ -9,11 +9,11 @@ - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/MigrationTools.TestExtensions/MigrationTools.TestExtensions.csproj b/src/MigrationTools.TestExtensions/MigrationTools.TestExtensions.csproj index 9c632baf6..03e9dca0f 100644 --- a/src/MigrationTools.TestExtensions/MigrationTools.TestExtensions.csproj +++ b/src/MigrationTools.TestExtensions/MigrationTools.TestExtensions.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/MigrationTools.Tests/MigrationTools.Tests.csproj b/src/MigrationTools.Tests/MigrationTools.Tests.csproj index d7807fe6b..3188fed48 100644 --- a/src/MigrationTools.Tests/MigrationTools.Tests.csproj +++ b/src/MigrationTools.Tests/MigrationTools.Tests.csproj @@ -9,11 +9,11 @@ - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/MigrationTools/DataContracts/FieldItem.cs b/src/MigrationTools/DataContracts/FieldItem.cs index 0e37348ae..7fad51a04 100644 --- a/src/MigrationTools/DataContracts/FieldItem.cs +++ b/src/MigrationTools/DataContracts/FieldItem.cs @@ -11,5 +11,6 @@ public class FieldItem public string Name { get; set; } public object Value { get; set; } public string FieldType { get; set; } + public bool IsIdentity { get; set; } } } \ No newline at end of file diff --git a/src/MigrationTools/DataContracts/IdentityItemData.cs b/src/MigrationTools/DataContracts/IdentityItemData.cs new file mode 100644 index 000000000..9dcae21a8 --- /dev/null +++ b/src/MigrationTools/DataContracts/IdentityItemData.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace MigrationTools.DataContracts +{ + public class IdentityItemData + { + public string FriendlyName { get; set; } + public string AccountName { get; set; } + } + + public class IdentityMapData + { + public IdentityItemData Source { get; set; } + public IdentityItemData target { get; set; } + } +} diff --git a/src/MigrationTools/DataContracts/RevisionItem.cs b/src/MigrationTools/DataContracts/RevisionItem.cs index 2be23b999..6da8c0ef0 100644 --- a/src/MigrationTools/DataContracts/RevisionItem.cs +++ b/src/MigrationTools/DataContracts/RevisionItem.cs @@ -9,6 +9,8 @@ public class RevisionItem public int Index { get; set; } public int Number { get; set; } public DateTime ChangedDate { get; set; } + + public DateTime OriginalChangedDate { get; set; } public string Type { get; set; } public Dictionary Fields { get; set; } public Dictionary EnricherData { get; set; } diff --git a/src/MigrationTools/MigrationTools.csproj b/src/MigrationTools/MigrationTools.csproj index c23ed87cd..47b578d18 100644 --- a/src/MigrationTools/MigrationTools.csproj +++ b/src/MigrationTools/MigrationTools.csproj @@ -23,11 +23,11 @@ - + - - + + diff --git a/src/MigrationTools/ProcessorEnrichers/ProcessorEnricherOptions.cs b/src/MigrationTools/ProcessorEnrichers/ProcessorEnricherOptions.cs index e140c27bc..4a767ca2e 100644 --- a/src/MigrationTools/ProcessorEnrichers/ProcessorEnricherOptions.cs +++ b/src/MigrationTools/ProcessorEnrichers/ProcessorEnricherOptions.cs @@ -5,8 +5,9 @@ namespace MigrationTools.Enrichers public abstract class ProcessorEnricherOptions : IProcessorEnricherOptions { /// - /// For internal use + /// If enabled this will run this migrator /// + /// true public bool Enabled { get; set; } public abstract Type ToConfigure { get; } diff --git a/src/MigrationTools/Services/TelemetryClientAdapter.cs b/src/MigrationTools/Services/TelemetryClientAdapter.cs index a6207770f..d656af26c 100644 --- a/src/MigrationTools/Services/TelemetryClientAdapter.cs +++ b/src/MigrationTools/Services/TelemetryClientAdapter.cs @@ -11,6 +11,7 @@ public class TelemetryClientAdapter : ITelemetryLogger public TelemetryClientAdapter(TelemetryClient telemetryClient) { + telemetryClient.InstrumentationKey = "2d666f84-b3fb-4dcf-9aad-65de038d2772"; telemetryClient.Context.Session.Id = Guid.NewGuid().ToString(); telemetryClient.Context.Device.OperatingSystem = Environment.OSVersion.ToString(); if (!(System.Reflection.Assembly.GetEntryAssembly() is null)) diff --git a/src/MigrationTools/_EngineV1/Configuration/EngineConfiguration.cs b/src/MigrationTools/_EngineV1/Configuration/EngineConfiguration.cs index 9fbf032a7..7d6ff45e9 100644 --- a/src/MigrationTools/_EngineV1/Configuration/EngineConfiguration.cs +++ b/src/MigrationTools/_EngineV1/Configuration/EngineConfiguration.cs @@ -13,14 +13,14 @@ public EngineConfiguration() public IMigrationClientConfig Source { get; set; } public IMigrationClientConfig Target { get; set; } - public List FieldMaps { get; set; } - public Dictionary GitRepoMapping { get; set; } + public List FieldMaps { get; set; } = new List(); + public Dictionary GitRepoMapping { get; set; } = new Dictionary(); public string LogLevel { get; private set; } - public List CommonEnrichersConfig { get; set; } + public List CommonEnrichersConfig { get; set; } = new List(); public List Processors { get; set; } public string Version { get; set; } public bool workaroundForQuerySOAPBugEnabled { get; set; } - public Dictionary WorkItemTypeDefinition { get; set; } + public Dictionary WorkItemTypeDefinition { get; set; } = new Dictionary(); } } diff --git a/src/MigrationTools/_EngineV1/Configuration/Processing/ExportUsersForMappingConfig.cs b/src/MigrationTools/_EngineV1/Configuration/Processing/ExportUsersForMappingConfig.cs index 8ca7c8796..10faa54bb 100644 --- a/src/MigrationTools/_EngineV1/Configuration/Processing/ExportUsersForMappingConfig.cs +++ b/src/MigrationTools/_EngineV1/Configuration/Processing/ExportUsersForMappingConfig.cs @@ -4,14 +4,18 @@ namespace MigrationTools._EngineV1.Configuration.Processing { public class ExportUsersForMappingConfig : IProcessorConfig { - public string LocalExportJsonFile { get; set; } public bool Enabled { get; set; } public string WIQLQuery { get; set; } - public List IdentityFieldsToCheck { get; set; } + + + /// `OnlyListUsersInWorkItems` + /// + /// true + public bool OnlyListUsersInWorkItems { get; set; } = true; public string Processor { - get { return "ExportUsersForMapping"; } + get { return "ExportUsersForMappingContext"; } } /// diff --git a/src/MigrationTools/_EngineV1/Configuration/Processing/WorkItemMigrationConfig.cs b/src/MigrationTools/_EngineV1/Configuration/Processing/WorkItemMigrationConfig.cs index 001a1fc11..6ed681219 100644 --- a/src/MigrationTools/_EngineV1/Configuration/Processing/WorkItemMigrationConfig.cs +++ b/src/MigrationTools/_EngineV1/Configuration/Processing/WorkItemMigrationConfig.cs @@ -40,18 +40,6 @@ public class WorkItemMigrationConfig : IWorkItemProcessorConfig /// ? public string Processor => "WorkItemMigrationContext"; - /// - /// If enabled this will migrate all of the attachments at the same time as the work item - /// - /// true - public bool AttachmentMigration { get; set; } - - /// - /// `AttachmentMigration` is set to true then you need to specify a working path for attachments to be saved locally. - /// - /// C:\temp\Migration\ - public string AttachmentWorkingPath { get; set; } - /// /// **beta** If enabled this will fix any image attachments URL's, work item mention URL's or user mentions in the HTML /// fields as well as discussion comments. You must specify a PersonalAccessToken in the Source project for Azure DevOps; @@ -87,13 +75,6 @@ public class WorkItemMigrationConfig : IWorkItemProcessorConfig /// false public bool PauseAfterEachWorkItem { get; set; } - /// - /// `AttachmentMigration` is set to true then you need to specify a max file size for upload in bites. - /// For Azure DevOps Services the default is 480,000,000 bites (60mb), for TFS its 32,000,000 bites (4mb). - /// - /// 480000000 - public int AttachmentMaxSize { get; set; } - /// /// This will create a json file with the revision history and attach it to the work item. Best used with `MaxRevisions` or `ReplayRevisions`. /// @@ -119,7 +100,6 @@ public class WorkItemMigrationConfig : IWorkItemProcessorConfig /// [] public IList WorkItemIDs { get; set; } - /// /// The maximum number of failures to tolerate before the migration fails. When set above zero, a work item migration error is logged but the migration will /// continue until the number of failed items reaches the configured value, after which the migration fails. @@ -151,10 +131,7 @@ public WorkItemMigrationConfig() Enabled = false; WorkItemCreateRetryLimit = 5; FilterWorkItemsThatAlreadyExistInTarget = false; - AttachmentMigration = true; FixHtmlAttachmentLinks = false; - AttachmentWorkingPath = "c:\\temp\\WorkItemAttachmentWorkingFolder\\"; - AttachmentMaxSize = 480000000; UpdateCreatedBy = true; UpdateCreatedDate = true; SkipToFinalRevisedWorkItemType = false; diff --git a/src/MigrationTools/_EngineV1/Processors/MigrationProcessorBase.cs b/src/MigrationTools/_EngineV1/Processors/MigrationProcessorBase.cs index 43a830df2..69202d4b1 100644 --- a/src/MigrationTools/_EngineV1/Processors/MigrationProcessorBase.cs +++ b/src/MigrationTools/_EngineV1/Processors/MigrationProcessorBase.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using MigrationTools._EngineV1.Configuration; using MigrationTools.Enrichers; @@ -90,13 +91,21 @@ protected void PullCommonEnrichersConfig (List().FirstOrDefault(); + TEnricherOptions config = default(TEnricherOptions); + if (commonEnricher == null) + { + commonEnricher= Services.GetService(); + } + if (commonEnrichersStore != null) + { + config = commonEnrichersStore.OfType().FirstOrDefault(); + } if (config == null) { var result = new TEnricherOptions(); result.SetDefaults(); commonEnricher.Configure(result); - Log.LogWarning("Using `{TEnricherOptions}` with Defaults... add a `{TEnricherOptions}` entry to `CommonEnrichersConfig` to customise the settings.", typeof(TEnricherOptions).Name); + Log.LogInformation("Using `{TEnricherOptions}` with Defaults... add a `{TEnricherOptions}` entry to `CommonEnrichersConfig` to customise the settings.", typeof(TEnricherOptions).Name); } else { diff --git a/src/VstsSyncMigrator.Core.Tests/VstsSyncMigrator.Core.Tests.csproj b/src/VstsSyncMigrator.Core.Tests/VstsSyncMigrator.Core.Tests.csproj index 6f851f3e0..06ccdefbd 100644 --- a/src/VstsSyncMigrator.Core.Tests/VstsSyncMigrator.Core.Tests.csproj +++ b/src/VstsSyncMigrator.Core.Tests/VstsSyncMigrator.Core.Tests.csproj @@ -7,11 +7,11 @@ - + - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/VstsSyncMigrator.Core.Tests/WorkItemMigrationTests.cs b/src/VstsSyncMigrator.Core.Tests/WorkItemMigrationTests.cs index 2dcc92e09..02f95049f 100644 --- a/src/VstsSyncMigrator.Core.Tests/WorkItemMigrationTests.cs +++ b/src/VstsSyncMigrator.Core.Tests/WorkItemMigrationTests.cs @@ -18,197 +18,13 @@ namespace VstsSyncMigrator.Core.Tests public class WorkItemMigrationTests { private ServiceProvider _services; - private WorkItemMigrationContext _underTest; [TestInitialize] public void Setup() { - _services = ServiceProviderHelper.GetServices(); - var nodeStructure = _services.GetRequiredService(); - nodeStructure.ApplySettings(new TfsNodeStructureSettings - { - FoundNodes = new Dictionary(), - SourceProjectName = "SourceServer", - TargetProjectName = "TargetServer", - }); - nodeStructure.Configure(new TfsNodeStructureOptions - { - AreaMaps = new Dictionary - { - { "SourceServer", "TargetServer" } - }, - IterationMaps = new Dictionary - { - { "SourceServer", "TargetServer" } - }, - }); - - _underTest = new WorkItemMigrationContext(_services.GetRequiredService(), - _services, - _services.GetRequiredService(), - _services.GetRequiredService>(), - nodeStructure, - _services.GetRequiredService(), - _services.GetRequiredService(), - _services.GetRequiredService(), - _services.GetRequiredService(), - _services.GetRequiredService(), - _services.GetRequiredService>()); - _underTest.Configure(new WorkItemMigrationConfig - { - - }); - } - - [TestMethod] - public void TestFixAreaPath_WhenNoAreaPathOrIterationPath_DoesntChangeQuery() - { - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - string targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); - - Assert.AreEqual(WIQLQueryBit, targetWIQLQueryBit); - } - - - [TestMethod] - public void TestFixAreaPath_WhenAreaPathInQuery_ChangesQuery() - { - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'SourceServer\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'TargetServer\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - string targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); - } - - [TestMethod] - public void TestFixAreaPath_WhenAreaPathInQuery_WithPrefixProjectToNodesEnabled_ChangesQuery() - { - var nodeStructure = _services.GetRequiredService(); - - // For this test we use the prefixing of the project node and no remapping rule - nodeStructure.Configure(new TfsNodeStructureOptions - { - AreaMaps = new Dictionary(), - IterationMaps = new Dictionary(), - }); - - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'SourceServer\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'TargetServer\SourceServer\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - string targetWIQLQuery = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQuery); - } - - [TestMethod] - public void TestFixAreaPath_WhenAreaPathInQuery_WithPrefixProjectToNodesDisabled_SupportsWhitespaces() - { - var nodeStructure = _services.GetRequiredService(); - - nodeStructure.ApplySettings(new TfsNodeStructureSettings - { - FoundNodes = new Dictionary(), - SourceProjectName = "Source Project", - TargetProjectName = "Target Project", - }); - - // For this test we use no remapping rule - nodeStructure.Configure(new TfsNodeStructureOptions - { - AreaMaps = new Dictionary(), - IterationMaps = new Dictionary(), - }); - - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'Source Project\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'Target Project\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - string targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "Source Project", "Target Project", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); - } - - [TestMethod] - public void TestFixAreaPath_WhenAreaPathInQuery_WithPrefixProjectToNodesEnabled_SupportsWhitespaces() - { - var nodeStructure = _services.GetRequiredService(); - - nodeStructure.ApplySettings(new TfsNodeStructureSettings - { - FoundNodes = new Dictionary(), - SourceProjectName = "Source Project", - TargetProjectName = "Target Project", - }); - - // For this test we use the prefixing of the project node and no remapping rules - nodeStructure.Configure(new TfsNodeStructureOptions - { - AreaMaps = new Dictionary(), - IterationMaps = new Dictionary(), - }); - - var WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'Source Project\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - var expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'Target Project\Source Project\Area\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - var targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "Source Project", "Target Project", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); - } - - [TestMethod] - public void TestFixAreaPath_WhenMultipleAreaPathInQuery_ChangesQuery() - { - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'SourceServer\Area\Path1' OR [System.AreaPath] = 'SourceServer\Area\Path2' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.AreaPath] = 'TargetServer\Area\Path1' OR [System.AreaPath] = 'TargetServer\Area\Path2' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - string targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); - } - - [TestMethod] - public void TestFixAreaPath_WhenAreaPathAtEndOfQuery_ChangesQuery() - { - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan') AND [System.AreaPath] = 'SourceServer\Area\Path1'"; - string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan') AND [System.AreaPath] = 'TargetServer\Area\Path1'"; - - string targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); - } - - [TestMethod] - public void TestFixIterationPath_WhenInQuery_ChangesQuery() - { - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.IterationPath] = 'SourceServer\Iteration\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND [System.IterationPath] = 'TargetServer\Iteration\Path1' AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - string targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); + _services = ServiceProviderHelper.GetServices(); } - [TestMethod] - public void TestFixAreaPathAndIteration_WhenMultipleOccuranceInQuery_ChangesQuery() - { - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND ([System.AreaPath] = 'SourceServer\Area\Path1' OR [System.AreaPath] = 'SourceServer\Area\Path2') AND ([System.IterationPath] = 'SourceServer\Iteration\Path1' OR [System.IterationPath] = 'SourceServer\Iteration\Path2') AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND ([System.AreaPath] = 'TargetServer\Area\Path1' OR [System.AreaPath] = 'TargetServer\Area\Path2') AND ([System.IterationPath] = 'TargetServer\Iteration\Path1' OR [System.IterationPath] = 'TargetServer\Iteration\Path2') AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - string targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); - } - - [TestMethod] - public void TestFixAreaPathAndIteration_WhenMultipleOccuranceWithMixtureOrEqualAndUnderOperatorsInQuery_ChangesQuery() - { - string WIQLQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND ([System.AreaPath] = 'SourceServer\Area\Path1' OR [System.AreaPath] UNDER 'SourceServer\Area\Path2') AND ([System.IterationPath] UNDER 'SourceServer\Iteration\Path1' OR [System.IterationPath] = 'SourceServer\Iteration\Path2') AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - string expectTargetQueryBit = @"SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = @TeamProject AND ([System.AreaPath] = 'TargetServer\Area\Path1' OR [System.AreaPath] UNDER 'TargetServer\Area\Path2') AND ([System.IterationPath] UNDER 'TargetServer\Iteration\Path1' OR [System.IterationPath] = 'TargetServer\Iteration\Path2') AND [Microsoft.VSTS.Common.ClosedDate] = '' AND [System.WorkItemType] NOT IN ('Test Suite', 'Test Plan')"; - - string targetWIQLQueryBit = _underTest.FixAreaPathAndIterationPathForTargetQuery(WIQLQueryBit, "SourceServer", "TargetServer", null); - - Assert.AreEqual(expectTargetQueryBit, targetWIQLQueryBit); - } + } } \ No newline at end of file diff --git a/src/VstsSyncMigrator.Core/Execution/MigrationContext/ExportUsersForMapping.cs b/src/VstsSyncMigrator.Core/Execution/MigrationContext/ExportUsersForMapping.cs index aa1aa150d..acdb6043c 100644 --- a/src/VstsSyncMigrator.Core/Execution/MigrationContext/ExportUsersForMapping.cs +++ b/src/VstsSyncMigrator.Core/Execution/MigrationContext/ExportUsersForMapping.cs @@ -4,54 +4,105 @@ using System.Linq; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using MigrationTools; using MigrationTools._EngineV1.Configuration; using MigrationTools._EngineV1.Configuration.Processing; +using MigrationTools._EngineV1.Processors; using MigrationTools.DataContracts; using MigrationTools.DataContracts.Process; +using MigrationTools.EndpointEnrichers; using MigrationTools.Enrichers; +using MigrationTools.ProcessorEnrichers; +using MigrationTools.ProcessorEnrichers.WorkItemProcessorEnrichers; +using Newtonsoft.Json; using VstsSyncMigrator._EngineV1.Processors; namespace VstsSyncMigrator.Core.Execution.MigrationContext { - public class ExportUsersForMapping : StaticProcessorBase + /// + /// ExportUsersForMappingContext is a tool used to create a starter mapping file for users between the source and target systems. + /// Use `ExportUsersForMappingConfig` to configure. + /// + /// ready + /// Work Items + public class ExportUsersForMappingContext : MigrationProcessorBase { private ExportUsersForMappingConfig _config; private TfsUserMappingEnricher _TfsUserMappingEnricher; - public ExportUsersForMapping(IServiceProvider services, IMigrationEngine me, ITelemetryLogger telemetry, ILogger logger) : base(services, me, telemetry, logger) - { - Logger = logger; - } + public override string Name { get { - return "ExportUsersForMapping"; + return "ExportUsersForMappingContext"; } } - public ILogger Logger { get; } + public ILogger Logger { get; } + + private EngineConfiguration _engineConfig; + + public ExportUsersForMappingContext(IMigrationEngine engine, IServiceProvider services, ITelemetryLogger telemetry, ILogger logger, IOptions engineConfig, TfsUserMappingEnricher userMappingEnricher) : base(engine, services, telemetry, logger) + { + Logger = logger; + _engineConfig = engineConfig.Value; + _TfsUserMappingEnricher = userMappingEnricher; + } public override void Configure(IProcessorConfig config) { _config = (ExportUsersForMappingConfig)config; - _TfsUserMappingEnricher = Services.GetRequiredService(); + ImportCommonEnricherConfigs(); + + } + + private void ImportCommonEnricherConfigs() + { + /// setup _engineConfig.CommonEnrichersConfig + if (_engineConfig.CommonEnrichersConfig == null) + { + Log.LogError("CommonEnrichersConfig cant be Null! it must be a minimum of `[]`"); + Environment.Exit(-1); + } + PullCommonEnrichersConfig(_engineConfig.CommonEnrichersConfig, _TfsUserMappingEnricher); } protected override void InternalExecute() { Stopwatch stopwatch = Stopwatch.StartNew(); - ////////////////////////////////////////////////// - List sourceWorkItems = Engine.Source.WorkItems.GetWorkItems(_config.WIQLQuery); - Log.LogInformation("Processing {0} work items from Source", sourceWorkItems.Count); - ///////////////////////////////////////////////// - Dictionary usersToMap = _TfsUserMappingEnricher.findUsersToMap(sourceWorkItems, _config.IdentityFieldsToCheck); + if(string.IsNullOrEmpty(_TfsUserMappingEnricher.Options.UserMappingFile)) + { + Log.LogError("UserMappingFile is not set"); + + throw new ArgumentNullException("UserMappingFile must be set on the TfsUserMappingEnricherOptions in CommonEnrichersConfig."); + } + + List usersToMap = new List(); + if (_config.OnlyListUsersInWorkItems) + { + Log.LogInformation("OnlyListUsersInWorkItems is true, only users in work items will be listed"); + List sourceWorkItems = Engine.Source.WorkItems.GetWorkItems(_config.WIQLQuery); + Log.LogInformation("Processed {0} work items from Source", sourceWorkItems.Count); - System.IO.File.WriteAllText(_config.LocalExportJsonFile, Newtonsoft.Json.JsonConvert.SerializeObject(usersToMap)); + usersToMap = _TfsUserMappingEnricher.GetUsersInSourceMappedToTargetForWorkItems(sourceWorkItems); + Log.LogInformation("Found {usersToMap} total mapped", usersToMap.Count); + } + else + { + Log.LogInformation("OnlyListUsersInWorkItems is false, all users will be listed"); + usersToMap = _TfsUserMappingEnricher.GetUsersInSourceMappedToTarget(); + Log.LogInformation("Found {usersToMap} total mapped", usersToMap.Count); + } + usersToMap = usersToMap.Where(x => x.Source.FriendlyName != x.target?.FriendlyName).ToList(); + Log.LogInformation("Filtered to {usersToMap} total viable mappings", usersToMap.Count); + Dictionary usermappings = usersToMap.ToDictionary(x => x.Source.FriendlyName, x => x.target?.FriendlyName); + System.IO.File.WriteAllText(_TfsUserMappingEnricher.Options.UserMappingFile, Newtonsoft.Json.JsonConvert.SerializeObject(usermappings, Formatting.Indented)); + Log.LogInformation("Writen to: {LocalExportJsonFile}", _TfsUserMappingEnricher.Options.UserMappingFile); ////////////////////////////////////////////////// stopwatch.Stop(); Log.LogInformation("DONE in {Elapsed} seconds"); diff --git a/src/VstsSyncMigrator.Core/Execution/MigrationContext/TestPlansAndSuitesMigrationContext.cs b/src/VstsSyncMigrator.Core/Execution/MigrationContext/TestPlansAndSuitesMigrationContext.cs index 17e559dc7..c3ff9e526 100644 --- a/src/VstsSyncMigrator.Core/Execution/MigrationContext/TestPlansAndSuitesMigrationContext.cs +++ b/src/VstsSyncMigrator.Core/Execution/MigrationContext/TestPlansAndSuitesMigrationContext.cs @@ -75,11 +75,7 @@ public override void Configure(IProcessorConfig config) { _config = (TestPlansAndSuitesMigrationConfig)config; - - var nodeStructureOptions = - _engineConfig.CommonEnrichersConfig.OfType().FirstOrDefault() - ?? throw new InvalidOperationException("Cannot use common node structure because it is not found."); - _nodeStructureEnricher.Configure(nodeStructureOptions); + PullCommonEnrichersConfig(_engineConfig.CommonEnrichersConfig, _nodeStructureEnricher); } diff --git a/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs b/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs index d70d8a72e..a1b988954 100644 --- a/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs +++ b/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs @@ -49,7 +49,6 @@ namespace VstsSyncMigrator.Engine /// Work Items public class WorkItemMigrationContext : MigrationProcessorBase { - private const string RegexPatternForAreaAndIterationPathsFix = "\\[?(?System.AreaPath|System.IterationPath)+\\]?[^']*'(?[^']*(?:''.[^']*)*)'"; private static int _count = 0; private static int _current = 0; @@ -60,10 +59,11 @@ public class WorkItemMigrationContext : MigrationProcessorBase private List _ignore; private ILogger contextLog; - private IAttachmentMigrationEnricher attachmentEnricher; + private TfsAttachmentEnricher _attachmentEnricher; private IWorkItemProcessorEnricher embededImagesEnricher; private IWorkItemProcessorEnricher _workItemEmbededLinkEnricher; private StringManipulatorEnricher _stringManipulatorEnricher; + private TfsUserMappingEnricher _userMappingEnricher; private TfsGitRepositoryEnricher gitRepositoryEnricher; private TfsNodeStructure _nodeStructureEnricher; private ITelemetryLogger _telemetry; @@ -80,6 +80,8 @@ public WorkItemMigrationContext(IMigrationEngine engine, IServiceProvider services, ITelemetryLogger telemetry, ILogger logger, + TfsUserMappingEnricher userMappingEnricher, + TfsAttachmentEnricher attachmentEnricher, TfsNodeStructure nodeStructureEnricher, TfsRevisionManager revisionManager, TfsWorkItemLinkEnricher workItemLinkEnricher, @@ -92,7 +94,9 @@ public WorkItemMigrationContext(IMigrationEngine engine, _telemetry = telemetry; _engineConfig = engineConfig.Value; contextLog = Serilog.Log.ForContext(); + _attachmentEnricher = attachmentEnricher; _nodeStructureEnricher = nodeStructureEnricher; + _userMappingEnricher = userMappingEnricher; _revisionManager = revisionManager; _workItemLinkEnricher = workItemLinkEnricher; _workItemEmbededLinkEnricher = workItemEmbeddedLinkEnricher; @@ -121,7 +125,8 @@ private void ImportCommonEnricherConfigs() PullCommonEnrichersConfig(_engineConfig.CommonEnrichersConfig, _revisionManager); PullCommonEnrichersConfig(_engineConfig.CommonEnrichersConfig, _workItemLinkEnricher); PullCommonEnrichersConfig(_engineConfig.CommonEnrichersConfig, _stringManipulatorEnricher); - + PullCommonEnrichersConfig(_engineConfig.CommonEnrichersConfig, _attachmentEnricher); + PullCommonEnrichersConfig(_engineConfig.CommonEnrichersConfig, _userMappingEnricher); } internal void TraceWriteLine(LogEventLevel level, string message, Dictionary properties = null) @@ -146,8 +151,7 @@ protected override void InternalExecute() ////////////////////////////////////////////////// ValidatePatTokenRequirement(); ////////////////////////////////////////////////// - var workItemServer = Engine.Source.GetService(); - attachmentEnricher = new TfsAttachmentEnricher(workItemServer, _config.AttachmentWorkingPath, _config.AttachmentMaxSize); + embededImagesEnricher = Services.GetRequiredService(); gitRepositoryEnricher = Services.GetRequiredService(); @@ -169,31 +173,11 @@ protected override void InternalExecute() contextLog.Information("Replay all revisions of {sourceWorkItemsCount} work items?", sourceWorkItems.Count); - //Validation: make sure that the ReflectedWorkItemId field name specified in the config exists in the target process, preferably on each work item type. - ////////////////////////////////////////////////// - contextLog.Information("Validating::Check all Target Work Items have the RefectedWorkItemId field"); - - var result = _validateConfig.ValidatingRequiredField( - Engine.Target.Config.AsTeamProjectConfig().ReflectedWorkItemIDFieldName, sourceWorkItems); - if (!result) - { - var ex = new InvalidFieldValueException( - "Not all work items in scope contain a valid ReflectedWorkItemId Field!"); - Log.LogError(ex, "Not all work items in scope contain a valid ReflectedWorkItemId Field!"); - throw ex; - } - ////////////////////////////////////////////////// + ValidateAllWorkItemTypesHaveReflectedWorkItemIdField(sourceWorkItems); ValiddateWorkItemTypesExistInTarget(sourceWorkItems); - ////////////////////////////////////////////////// - - contextLog.Information("Validating::Check that all Area & Iteration paths from Source have a valid mapping on Target"); - List nodeStructureMissingItems = _nodeStructureEnricher.GetMissingRevisionNodes(sourceWorkItems); - if (_nodeStructureEnricher.ValidateTargetNodesExist(nodeStructureMissingItems)) - { - throw new Exception("Missing Iterations in Target preventing progress, check log for list. To continue you MUST configure IterationMaps or AreaMaps that matches the missing paths.."); - } - + ValidateAllNodesExistOrAreMapped(sourceWorkItems); + ValidateAllUsersExistOrAreMapped(sourceWorkItems); ////////////////////////////////////////////////// contextLog.Information("Found target project as {@destProject}", Engine.Target.WorkItems.Project.Name); @@ -206,7 +190,7 @@ protected override void InternalExecute() "[FilterWorkItemsThatAlreadyExistInTarget] is enabled. Searching for work items that have already been migrated to the target...", sourceWorkItems.Count()); - string targetWIQLQuery = FixAreaPathAndIterationPathForTargetQuery(_config.WIQLQuery, + string targetWIQLQuery = _nodeStructureEnricher.FixAreaPathAndIterationPathForTargetQuery(_config.WIQLQuery, Engine.Source.WorkItems.Project.Name, Engine.Target.WorkItems.Project.Name, contextLog); sourceWorkItems = ((TfsWorkItemMigrationClient)Engine.Target.WorkItems).FilterExistingWorkItems( @@ -284,6 +268,44 @@ protected override void InternalExecute() } } + private void ValidateAllUsersExistOrAreMapped(List sourceWorkItems) + { + + contextLog.Information("Validating::Check that all users in the source exist in the target or are mapped!"); + List usersToMap = new List(); + usersToMap = _userMappingEnricher.GetUsersInSourceMappedToTargetForWorkItems(sourceWorkItems); + if (usersToMap.Count > 0) + { + Log.LogWarning("Validating Failed! There are {usersToMap} users that exist in the source that do not exist in the target. This will not cause any errors, but may result in disconnected users that could have been mapped. Use the ExportUsersForMapping processor to create a list of mappable users. Then Import using ", usersToMap.Count); + } + + } + + private void ValidateAllNodesExistOrAreMapped(List sourceWorkItems) + { + contextLog.Information("Validating::Check that all Area & Iteration paths from Source have a valid mapping on Target"); + List nodeStructureMissingItems = _nodeStructureEnricher.GetMissingRevisionNodes(sourceWorkItems); + if (_nodeStructureEnricher.ValidateTargetNodesExist(nodeStructureMissingItems)) + { + throw new Exception("Missing Iterations in Target preventing progress, check log for list. To continue you MUST configure IterationMaps or AreaMaps that matches the missing paths.."); + } + } + + private void ValidateAllWorkItemTypesHaveReflectedWorkItemIdField(List sourceWorkItems) + { + contextLog.Information("Validating::Check all Target Work Items have the RefectedWorkItemId field"); + + var result = _validateConfig.ValidatingRequiredField( + Engine.Target.Config.AsTeamProjectConfig().ReflectedWorkItemIDFieldName, sourceWorkItems); + if (!result) + { + var ex = new InvalidFieldValueException( + "Not all work items in scope contain a valid ReflectedWorkItemId Field!"); + Log.LogError(ex, "Not all work items in scope contain a valid ReflectedWorkItemId Field!"); + throw ex; + } + } + private void ValiddateWorkItemTypesExistInTarget(List sourceWorkItems) { contextLog.Information("Validating::Check that all work item types needed in the Target exist or are mapped"); @@ -318,7 +340,7 @@ private void ValiddateWorkItemTypesExistInTarget(List sourceWorkIt if (!allTypesMapped) { var ex = new Exception( - "Not all WorkItemTypes present in the Source are present in the Target or mapped!"); + "Not all WorkItemTypes present in the Source are present in the Target or mapped! Filter them from the query, or map the to target types."); Log.LogError(ex, "Not all WorkItemTypes present in the Source are present in the Target or mapped using `WorkItemTypeDefinition` in the config."); throw ex; } @@ -340,55 +362,6 @@ private void ValidatePatTokenRequirement() } } - internal string FixAreaPathAndIterationPathForTargetQuery(string sourceWIQLQuery, string sourceProject, string targetProject, ILogger? contextLog) - { - - string targetWIQLQuery = sourceWIQLQuery; - - if (string.IsNullOrWhiteSpace(targetWIQLQuery)) - { - return targetWIQLQuery; - } - - var matches = Regex.Matches(targetWIQLQuery, RegexPatternForAreaAndIterationPathsFix); - - - if (string.IsNullOrWhiteSpace(sourceProject) - || string.IsNullOrWhiteSpace(targetProject) - || sourceProject == targetProject) - { - return targetWIQLQuery; - } - - foreach (Match match in matches) - { - var value = match.Groups["value"].Value; - if (string.IsNullOrWhiteSpace(value) || !value.StartsWith(sourceProject)) - continue; - - var fieldType = match.Groups["key"].Value; - TfsNodeStructureType structureType; - switch (fieldType) - { - case "System.AreaPath": - structureType = TfsNodeStructureType.Area; - break; - case "System.IterationPath": - structureType = TfsNodeStructureType.Iteration; - break; - default: - throw new InvalidOperationException($"Field type {fieldType} is not supported for query remapping."); - } - - var remappedPath = _nodeStructureEnricher.GetNewNodeName(value, structureType); - targetWIQLQuery = targetWIQLQuery.Replace(value, remappedPath); - } - - contextLog?.Information("[FilterWorkItemsThatAlreadyExistInTarget] is enabled. Source project {sourceProject} is replaced with target project {targetProject} on the WIQLQueryBit which resulted into this target WIQLQueryBit \"{targetWIQLQueryBit}\" .", sourceProject, targetProject, targetWIQLQuery); - - return targetWIQLQuery; - } - private static bool IsNumeric(string val, NumberStyles numberStyle) { double result; @@ -472,6 +445,7 @@ private void PopulateWorkItem(WorkItemData oldWorkItemData, WorkItemData newWork foreach (Field f in oldWorkItem.Fields) { + _userMappingEnricher.MapUserIdentityField(f); if (newWorkItem.Fields.Contains(f.ReferenceName) == false) { var missedMigratedValue = oldWorkItem.Fields[f.ReferenceName].Value; @@ -570,7 +544,7 @@ private async Task ProcessWorkItemAsync(WorkItemData sourceWorkItem, int retryLi { "sourceWorkItemRev", sourceWorkItem.Rev }, { "ReplayRevisions", _revisionManager.Options.ReplayRevisions }} ); - List revisionsToMigrate = _revisionManager.GetRevisionsToMigrate(sourceWorkItem, targetWorkItem); + List revisionsToMigrate = _revisionManager.GetRevisionsToMigrate(sourceWorkItem.Revisions.Values.ToList(), targetWorkItem.Revisions.Values.ToList()); if (targetWorkItem == null) { targetWorkItem = ReplayRevisions(revisionsToMigrate, sourceWorkItem, null); @@ -674,10 +648,10 @@ private async Task ProcessWorkItemAsync(WorkItemData sourceWorkItem, int retryLi private void ProcessWorkItemAttachments(WorkItemData sourceWorkItem, WorkItemData targetWorkItem, bool save = true) { - if (targetWorkItem != null && _config.AttachmentMigration && sourceWorkItem.ToWorkItem().Attachments.Count > 0) + if (targetWorkItem != null && _attachmentEnricher.Options.Enabled && sourceWorkItem.ToWorkItem().Attachments.Count > 0) { - TraceWriteLine(LogEventLevel.Information, "Attachemnts {SourceWorkItemAttachmentCount} | LinkMigrator:{AttachmentMigration}", new Dictionary() { { "SourceWorkItemAttachmentCount", sourceWorkItem.ToWorkItem().Attachments.Count }, { "AttachmentMigration", _config.AttachmentMigration } }); - attachmentEnricher.ProcessAttachemnts(sourceWorkItem, targetWorkItem, save); + TraceWriteLine(LogEventLevel.Information, "Attachemnts {SourceWorkItemAttachmentCount} | LinkMigrator:{AttachmentMigration}", new Dictionary() { { "SourceWorkItemAttachmentCount", sourceWorkItem.ToWorkItem().Attachments.Count }, { "AttachmentMigration", _attachmentEnricher.Options.Enabled } }); + _attachmentEnricher.ProcessAttachemnts(sourceWorkItem, targetWorkItem, save); AddMetric("Attachments", processWorkItemMetrics, targetWorkItem.ToWorkItem().AttachedFileCount); } } @@ -882,7 +856,7 @@ private WorkItemData ReplayRevisions(List revisionsToMigrate, Work } targetWorkItem.SaveToAzureDevOps(); - attachmentEnricher.CleanUpAfterSave(); + _attachmentEnricher.CleanUpAfterSave(); TraceWriteLine(LogEventLevel.Information, "...Saved as {TargetWorkItemId}", new Dictionary { { "TargetWorkItemId", targetWorkItem.Id } }); } } diff --git a/src/VstsSyncMigrator.Core/ServiceCollectionExtensions.cs b/src/VstsSyncMigrator.Core/ServiceCollectionExtensions.cs index cc7a4aa5e..919a07dc4 100644 --- a/src/VstsSyncMigrator.Core/ServiceCollectionExtensions.cs +++ b/src/VstsSyncMigrator.Core/ServiceCollectionExtensions.cs @@ -17,7 +17,7 @@ public static void AddMigrationToolServicesForClientLegacyCore(this IServiceColl context.AddSingleton(); context.AddSingleton(); context.AddSingleton(); - context.AddSingleton(); + context.AddSingleton(); context.AddSingleton(); context.AddSingleton(); context.AddSingleton(); @@ -26,6 +26,7 @@ public static void AddMigrationToolServicesForClientLegacyCore(this IServiceColl context.AddSingleton(); context.AddSingleton(); context.AddSingleton(); + } } } \ No newline at end of file diff --git a/src/VstsSyncMigrator.Core/VstsSyncMigrator.Core.csproj b/src/VstsSyncMigrator.Core/VstsSyncMigrator.Core.csproj index e69f06e6f..5435e237f 100644 --- a/src/VstsSyncMigrator.Core/VstsSyncMigrator.Core.csproj +++ b/src/VstsSyncMigrator.Core/VstsSyncMigrator.Core.csproj @@ -22,8 +22,8 @@ - - + + all none contentFiles;analyzers From ea353de21e0caffea7ce2634b715aab50eb6ddb3 Mon Sep 17 00:00:00 2001 From: "Martin Hinshelwood nkdAgility.com" Date: Tue, 19 Mar 2024 14:35:13 +0000 Subject: [PATCH 2/2] [preview] Fix for `GetUsersInSourceMappedToTargetForWorkItems` being null (#1991) --- .../Execution/MigrationContext/WorkItemMigrationContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs b/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs index a1b988954..9a778c936 100644 --- a/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs +++ b/src/VstsSyncMigrator.Core/Execution/MigrationContext/WorkItemMigrationContext.cs @@ -274,7 +274,7 @@ private void ValidateAllUsersExistOrAreMapped(List sourceWorkItems contextLog.Information("Validating::Check that all users in the source exist in the target or are mapped!"); List usersToMap = new List(); usersToMap = _userMappingEnricher.GetUsersInSourceMappedToTargetForWorkItems(sourceWorkItems); - if (usersToMap.Count > 0) + if (usersToMap != null && usersToMap?.Count > 0) { Log.LogWarning("Validating Failed! There are {usersToMap} users that exist in the source that do not exist in the target. This will not cause any errors, but may result in disconnected users that could have been mapped. Use the ExportUsersForMapping processor to create a list of mappable users. Then Import using ", usersToMap.Count); }