search

nkdAgility / azure-devops-migration-tools

Azure DevOps Migration Tools allow you to migrate Teams, Backlogs, Tasks, Test Cases, and Plans & Suits from one Project to another in Azure DevOps / TFS both within the same Organisation, and between Organisations.
https://nkdagility.com/learn/azure-devops-migration-tools/
MIT License
493 stars 328 forks source link

Error running query Microsoft.TeamFoundation.WorkItemTracking.Client.ValidationException: Empty identifier is not allowed. The error is caused by «[]». #1648

Closed HFSmanderson closed 11 months ago

HFSmanderson commented 1 year ago

Version

Source Version

Azure DevOps Service

Target Version

Azure DevOps Service

Relevant configuration

{
  "ChangeSetMappingFile": null,
  "Source": {
    "$type": "TfsTeamProjectConfig",
    "Collection": "/",
    "Project": "HFSIngestionService",
    "ReflectedWorkItemIDFieldName": "AzureDevOps",
    "AllowCrossProjectLinking": false,
    "AuthenticationMode": "Prompt",
    "PersonalAccessToken": "",
    "PersonalAccessTokenVariableName": "migration",
    "LanguageMaps": {
      "AreaPath": "Area",
      "IterationPath": "Iteration"
    }
  },
  "Target": {
    "$type": "TfsTeamProjectConfig",
    "Collection": "",
    "Project": "NewHFSIngestionEngine",
    "Custom.ReflectedWorkItemIDFieldName": "AzureDevOps",
    "AllowCrossProjectLinking": false,
    "AuthenticationMode": "Prompt",
    "PersonalAccessToken": "",
    "PersonalAccessTokenVariableName": "migration",
    "LanguageMaps": {
      "AreaPath": "Area",
      "IterationPath": "Iteration"
    }
  },
  "FieldMaps": [
    {
      "$type": "MultiValueConditionalMapConfig",
      "WorkItemTypeName": "*",
      "sourceFieldsAndValues": {
        "Field1": "Value1",
        "Field2": "Value2"
      },
      "targetFieldsAndValues": {
        "Field1": "Value1",
        "Field2": "Value2"
      }
    },
    {
      "$type": "FieldSkipMapConfig",
      "WorkItemTypeName": "*",
      "targetField": "Custom.ReflectedWorkItemId"
    },
    {
      "$type": "FieldValueMapConfig",
      "WorkItemTypeName": "*",
      "sourceField": "System.State",
      "targetField": "System.State",
      "defaultValue": "New",
      "valueMapping": {
        "Approved": "New",
        "New": "New",
        "Committed": "Active",
        "In Progress": "Active",
        "To Do": "New",
        "Done": "Closed",
        "Removed": "Removed"
      }
    },
    {
      "$type": "FieldtoFieldMapConfig",
      "WorkItemTypeName": "*",
      "sourceField": "Microsoft.VSTS.Common.BacklogPriority",
      "targetField": "Microsoft.VSTS.Common.StackRank",
      "defaultValue": null
    },
    {
      "$type": "FieldtoFieldMultiMapConfig",
      "WorkItemTypeName": "*",
      "SourceToTargetMappings": {
        "SourceField1": "TargetField1",
        "SourceField2": "TargetField2"
      }
    },
    {
      "$type": "FieldtoTagMapConfig",
      "WorkItemTypeName": "*",
      "sourceField": "System.State",
      "formatExpression": "ScrumState:{0}"
    },
    {
      "$type": "FieldMergeMapConfig",
      "WorkItemTypeName": "*",
      "sourceFields": [
        "System.Description",
        "Microsoft.VSTS.Common.AcceptanceCriteria"
      ],
      "targetField": "System.Description",
      "formatExpression": "{0} <br/><br/><h3>Acceptance Criteria</h3>{1}"
    },
    {
      "$type": "RegexFieldMapConfig",
      "WorkItemTypeName": "*",
      "sourceField": "COMPANY.PRODUCT.Release",
      "targetField": "COMPANY.DEVISION.MinorReleaseVersion",
      "pattern": "PRODUCT \\d{4}.(\\d{1})",
      "replacement": "$1"
    },
    {
      "$type": "FieldValuetoTagMapConfig",
      "WorkItemTypeName": "*",
      "sourceField": "Microsoft.VSTS.CMMI.Blocked",
      "pattern": "Yes",
      "formatExpression": "{0}"
    },
    {
      "$type": "TreeToTagMapConfig",
      "WorkItemTypeName": "*",
      "toSkip": 3,
      "timeTravel": 1
    }
  ],
  "GitRepoMapping": null,
  "LogLevel": "Information",
  "CommonEnrichersConfig": null,
  "Processors": [
    {
      "$type": "WorkItemMigrationConfig",
      "Enabled": true,
      "ReplayRevisions": true,
      "PrefixProjectToNodes": false,
      "UpdateCreatedDate": true,
      "UpdateCreatedBy": true,
      "WIQLQueryBit": " AND [System.WorkItemType] NOT IN ('Shared Steps','Shared Parameter','Feedback Request')",
      "WIQLOrderBit": "[System.ChangedDate] desc",
      "LinkMigration": true,
      "AttachmentMigration": true,
      "AttachmentWorkingPath": "c:\\temp\\WorkItemAttachmentWorkingFolder\\",
      "FixHtmlAttachmentLinks": false,
      "SkipToFinalRevisedWorkItemType": true,
      "WorkItemCreateRetryLimit":5,
      "FilterWorkItemsThatAlreadyExistInTarget": true,
      "PauseAfterEachWorkItem": false,
      "AttachmentMaxSize": 480000000,
      "AttachRevisionHistory": false,
      "LinkMigrationSaveEachAsAdded": false,
      "GenerateMigrationComment": true,
      "WorkItemIDs": null,
      "MaxRevisions": 0,
      "UseCommonNodeStructureEnricherConfig": false,
      "StopMigrationOnMissingAreaIterationNodes": true,
      "NodeBasePaths": [

      ],
      "AreaMaps": {},
      "IterationMaps": {},
      "MaxGracefulFailures": 0,
      "SkipRevisionWithInvalidIterationPath": false,
      "SkipRevisionWithInvalidAreaPath": false
    }
  ],
  "Version": "13.1",
  "workaroundForQuerySOAPBugEnabled": false,
  "WorkItemTypeDefinition": {
    "sourceWorkItemTypeName": "targetWorkItemTypeName"
  },
  "Endpoints": {
    "InMemoryWorkItemEndpoints": [
      {
        "Name": "Source",
        "EndpointEnrichers": null
      },
      {
        "Name": "Target",
        "EndpointEnrichers": null
      }
    ]
  }
}

Relevant log output

14:47:24 ERR] WorkItemMigration The Processor MigrationEngine entered the failed state...stopping run
[14:47:24 INF] Application is shutting down...
[14:47:24 INF] Application Ending
[14:47:24 INF] The application ran in 00:00:39.7018850 and finished at 09/13/2023 14:47:24
PS C:\tools\migrationtools> .\migration.exe execute --config .\configuration.json
[14:54:44 INF] Application Starting
[14:54:44 INF]                                       &@&
[14:54:44 INF]                                    @@(((((@
[14:54:44 INF]                                   @(((((((((@
[14:54:44 INF]                                 @(((((((((((((&
[14:54:44 INF]                               ##((((((@ @((((((@@
[14:54:44 INF]                              @((((((@     @((((((&
[14:54:44 INF]                             @(((((#        @((((((@
[14:54:44 INF]                            &(((((&           &(((((@
[14:54:44 INF]                           @(((((&             &(((((@
[14:54:44 INF]                           &(((((@#&@((.((&@@@(#(((((@
[14:54:44 INF]                          #((((#..................#@((&
[14:54:44 INF]                        &@(((((&......................(@
[14:54:44 INF]                      @.(&((((&...&&        &@&..........&@
[14:54:44 INF]                    @...@(((((@                   @#.......((
[14:54:44 INF]                  &.....@(((((@                   @((@.......&
[14:54:44 INF]                 @......@(((((                    #((((&.......&
[14:54:44 INF]                #.....( &(((((         @@@        ((((((@@......@
[14:54:44 INF]               &.....@  @(((&@@#(((((((((((((((((#@(((((&  ......@
[14:54:44 INF]              @.....@  &@&((((((((((((((((((((((((@(((((@#  ......@
[14:54:44 INF]             @.....&@(((((((((((((((&&@@@@@(((((@((((#(((#@(....&
[14:54:44 INF]             @.....&((((((((&@@&                 @(((((@(((((((@...#
[14:54:44 INF]             &....((((((@@(((((@                &@(((((@&((((((((#&&
[14:54:44 INF]            @(....&((@    @(((((@               @(((((@    @(((((((##
[14:54:44 INF]          @(#(....&        &(((((@             @(((((&       &@(((((((&
[14:54:44 INF]        &@(((&.....        @((((((&           @(((((       &.(&((((((@
[14:54:44 INF]       @(((((@.....&        (((((@        &@(((((&         @....@((((((@
[14:54:44 INF]      @(((((#@.....(          &(((((@&     ##(((((&         @.....@@((((((@
[14:54:44 INF]    (&(((((@  &.....@&         @((((((@   @((((((@         @......   @(((((@
[14:54:44 INF]    &(((((@    @.....#&         @#((((((@((((((#          @......&    @(((((@
[14:54:44 INF]   @(((((@      &......&          @(((((((@#((@         &@......       @(((((@
[14:54:44 INF]  @(((((@        @......@&        @@@(((((((&@&        @......(         #(((((@
[14:54:44 INF]  #((((&           &.......@  &@&(((((@#((((((((@@& &@.......@          ((((&
[14:54:44 INF] &(((((@@           @(....&@#((((((((((@ @(((((((#@........@            &@(((((@
[14:54:44 INF] &(((((((((((((((((((((((((((((((((&@@@@@@@@@&...........@(((((((((((((((((((((@
[14:54:44 INF] @(((((((((((((((((((((((((((((&@(....................@#((((((((((((((((((((((#@
[14:54:44 INF]       @((((((((((((((&@&  &&...................@   @@#((((((((((((((#@@
[14:54:44 INF]
[14:54:44 INF] ===============================================================================
[14:54:44 INF] Azure DevOps Migration Tools [Object Model]
[14:54:44 INF] v13.1.1.0
[14:54:44 INF] naked Agility with Martin Hinshelwood
[14:54:44 INF] ===============================================================================
[14:54:44 INF] Telemetry Note: We use Application Insights to collect telemetry on performance & feature usage for the tools to help our developers target features. This data is tied to a session ID that is generated and shown in the logs. This can help with debugging.
[14:54:44 INF] Start Time: 09/13/2023 14:54:44
[14:54:44 INF] Running with args: execute
[14:54:44 INF] OSVersion: Microsoft Windows NT 6.2.9200.0
[14:54:44 INF] Version: 13.1.1.0
[14:54:45 INF] Latest version detected as 13.1.1.0
[14:54:45 INF] Application started. Press Ctrl+C to shut down.
[14:54:45 INF] Hosting environment: Production
[14:54:45 INF] Content root path: C:\tools\migrationtools
[14:54:45 INF] Press Ctrl+C to shut down.
[14:54:45 INF] Config Found, creating engine host
[14:54:45 INF] Creating Migration Engine 
[14:54:45 INF] ProcessorContainer: Of 1 configured Processors only 1 are enabled
[14:54:45 INF] ProcessorContainer: Adding Processor WorkItemMigrationContext
[14:54:45 INF] Logging has been configured and is set to: Information.
[14:54:45 INF]                               Max Logfile: Verbose.
[14:54:45 INF]                               Max Console: Debug.
[14:54:45 INF]                  Max Application Insights: Error.
[14:54:45 INF] The Max log levels above show where to go look for extra info. e.g. Even if you set the log level to Verbose you will only see that info in the Log File, however everything up to Debug will be in the Console.
[14:54:45 INF] FieldMapContainer: Adding FieldMap MultiValueConditionalMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap FieldSkipMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap FieldValueMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap FieldToFieldMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap FieldtoFieldMultiMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap FieldToTagFieldMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap FieldMergeMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap RegexFieldMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap FieldValuetoTagMap for *
[14:54:45 INF] FieldMapContainer: Adding FieldMap TreeToTagFieldMap for *
[14:54:45 INF] Beginning run of 1 processors
[14:54:45 INF] Processor: WorkItemMigration
[14:54:45 INF] Migration Context Start: WorkItemMigration
[14:54:45 INF] WorkItemMigrationContext::InternalExecute
[14:54:45 INF] TfsMigrationClient::GetDependantTfsCollection:AuthenticationMode(Prompt)
[14:54:45 INF] TfsMigrationClient::GetDependantTfsCollection: Prompting for credentials
[14:54:45 INF] MigrationClient: Connecting to 
[14:54:45 INF] MigrationClient: validating security for {"IsContainer": false, "UniqueName": "", "Descriptor": {"Data": null, "Identifier": "", "IdentityType": "Microsoft.IdentityModel.Claims.ClaimsIdentity", "$type": "IdentityDescriptor"}, "DisplayName": "Mark Anderson", "IsActive": true, "MemberOf": [], "Members": [], "TeamFoundationId": , "UniqueUserId": 0, "$type": "TeamFoundationIdentity"}
[14:54:45 INF] MigrationClient: Access granted to / for 
[14:54:45 INF] TfsMigrationClient::GetDependantTfsCollection:AuthenticationMode(Prompt)
[14:54:45 INF] TfsMigrationClient::GetDependantTfsCollection: Prompting for credentials
[14:54:45 INF] MigrationClient: Connecting to 
[14:54:46 INF] MigrationClient: validating security for {"IsContainer": false, "UniqueName": "manderson@healthfundsolutions.com", "Descriptor": {"Data": null, "Identifier": "", "IdentityType": "Microsoft.IdentityModel.Claims.ClaimsIdentity", "$type": "IdentityDescriptor"}, "DisplayName": "", "IsActive": true, "MemberOf": [], "Members": [], "TeamFoundationId": "", "UniqueUserId": 0, "$type": "TeamFoundationIdentity"}
[14:54:46 INF] MigrationClient: Access granted to  for m)
[14:54:47 INF] Migrating all Nodes before the Processor run.
[14:54:48 INF]  Processing Node: \NewHFSIngestionEngine\Iteration\Sprint Zero, start date: 12/22/2022 00:00:00, finish date: 12/30/2022 00:00:00
[14:54:48 INF]  Processing Node: \NewHFSIngestionEngine\Iteration\Sprint 1, start date: 01/03/2023 00:00:00, finish date: 01/12/2023 00:00:00
[14:54:48 INF]  Processing Node: \NewHFSIngestionEngine\Iteration\Sprint 2, start date: 01/13/2023 00:00:00, finish date: 01/26/2023 00:00:00
[14:54:48 INF]  Processing Node: \NewHFSIngestionEngine\Iteration\Sprint 3, start date: 01/27/2023 00:00:00, finish date: 02/09/2023 00:00:00
[14:54:48 INF]  Processing Node: \NewHFSIngestionEngine\Iteration\Sprint 4, start date: 02/10/2023 00:00:00, finish date: 02/23/2023 00:00:00
[14:54:49 INF]  Processing Node: \NewHFSIngestionEngine\Iteration\Sprint 5, start date: 02/24/2023 00:00:00, finish date: 03/09/2023 00:00:00
[14:54:49 INF]  Processing Node: \NewHFSIngestionEngine\Iteration\Sprint 6, start date: 03/10/2023 00:00:00, finish date: 03/23/2023 00:00:00
[14:54:49 INF] Querying items to be migrated: SELECT [System.Id], [System.Tags] FROM WorkItems WHERE [System.TeamProject] = @TeamProject  AND [System.WorkItemType] NOT IN ('Shared Steps','Shared Parameter','Feedback Request') ORDER BY [System.ChangedDate] desc ...
[14:54:50 INF] 719 Work items received, verifying
[14:54:52 INF] Loading 719 Work Items
[14:55:22 INF] 234/719 32.55%
[14:55:52 INF] 510/719 70.93%
[14:56:15 INF] Replay all revisions of 719 work items?
[14:56:15 INF] ValidateTargetNodesExist::Checking all Nodes on Work items
[14:56:15 INF] Found target project as NewHFSIngestionEngine
[14:56:15 INF] [FilterWorkItemsThatAlreadyExistInTarget] is enabled. Searching for work items that have already been migrated to the target...
[14:56:15 INF] [FilterWorkItemsThatAlreadyExistInTarget] is enabled. Source project HFSIngestionService is replaced with target project NewHFSIngestionEngine on the WIQLQueryBit which resulted into this target WIQLQueryBit " AND [System.WorkItemType] NOT IN ('Shared Steps','Shared Parameter','Feedback Request')" .
[14:56:15 ERR]  Error running query
Microsoft.TeamFoundation.WorkItemTracking.Client.ValidationException: Empty identifier is not allowed. The error is caused by «[]».
   at Microsoft.TeamFoundation.WorkItemTracking.Client.Query.Initialize(WorkItemStore store, String wiql, IDictionary context, Int32[] ids, Int32[] revs, Boolean dayPrecision)
   at Microsoft.TeamFoundation.WorkItemTracking.Client.WorkItemStore.Query(String wiql, IDictionary context)
   at MigrationTools._EngineV1.Clients.TfsWorkItemQuery.GetWorkItemsFromQuery(TfsWorkItemMigrationClient wiClient) in D:\a\1\s\src\MigrationTools.Clients.AzureDevops.ObjectModel\_EngineV1\Clients\TfsWorkItemQuery.cs:line 51
[14:56:15 INF] DONE in 00:01:26.0322723
[14:56:15 FTL] Error while running WorkItemMigration
Microsoft.TeamFoundation.WorkItemTracking.Client.ValidationException: Empty identifier is not allowed. The error is caused by «[]».
   at Microsoft.TeamFoundation.WorkItemTracking.Client.Query.Initialize(WorkItemStore store, String wiql, IDictionary context, Int32[] ids, Int32[] revs, Boolean dayPrecision)
   at Microsoft.TeamFoundation.WorkItemTracking.Client.WorkItemStore.Query(String wiql, IDictionary context)
   at MigrationTools._EngineV1.Clients.TfsWorkItemQuery.GetWorkItemsFromQuery(TfsWorkItemMigrationClient wiClient) in D:\a\1\s\src\MigrationTools.Clients.AzureDevops.ObjectModel\_EngineV1\Clients\TfsWorkItemQuery.cs:line 84
   at MigrationTools._EngineV1.Clients.TfsWorkItemQuery.GetInternalWorkItems() in D:\a\1\s\src\MigrationTools.Clients.AzureDevops.ObjectModel\_EngineV1\Clients\TfsWorkItemQuery.cs:line 41
   at MigrationTools._EngineV1.Clients.TfsWorkItemQuery.GetWorkItems() in D:\a\1\s\src\MigrationTools.Clients.AzureDevops.ObjectModel\_EngineV1\Clients\TfsWorkItemQuery.cs:line 26
   at MigrationTools._EngineV1.Clients.TfsWorkItemMigrationClient.FilterExistingWorkItems(List`1 sourceWorkItems, TfsWiqlDefinition wiqlDefinition, TfsWorkItemMigrationClient sourceWorkItemMigrationClient) in D:\a\1\s\src\MigrationTools.Clients.AzureDevops.ObjectModel\_EngineV1\Clients\TfsWorkItemMigrationClient.cs:line 48
   at VstsSyncMigrator.Engine.WorkItemMigrationContext.InternalExecute() in D:\a\1\s\src\VstsSyncMigrator.Core\Execution\MigrationContext\WorkItemMigrationContext.cs:line 190
   at MigrationTools._EngineV1.Processors.MigrationProcessorBase.Execute() in D:\a\1\s\src\MigrationTools\_EngineV1\Processors\MigrationProcessorBase.cs:line 47
[14:56:15 ERR] WorkItemMigration The Processor MigrationEngine entered the failed state...stopping run
[14:56:15 INF] Application is shutting down...
[14:56:15 INF] Application Ending
[14:56:15 INF] The application ran in 00:01:31.1373789 and finished at 09/13/2023 14:56:15
PS C:\tools\migrationtools>

What happened?

I am migrating an existing project in DevOps to a new Organization Project. We are receiving an Error while running WorkItemMigration Microsoft.TeamFoundation.WorkItemTracking.Client.ValidationException: Empty identifier is not allowed. The error is caused by «[]»

We are able to run the first select query from the source but the error happens on second query for the targe.

Debug in Visual Studio

MrHinsh commented 1 year ago

Could be the extra space in the query!

"WIQLQueryBit": " AND [System.WorkItemType] NOT IN ('Shared Steps','Shared Parameter','Feedback Request')",

instead of

"WIQLQueryBit": "AND [System.WorkItemType] NOT IN ('Shared Steps','Shared Parameter','Feedback Request')",
github-actions[bot] commented 11 months ago

This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 10 days