Unable to perform Relocate with Network change from VSS to VDS

[samyama-digital]

Describe the bug

We are trying to perform cross vCenter Migration of workloads. During this, when we are trying to migrate workloads from standard switch to a different switch (in this case Distributed), I am getting below error.

Please note that, With the same user credentials, we are able to migrate the workloads from vCenter Web Client successfully. Only while trying to do it through pyvmomi, we are facing this problem.

"Permission to perform this operation was denied. NoPermission.message.format The virtual machine or template has a NIC connected to a network that does not exist at destination host and user does not have the Host.Config.Network privilege at destination host. Host.Config.Network privilege is required to create a network."

Please note that, for the same configuration, checkRelocate works successfully and does not give any error.

Network, that we are passing as destination is a VDS (Latest Version, something like NSX I believe) is available on the target. Object we are fetching

Code for Migration is below.

def migrate_vms(self, vm, validation):
    """
    this function relocates the listed VMs.. 
    assumes, source and target as defined here.
    We should have following attributes in the above list.
    every object in the list would be a JSON object containing...
    source : vm-id, name
    target : cluster, dataStore, resourcePool, vmFolder, network
    addl: source-offline (true/false), target_power_on (true/false),
    attributes for the target should at-least have cluster, datastore and network(need to check).
    Looks like, if cross vCenter migration, resourcePool need to be set.

    There is a parameter (check), which can check feasibility...if check=true, it just calls 
    migration feasibility and returns the result. if check is false, then we are migrating.

    """
    self.validation = validation
    vm_migrate_task_list = []
    retObj = {
        "task" : None,
        "msg" : ""
    }

    resourcePool = None
    dest_host = None
    vmFolder = None
    datastore = None
    network = None
    hosts = []
    vmObj = None
    nw = None

    if vm['cluster-id'] == "" or vm['cluster-id']=='null':
        retObj['task'] = None
        retObj['msg'] = "Invalid Cluster provided"
        return retObj
        #raise ConnectException("Please provide Target Cluster / LZ Details for Migraiton.")
        #exit()

    if vm['cluster-id'] != "":  #and vm['network'] != ""
        #now, based on cluster, get the cluster object.
        #get the default resource pool for the cluster,if resource pool is not specified.

        dest_host = None
        target_content = self.target_obj.si.RetrieveContent()
        source_content = self.si.RetrieveContent()

        target_hosts = self.propObj.findObjInList(self.target_obj.host_result,"cluster-id",vm['cluster-id'])

        for h in target_hosts:
            host = self.propObj.get_object(target_content, \
                        [vim.HostSystem],h['name'])
            if host:
                vmotion_enabled = host.summary.config.vmotionEnabled

                if host.summary.runtime.inMaintenanceMode or host.summary.runtime.connectionState != 'connected' \
                    or host.summary.runtime.inQuarantineMode or host.summary.runtime.powerState != 'poweredOn' \
                    or not host.summary.config.vmotionEnabled:
                    vmotion_enabled = False
                if host.summary.runtime.connectionState=="connected" and vmotion_enabled:
                    hosts.append(host)

        if len(hosts)>0:
            if len(hosts)==1:
                dest_host=self.propObj.get_object(target_content,[vim.HostSystem],hosts[0].name)
            else:
                rand_no = randrange(len(hosts))
                dest_host=self.propObj.get_object(target_content,[vim.HostSystem],hosts[rand_no].name)

        if dest_host is not None:
            if vm['datastore']!="":
                datastore = self.propObj.get_object(target_content,[vim.Datastore],vm['datastore'])

            if vm['resource-pool']!="":
                resourcePool = self.propObj.get_object(target_content,[vim.ResourcePool],vm['resource-pool'])
            else:
                managed_entity =  self.propObj.get_object(target_content,[vim.ManagedEntity],vm['cluster'])
                resourcePool = managed_entity.resourcePool

            if resourcePool is None:
                retObj['task'] = None
                retObj['msg'] = "No Default Resource Pools available"
                return retObj
                #raise ConnectException("Invalid Paramters. Working Resource Pool not found")

            if vm['folder']!="":
                vmFolder=self.propObj.get_object(target_content,[vim.Folder],vm['folder'])

            if vm['network']!="":
                    nw = self.propObj.get_object(target_content,[vim.Network],vm['network'])

            spec = vim.vm.RelocateSpec()

            if datastore is not None:
                #disks = self.propObj.collect_template_disks(vm)
                #spec.disk = self.propObj.construct_locator(disks,datastore)
                spec.datastore = datastore

            spec.host = dest_host
            #test = vim.vm.check.ProvisioningChecker(vm['vm-id'],self.source_obj.si._stub)
            if vmFolder is not None:
                spec.folder = vmFolder

            if resourcePool is not None:
                spec.pool = resourcePool

            vmObj= self.propObj.get_object(source_content,[vim.VirtualMachine],vm['vm-name'])

            if vmObj is None:
                retObj['task'] = None
                retObj['msg'] = "Invalid VM"
                return retObj
                #raise ConnectException("Unable to find the VM. Please retry")
                #return None

            if nw is not None:
                network_specs = []
                for nic in vmObj.config.hardware.device:
                    if isinstance(nic, vim.vm.device.VirtualEthernetCard):
                        network_spec = vim.vm.device.VirtualDeviceSpec()
                        network_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.edit
                        network_spec.device = nic
                        network_spec.device.backing = vim.vm.device.VirtualEthernetCard.NetworkBackingInfo()
                        network_spec.device.backing.network = nw
                        network_spec.device.backing.deviceName = nw.name

                        network_specs.append(network_spec)

                spec.deviceChange = network_specs   

            service = vim.ServiceLocator()
            credentials = vim.ServiceLocator.Credential()
            pwd = vim.ServiceLocator.NamePassword()

            pwd.username = self.target_obj.username
            pwd.password = self.target_obj.password
            credentials = pwd
            service.credential = credentials
            service.instanceUuid =  self.target_obj.siUUID
            service.url = "https://" + self.target_obj.host_name  #+ ":443/sdk"
            #dest_host = vim.HostSystem
            #tick = dest_host.AcquireCimServicesTicket()
            service.sslThumbprint = self.target_obj.thumbprint
            spec.service = service

            if self.validation:
                #perform feasibility checks
                #test = vim.vm.check.ProvisioningChecker(vm['vm-id'],self.source_obj.si._stub)
                #task = test.CheckRelocate(vmObj,spec)
                task = source_content.vmProvisioningChecker.CheckRelocate(vm=vmObj, spec=spec)
                retObj['task'] = task
            else:
                task = vmObj.Relocate(spec=spec)  
                if task:
                    while task.info.state=='queued':
                        pass 
                retObj['task'] = task          
        else:
            retObj['task'] = None
            retObj['msg'] = "No Hosts available"
        return retObj

Reproduction steps

None

Expected behavior

Should give us back the task object and should run the migration task at vCenter

Additional context

No response

[prziborowski]

                        network_spec.device.backing = vim.vm.device.VirtualEthernetCard.NetworkBackingInfo()
                        network_spec.device.backing.network = nw
                        network_spec.device.backing.deviceName = nw.name

I don't think this will give you a vDS. The network backing for vDS is vim.vm.device.VirtualEthernetCard.DistributedVirtualPortBackingInfo. And if you have a NSX-v, then I believe it is vim.vm.device.VirtualEthernetCard.OpaqueNetworkBackingInfo.

For vDS, you need to specify the port connection, which uses a number of fields of the vDS/virtual portgroup. For opaque network, somewhere there is a networkId and networkType that you'd get from the network.

[samyama-digital]

Is there a way for us find out what kind of switch by the object? for e.g. if I get the network object using vim.Network and identify if that object is standard Switch or Distributed or NSX-v?

[prziborowski]

You might be able to try:

if isinstance(nw, vim.dvs.DistributedVirtualPortgroup): # This is a vDS portgroup
    network_spec.device.backing = vim.vm.device.VirtualEthernetCard.DistributedVirtualPortBackingInfo(
        port= vim.dvs.PortConnection(
            portgroupKey=nw.key,
            switchUuid=nw.config.distributedVirtualSwitch.uuid,
        )
    )

elif isinstance(nw, vim.OpaqueNetwork): # This is an opaque NSX network
    network_spec.device.backing = vim.vm.device.VirtualEthernetCard.OpaqueNetworkBackingInfo(
        opaqueNetworkId=nw.summary.opaqueNetworkId,
        opaqueNetworkType=nw.summary.opaqueNetworkType)

[samyama-digital]

Thanks @prziborowski . This worked like Magic. Here is the complete code.

def migrate_vms(self, vm, validation):
    """
    this function relocates the listed VMs.. 
    assumes, source and target as defined here.
    We should have following attributes in the above list.
    every object in the list would be a JSON object containing...
    source : vm-id, name
    target : cluster, dataStore, resourcePool, vmFolder, network
    addl: source-offline (true/false), target_power_on (true/false),
    attributes for the target should at-least have cluster, datastore and network(need to check).
    Looks like, if cross vCenter migration, resourcePool need to be set.

    There is a parameter (check), which can check feasibility...if check=true, it just calls 
    migration feasibility and returns the result. if check is false, then we are migrating.

    """
    self.validation = validation
    vm_migrate_task_list = []
    retObj = {
        "task" : None,
        "msg" : ""
    }

    resourcePool = None
    dest_host = None
    vmFolder = None
    datastore = None
    network = None
    hosts = []
    vmObj = None
    nw = None

    if vm['cluster-id'] == "" or vm['cluster-id']=='null':
        retObj['task'] = None
        retObj['msg'] = "Invalid Cluster provided"
        return retObj
        #raise ConnectException("Please provide Target Cluster / LZ Details for Migraiton.")
        #exit()

    if vm['cluster-id'] != "":  #and vm['network'] != ""
        #now, based on cluster, get the cluster object.
        #get the default resource pool for the cluster,if resource pool is not specified.

        dest_host = None
        target_content = self.target_obj.si.RetrieveContent()
        source_content = self.si.RetrieveContent()

        target_hosts = self.propObj.findObjInList(self.target_obj.host_result,"cluster-id",vm['cluster-id'])

        for h in target_hosts:
            host = self.propObj.get_object(target_content, \
                        [vim.HostSystem],h['name'])
            if host:
                vmotion_enabled = host.summary.config.vmotionEnabled

                if host.summary.runtime.inMaintenanceMode or host.summary.runtime.connectionState != 'connected' \
                    or host.summary.runtime.inQuarantineMode or host.summary.runtime.powerState != 'poweredOn' \
                    or not host.summary.config.vmotionEnabled:
                    vmotion_enabled = False
                if host.summary.runtime.connectionState=="connected" and vmotion_enabled:
                    hosts.append(host)

        if len(hosts)>0:
            if len(hosts)==1:
                dest_host=self.propObj.get_object(target_content,[vim.HostSystem],hosts[0].name)
            else:
                rand_no = randrange(len(hosts))
                dest_host=self.propObj.get_object(target_content,[vim.HostSystem],hosts[rand_no].name)

        if dest_host is not None:
            if vm['datastore']!="":
                datastore = self.propObj.get_object(target_content,[vim.Datastore],vm['datastore'])

            if vm['resource-pool']!="":
                resourcePool = self.propObj.get_object(target_content,[vim.ResourcePool],vm['resource-pool'])
            else:
                managed_entity =  self.propObj.get_object(target_content,[vim.ManagedEntity],vm['cluster'])
                resourcePool = managed_entity.resourcePool

            if resourcePool is None:
                retObj['task'] = None
                retObj['msg'] = "No Default Resource Pools available"
                return retObj
                #raise ConnectException("Invalid Paramters. Working Resource Pool not found")

            if vm['folder']!="":
                vmFolder=self.propObj.get_object(target_content,[vim.Folder],vm['folder'])

            #Naveen Changes for PCE Compatibility
            switch_type = "NSX"
            if vm['network']!="":
                #we are checking for Switch Type...
                nw = self.propObj.get_object(target_content,[vim.Network],vm['network'])
                if nw is not None:
                    if not isinstance(nw,vim.OpaqueNetwork):
                        if hasattr(nw, "portKeys"):
                            switch_type = "VDS"
                        else:
                            switch_type = "VSS"

            spec = vim.vm.RelocateSpec()

            if datastore is not None:
                #disks = self.propObj.collect_template_disks(vm)
                #spec.disk = self.propObj.construct_locator(disks,datastore)
                spec.datastore = datastore

            spec.host = dest_host
            #test = vim.vm.check.ProvisioningChecker(vm['vm-id'],self.source_obj.si._stub)
            if vmFolder is not None:
                spec.folder = vmFolder

            if resourcePool is not None:
                spec.pool = resourcePool

            vmObj= self.propObj.get_object(source_content,[vim.VirtualMachine],vm['vm-name'])

            if vmObj is None:
                retObj['task'] = None
                retObj['msg'] = "Invalid VM"
                return retObj
                #raise ConnectException("Unable to find the VM. Please retry")
                #return None

            if nw is not None:
                network_specs = []
                #now we need to check if the target is VSS or VDS.
                #is_distributed_switch = nw.isPortGroupType(vim.dvs.PortGroupType.DISTRIBUTED)
                #is_distributed_switch = nw.isDistributedVirtualPortgroup()
                for nic in vmObj.config.hardware.device:
                    if isinstance(nic, vim.vm.device.VirtualEthernetCard):
                        network_spec = vim.vm.device.VirtualDeviceSpec()
                        network_spec.operation = vim.vm.device.VirtualDeviceSpec.Operation.edit
                        network_spec.device = nic
                        if switch_type=="NSX":
                            #if Opaque network.
                            network_spec.device.backing = vim.vm.device.VirtualEthernetCard.OpaqueNetworkBackingInfo()
                            network_spec.device.backing.opaqueNetworkType = nw.summary.opaqueNetworkType
                            network_spec.device.backing.opaqueNetworkId = nw.summary.opaqueNetworkId

                        elif switch_type=="VDS":
                            network_spec.device.backing = vim.vm.device.VirtualEthernetCard.DistributedVirtualPortBackingInfo()
                            dvs_port_connection = vim.dvs.PortConnection()
                            dvs_port_connection.portgroupKey = nw.key
                            dvs_port_connection.switchUuid = nw.config.distributedVirtualSwitch.uuid
                            network_spec.device.backing.port = dvs_port_connection        
                        else:
                            #if VSS
                            network_spec.device.backing = vim.vm.device.VirtualEthernetCard.NetworkBackingInfo()
                            network_spec.device.backing.network = nw
                            network_spec.device.backing.deviceName = vm['network']                    

                        network_specs.append(network_spec)

                spec.deviceChange = network_specs   

            service = vim.ServiceLocator()
            credentials = vim.ServiceLocator.Credential()
            pwd = vim.ServiceLocator.NamePassword()

            pwd.username = self.target_obj.username
            pwd.password = self.target_obj.password
            credentials = pwd
            service.credential = credentials
            service.instanceUuid =  self.target_obj.siUUID
            service.url = "https://" + self.target_obj.host_name  #+ ":443/sdk"
            #dest_host = vim.HostSystem
            #tick = dest_host.AcquireCimServicesTicket()
            service.sslThumbprint = self.target_obj.thumbprint
            spec.service = service

            if self.validation:
                #perform feasibility checks
                #test = vim.vm.check.ProvisioningChecker(vm['vm-id'],self.source_obj.si._stub)
                #task = test.CheckRelocate(vmObj,spec)
                task = source_content.vmProvisioningChecker.CheckRelocate(vm=vmObj, spec=spec)
                retObj['task'] = task
            else:
                task = vmObj.Relocate(spec=spec)  
                if task:
                    while task.info.state=='queued':
                        pass 
                retObj['task'] = task          
        else:
            retObj['task'] = None
            retObj['msg'] = "No Hosts available"
        return retObj