`test_hdinsight_application` live test fails: User SubscriptionId 'xxx' does not have cores left to create resource 'hdicli-zoxhlim5v'. Required: 40, Available: 20.

[jiasli]

________________________________ HDInsightClusterTests.test_hdinsight_application ________________________________
self = <azure.cli.testsdk.base.ExecutionResult object at 0x7fb7d7652410>
cli_ctx = <azure.cli.core.mock.DummyCli object at 0x7fb7da78a950>
command = 'hdinsight create -n hdicli-zoxhlim5v -g hdicli-is6rq -l southcentralus -p Password1! -t spark --no-validation-timeout --storage-account hdicli66pocjqp2sdo7bk2xs --storage-container default --ssh-user sshuser --ssh-password Password1!'
expect_failure = False

    def _in_process_execute(self, cli_ctx, command, expect_failure=False):
        from io import StringIO
        from vcr.errors import CannotOverwriteExistingCassetteException

        if command.startswith('az '):
            command = command[3:]

        stdout_buf = StringIO()
        logging_buf = StringIO()
        try:
            # issue: stderr cannot be redirect in this form, as a result some failure information
            # is lost when command fails.
>           self.exit_code = cli_ctx.invoke(shlex.split(command), out_file=stdout_buf) or 0

src/azure-cli-testsdk/azure/cli/testsdk/base.py:302:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../py310/lib/python3.10/site-packages/knack/cli.py:245: in invoke
    exit_code = self.exception_handler(ex)
src/azure-cli-core/azure/cli/core/__init__.py:127: in exception_handler
    return handle_exception(ex)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

ex = HttpResponseError("(BadRequest) User SubscriptionId '0b1f6471-1bf0-4dda-aec3-cb9272f09590' does not have cores left to...-4dda-aec3-cb9272f09590' does not have cores left to create resource 'hdicli-zoxhlim5v'. Required: 40, Available: 20.")
args = (), kwargs = {}

    def _handle_main_exception(ex, *args, **kwargs):  # pylint: disable=unused-argument
        if isinstance(ex, CannotOverwriteExistingCassetteException):
            # This exception usually caused by a no match HTTP request. This is a product error
            # that is caused by change of SDK invocation.
            raise ex

>       raise CliExecutionError(ex)
E       azure.cli.testsdk.exceptions.CliExecutionError: The CLI throws exception HttpResponseError during execution and fails the command.

src/azure-cli-testsdk/azure/cli/testsdk/patches.py:35: CliExecutionError

During handling of the above exception, another exception occurred:

self = <azure.cli.command_modules.hdinsight.tests.latest.test_hdinsight_commands.HDInsightClusterTests testMethod=test_hdinsight_application>
storage_account_info = ('hdicli66pocjqp2sdo7bk2xs', 'poNtnKliTIiWJ7qNksIPehVRWRxXRm0zC0Y5bASrUy5sInt0QiZkmE/yuUWYXICCX1k/fRm2oi7G+AStLichyQ==\n')

    @ResourceGroupPreparer(name_prefix='hdicli-', location=location, random_name_length=12)
    @StorageAccountPreparer(name_prefix='hdicli', location=location, parameter_name='storage_account')
    def test_hdinsight_application(self, storage_account_info):
>       self._create_hdinsight_cluster(
            HDInsightClusterTests._wasb_arguments(storage_account_info),
            HDInsightClusterTests._with_explicit_ssh_creds()
        )

src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py:222:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
src/azure-cli/azure/cli/command_modules/hdinsight/tests/latest/test_hdinsight_commands.py:577: in _create_hdinsight_cluster
    self.cmd(create_cluster_format, checks=[
src/azure-cli-testsdk/azure/cli/testsdk/base.py:176: in cmd
    return execute(self.cli_ctx, command, expect_failure=expect_failure).assert_with_checks(checks)
src/azure-cli-testsdk/azure/cli/testsdk/base.py:251: in __init__
    self._in_process_execute(cli_ctx, command, expect_failure=expect_failure)
src/azure-cli-testsdk/azure/cli/testsdk/base.py:314: in _in_process_execute
    raise ex.exception
../py310/lib/python3.10/site-packages/knack/cli.py:233: in invoke
    cmd_result = self.invocation.execute(args)
src/azure-cli-core/azure/cli/core/commands/__init__.py:664: in execute
    raise ex
src/azure-cli-core/azure/cli/core/commands/__init__.py:731: in _run_jobs_serially
    results.append(self._run_job(expanded_arg, cmd_copy))
src/azure-cli-core/azure/cli/core/commands/__init__.py:701: in _run_job
    result = cmd_copy(params)
src/azure-cli-core/azure/cli/core/commands/__init__.py:334: in __call__
    return self.handler(*args, **kwargs)
src/azure-cli-core/azure/cli/core/commands/command_operation.py:121: in handler
    return op(**command_args)
src/azure-cli/azure/cli/command_modules/hdinsight/custom.py:402: in create_cluster
    return sdk_no_wait(no_wait, client.begin_create, resource_group_name, cluster_name, create_params)
src/azure-cli-core/azure/cli/core/util.py:710: in sdk_no_wait
    return func(*args, **kwargs)
../py310/lib/python3.10/site-packages/azure/mgmt/hdinsight/operations/_clusters_operations.py:139: in begin_create
    raw_result = self._create_initial(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <azure.mgmt.hdinsight.operations._clusters_operations.ClustersOperations object at 0x7fb7d75d8610>
resource_group_name = 'hdicli-is6rq', cluster_name = 'hdicli-zoxhlim5v'
parameters = <azure.mgmt.hdinsight.models._models_py3.ClusterCreateParametersExtended object at 0x7fb7d75e6e60>
kwargs = {}, cls = <function ClustersOperations.begin_create.<locals>.<lambda> at 0x7fb7d75b9c60>
error_map = {401: <class 'azure.core.exceptions.ClientAuthenticationError'>, 404: <class 'azure.core.exceptions.ResourceNotFoundError'>, 409: <class 'azure.core.exceptions.ResourceExistsError'>}
api_version = '2021-06-01', content_type = 'application/json', accept = 'application/json'
url = 'https://management.azure.com/subscriptions/0b1f6471-1bf0-4dda-aec3-cb9272f09590/resourceGroups/hdicli-is6rq/providers/Microsoft.HDInsight/clusters/hdicli-zoxhlim5v'
path_format_arguments = {'clusterName': 'hdicli-zoxhlim5v', 'resourceGroupName': 'hdicli-is6rq', 'subscriptionId': '0b1f6471-1bf0-4dda-aec3-cb9272f09590'}

    def _create_initial(
        self,
        resource_group_name,  # type: str
        cluster_name,  # type: str
        parameters,  # type: "_models.ClusterCreateParametersExtended"
        **kwargs  # type: Any
    ):
        # type: (...) -> "_models.Cluster"
        cls = kwargs.pop('cls', None)  # type: ClsType["_models.Cluster"]
        error_map = {
            401: ClientAuthenticationError, 404: ResourceNotFoundError, 409: ResourceExistsError
        }
        error_map.update(kwargs.pop('error_map', {}))
        api_version = "2021-06-01"
        content_type = kwargs.pop("content_type", "application/json")
        accept = "application/json"

        # Construct URL
        url = self._create_initial.metadata['url']  # type: ignore
        path_format_arguments = {
            'subscriptionId': self._serialize.url("self._config.subscription_id", self._config.subscription_id, 'str'),
            'resourceGroupName': self._serialize.url("resource_group_name", resource_group_name, 'str'),
            'clusterName': self._serialize.url("cluster_name", cluster_name, 'str'),
        }
        url = self._client.format_url(url, **path_format_arguments)

        # Construct parameters
        query_parameters = {}  # type: Dict[str, Any]
        query_parameters['api-version'] = self._serialize.query("api_version", api_version, 'str')

        # Construct headers
        header_parameters = {}  # type: Dict[str, Any]
        header_parameters['Content-Type'] = self._serialize.header("content_type", content_type, 'str')
        header_parameters['Accept'] = self._serialize.header("accept", accept, 'str')

        body_content_kwargs = {}  # type: Dict[str, Any]
        body_content = self._serialize.body(parameters, 'ClusterCreateParametersExtended')
        body_content_kwargs['content'] = body_content
        request = self._client.put(url, query_parameters, header_parameters, **body_content_kwargs)
        pipeline_response = self._client._pipeline.run(request, stream=False, **kwargs)
        response = pipeline_response.http_response

        if response.status_code not in [200]:
            map_error(status_code=response.status_code, response=response, error_map=error_map)
            error = self._deserialize.failsafe_deserialize(_models.ErrorResponse, response)
>           raise HttpResponseError(response=response, model=error, error_format=ARMErrorFormat)
E           azure.core.exceptions.HttpResponseError: (BadRequest) User SubscriptionId '0b1f6471-1bf0-4dda-aec3-cb9272f09590' does not have cores left to create resource 'hdicli-zoxhlim5v'. Required: 40, Available: 20.
E           Code: BadRequest
E           Message: User SubscriptionId '0b1f6471-1bf0-4dda-aec3-cb9272f09590' does not have cores left to create resource 'hdicli-zoxhlim5v'. Required: 40, Available: 20.

../py310/lib/python3.10/site-packages/azure/mgmt/hdinsight/operations/_clusters_operations.py:95: HttpResponseError

[yonzhan]

Thank you for opening this issue, we will look into it.

[jiasli]

According to Azure Portal, we do have enough cores left in southcentralus:

[jiasli]

test_hdinsight_cluster_kafka_with_rest_proxy fails with the same error:

        if response.status_code not in [200]:
            map_error(status_code=response.status_code, response=response, error_map=error_map)
            error = self._deserialize.failsafe_deserialize(_models.ErrorResponse, response)
>           raise HttpResponseError(response=response, model=error, error_format=ARMErrorFormat)
E           azure.core.exceptions.HttpResponseError: (BadRequest) User SubscriptionId '0b1f6471-1bf0-4dda-aec3-cb9272f09590' does not have cores left to create resource 'hdicli-zrdqatctc'. Required: 36, Available: 24.
E           Code: BadRequest
E           Message: User SubscriptionId '0b1f6471-1bf0-4dda-aec3-cb9272f09590' does not have cores left to create resource 'hdicli-zrdqatctc'. Required: 36, Available: 24.

../py310/lib/python3.10/site-packages/azure/mgmt/hdinsight/operations/_clusters_operations.py:95: HttpResponseError