dbx deploy not working for single node clusters

[chinwobble]

When trying to run dbx deploy on the following job its fails.

{
                "name": "anabricks_cd-sample",
                "new_cluster": {
                    "spark_version": "7.3.x-scala2.12",
                    "spark_conf": {
                        "spark.databricks.delta.preview.enabled": "true",
                        "spark.master": "local[*]",
                        "spark.databricks.cluster.profile": "singleNode"
                    },
                    "node_type_id": "Standard_DS3_v2",
                    "custom_tags": {
                        "ResourceClass": "SingleNode",
                        "job": "anabricks_sample"
                    },
                    "enable_elastic_disk": true,
                    "init_scripts": [
                        {
                            "dbfs": {
                                "destination": "dbfs:/monitoring/datadog_install_driver_only.sh"
                            }
                        }
                    ],
                    "azure_attributes": {
                        "availability": "ON_DEMAND_AZURE"
                    },
                    "num_workers": 0
                },
                "libraries": [],
                "email_notifications": {
                    "on_start": [],
                    "on_success": [],
                    "on_failure": []
                },
                "max_retries": 0,
                "spark_python_task": {
                    "python_file": "anabricks_cd/jobs/sample/entrypoint.py",
                    "parameters": [
                        "--conf-file",
                        "conf/test/sample.json"
                    ]
                }
            }

The error is

Traceback (most recent call last):
  File "c:\tools\miniconda3\envs\risbricks\lib\runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "c:\tools\miniconda3\envs\risbricks\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "C:\tools\miniconda3\envs\risbricks\Scripts\dbx.exe\__main__.py", line 7, in <module>
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\click\core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\click\core.py", line 782, in main
    rv = self.invoke(ctx)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\click\core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\click\core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\click\core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\dbx\commands\deploy.py", line 94, in deploy
    requirements_payload, package_requirement, _file_uploader)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\dbx\commands\deploy.py", line 191, in _adjust_job_definitions
    _walk_content(adjustment_callback, job)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\dbx\commands\deploy.py", line 244, in _walk_content
    _walk_content(func, item, content, key)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\dbx\commands\deploy.py", line 244, in _walk_content
    _walk_content(func, item, content, key)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\dbx\commands\deploy.py", line 244, in _walk_content
    _walk_content(func, item, content, key)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\dbx\commands\deploy.py", line 249, in _walk_content
    parent[index] = func(content)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\dbx\commands\deploy.py", line 188, in <lambda>
    adjustment_callback = lambda p: _adjust_path(p, artifact_base_uri, file_uploader)
  File "c:\tools\miniconda3\envs\risbricks\lib\site-packages\dbx\commands\deploy.py", line 256, in _adjust_path
    elif pathlib.Path(candidate).exists():
  File "c:\tools\miniconda3\envs\risbricks\lib\pathlib.py", line 1336, in exists
    self.stat()
  File "c:\tools\miniconda3\envs\risbricks\lib\pathlib.py", line 1158, in stat
    return self._accessor.stat(self)
  File "c:\tools\miniconda3\envs\risbricks\lib\pathlib.py", line 387, in wrapped
    return strfunc(str(pathobj), *args)
OSError: [WinError 123] The filename, directory name, or volume label syntax is incorrect: 'local[*]'

The _adjust_path function needs to be changed to skip spark_conf

If I remove "spark.master": "local[*]", the job will deploy however it won't start since driver will keep throwing this warning

Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources

[renardeinside]

Hi @chinwobble ! Thanks for raising the issue. I'll provide a fix in the next release