cannot create cluster on coiled

[Ananth-9]

hi, I have created the access token key on the coiled and I ran the same code which is mentioned on the coiled website

`cluster = coiled.Cluster(n_workers=20)

connect a Dask client to the cluster

client = cluster.get_client()

link to Dask scheduler dashboard

print('Dashboard:', client.dashboard_link)`

but after entering the key it says the authentication is successful in the scanning Python package phase it throws a type error saying

TypeError: expected str, bytes or os.PathLike object, not NoneType

any help is appreciated thanks in advance

[dchudz]

Thanks for the report, @Ananth-9! We'll take a look today.

You're probably best off waiting for us to address this quickly, but you could also sidestep the issue by using Coiled software environments (https://docs.coiled.io/user_guide/software_environment.html), or a docker container (coiled.Cluster(container=...)).

[dan-blanchard]

@Ananth-9, to help us better debug your issue, could you please send us the results of running coiled package-sync scan --csv and coiled package-sync debug --csv in the directory your code is in?

[dchudz]

@Ananth-9 Feel free to email support@coiled.io, if you don't want to put that info in this public issue.

[dan-blanchard]

For posterity, the full stack trace is:

TypeError                                 Traceback (most recent call last)
Cell In[2], line 2
      1 # create a remote Dask cluster with Coiled
----> 2 cluster = coiled.Cluster()
      4 # connect a Dask client to the cluster
      5 client = cluster.get_client()

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\coiled\_beta\cluster.py:599, in Cluster.__init__(self, name, software, container, n_workers, worker_class, worker_options, worker_vm_types, worker_cpu, worker_memory, worker_disk_size, worker_gpu, worker_gpu_type, scheduler_class, scheduler_options, scheduler_vm_types, scheduler_cpu, scheduler_memory, scheduler_gpu, asynchronous, cloud, account, shutdown_on_close, use_scheduler_public_ip, use_dashboard_https, credentials, credentials_duration_seconds, timeout, environ, tags, send_dask_config, backend_options, show_widget, configure_logging, wait_for_workers, package_sync, package_sync_strict, package_sync_ignore, package_sync_fail_on, private_to_creator, use_best_zone, compute_purchase_option, scheduler_port, allow_ingress_from, allow_ssh, jupyter, region, arm)
    597     error = e
    598     self.close()
--> 599     raise e.with_traceback(truncate_traceback(e.__traceback__))
    600 finally:
    601     if error:

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\coiled\_beta\cluster.py:580, in Cluster.__init__(self, name, software, container, n_workers, worker_class, worker_options, worker_vm_types, worker_cpu, worker_memory, worker_disk_size, worker_gpu, worker_gpu_type, scheduler_class, scheduler_options, scheduler_vm_types, scheduler_cpu, scheduler_memory, scheduler_gpu, asynchronous, cloud, account, shutdown_on_close, use_scheduler_public_ip, use_dashboard_https, credentials, credentials_duration_seconds, timeout, environ, tags, send_dask_config, backend_options, show_widget, configure_logging, wait_for_workers, package_sync, package_sync_strict, package_sync_ignore, package_sync_fail_on, private_to_creator, use_best_zone, compute_purchase_option, scheduler_port, allow_ingress_from, allow_ssh, jupyter, region, arm)
    578 error = None
    579 try:
--> 580     self.sync(self._start)
    581 except (ClusterCreationError, InstanceTypeError) as e:
    582     error = e

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\distributed\utils.py:351, in SyncMethodMixin.sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
    349     return future
    350 else:
--> 351     return sync(
    352         self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
    353     )

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\distributed\utils.py:418, in sync(loop, func, callback_timeout, *args, **kwargs)
    416 if error:
    417     typ, exc, tb = error
--> 418     raise exc.with_traceback(tb)
    419 else:
    420     return result

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\distributed\utils.py:391, in sync.<locals>.f()
    389         future = wait_for(future, callback_timeout)
    390     future = asyncio.ensure_future(future)
--> 391     result = yield future
    392 except Exception:
    393     error = sys.exc_info()

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\tornado\gen.py:767, in Runner.run(self)
    765 try:
    766     try:
--> 767         value = future.result()
    768     except Exception as e:
    769         # Save the exception for later. It's important that
    770         # gen.throw() not be called inside this try/except block
    771         # because that makes sys.exc_info behave unexpectedly.
    772         exc: Optional[Exception] = e

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\coiled\_beta\cluster.py:1102, in Cluster._start(self)
   1095 if not self.package_sync:
   1096     parse_identifier(
   1097         self.software_environment,
   1098         property_name="software_environment",
   1099         can_have_revision=False,
   1100     )
-> 1102 senv_v2_id = await self._determine_senv(architecture=architecture, gpu_enabled=self._is_gpu_cluster)
   1104 self.cluster_id, cluster_existed = await cloud._create_cluster(
   1105     account=self.account,
   1106     name=self.name,
   (...)
   1125     private_to_creator=self.private_to_creator,
   1126 )
   1127 cluster_created = not cluster_existed

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\coiled\_beta\cluster.py:798, in Cluster._determine_senv(self, architecture, gpu_enabled)
    796         package_level_lookup[(package, "conda")] = PackageLevelEnum.IGNORE
    797         package_level_lookup[(package, "pip")] = PackageLevelEnum.IGNORE
--> 798 approximation = await magic.create_environment_approximation(
    799     cloud=self.cloud,
    800     only=self.package_sync_only,
    801     priorities=package_level_lookup,
    802     strict=self.package_sync_strict,
    803     progress=progress,
    804     architecture=architecture,
    805 )
    807 if not self.package_sync_only:
    808     # if we're not operating on a subset, check
    809     # all the coiled defined critical packages are present
    810     packages_by_name: Dict[str, magic.ResolvedPackageInfo] = {p["name"]: p for p in approximation}

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\coiled\magic.py:436, in create_environment_approximation(cloud, priorities, only, strict, progress, architecture)
    427 @track_context
    428 async def create_environment_approximation(
    429     cloud: CloudBeta,
   (...)
    434     architecture: ArchitectureTypesEnum = ArchitectureTypesEnum.X86_64,
    435 ) -> typing.List[ResolvedPackageInfo]:
--> 436     packages = await scan_prefix(progress=progress)
    437     bad_packages = await check_pip_happy(progress)
    438     if bad_packages:

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\coiled\scan.py:278, in scan_prefix(prefix, progress, locations)
    274 pip_env_future = asyncio.create_task(
    275     scan_pip(locations=locations or [Path(p) for p in sys.path], progress=progress)
    276 )
    277 conda_env = await conda_env_future
--> 278 pip_env = await pip_env_future
    279 filtered_conda = {}
    280 # the pip list is the "truth" of what is imported for python deps

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\coiled\scan.py:258, in scan_pip(locations, progress)
    252 if progress:
    253     for task in progress.track(
    254         asyncio.as_completed([handle_dist(dist, locations) for dist in dists]),
    255         total=len(dists),
    256         description=f"Scanning {len(dists)} python packages",
    257     ):
--> 258         packages.append(await task)
    259 else:
    260     packages = await asyncio.gather(*(handle_dist(dist, locations) for dist in dists))

File ~\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py:614, in as_completed.<locals>._wait_for_one()
    611 if f is None:
    612     # Dummy value from _on_timeout().
    613     raise exceptions.TimeoutError
--> 614 return f.result()

File ~\AppData\Local\Programs\Python\Python39\lib\site-packages\coiled\scan.py:193, in handle_dist(dist, locations)
    191 for location in locations:
    192     if os.name == "nt":
--> 193         egg_link_pth = location / Path(dist.name).with_suffix(".egg-link")
    194         if egg_link_pth.is_file():
    195             egg_links.append(location / Path(dist.name).with_suffix(".egg-link"))

File ~\AppData\Local\Programs\Python\Python39\lib\pathlib.py:1071, in Path.__new__(cls, *args, **kwargs)
   1069 if cls is Path:
   1070     cls = WindowsPath if os.name == 'nt' else PosixPath
-> 1071 self = cls._from_parts(args, init=False)
   1072 if not self._flavour.is_supported:
   1073     raise NotImplementedError("cannot instantiate %r on your system"
   1074                               % (cls.__name__,))

File ~\AppData\Local\Programs\Python\Python39\lib\pathlib.py:696, in PurePath._from_parts(cls, args, init)
    691 @classmethod
    692 def _from_parts(cls, args, init=True):
    693     # We need to call _parse_args on the instance, so as to get the
    694     # right flavour.
    695     self = object.__new__(cls)
--> 696     drv, root, parts = self._parse_args(args)
    697     self._drv = drv
    698     self._root = root

File ~\AppData\Local\Programs\Python\Python39\lib\pathlib.py:680, in PurePath._parse_args(cls, args)
    678     parts += a._parts
    679 else:
--> 680     a = os.fspath(a)
    681     if isinstance(a, str):
    682         # Force-cast str subclasses to str (issue #21127)
    683         parts.append(str(a))

TypeError: expected str, bytes or os.PathLike object, not NoneType