nrdg / cloudknot

A python library to run your existing code on AWS Batch
https://nrdg.github.io/cloudknot/
Other
71 stars 17 forks source link

Cannot load `ck.Knot` by name #257

Open bloomdt-uw opened 4 years ago

bloomdt-uw commented 4 years ago

Command:

knot = ck.Knot(
    name="vis-rois-2020-10-20T12-39-24"
)

Stacktrace:

---------------------------------------------------------------------------
CloudknotInputError                       Traceback (most recent call last)
<ipython-input-10-3a52aa6ea509> in <module>
----> 1 knot = ck.Knot(
      2     name="vis-rois-2020-10-20T12-39-24"
      3 )

~/anaconda/envs/pyAFQ/lib/python3.8/site-packages/cloudknot/cloudknot.py in __init__(self, name, pars, pars_policies, docker_image, base_image, func, image_script_path, image_work_dir, image_github_installs, username, repo_name, image_tags, job_definition_name, job_def_vcpus, memory, retries, compute_environment_name, instance_types, min_vcpus, max_vcpus, desired_vcpus, volume_size, image_id, ec2_key_pair, bid_percentage, job_queue_name, priority)
   1535             self._pars, pars_cleanup = futures["pars"].result()
   1536 
-> 1537             self._docker_image, self._docker_repo, repo_cleanup = futures[
   1538                 "docker-image"
   1539             ].result()

~/anaconda/envs/pyAFQ/lib/python3.8/concurrent/futures/_base.py in result(self, timeout)
    430                 raise CancelledError()
    431             elif self._state == FINISHED:
--> 432                 return self.__get_result()
    433 
    434             self._condition.wait(timeout)

~/anaconda/envs/pyAFQ/lib/python3.8/concurrent/futures/_base.py in __get_result(self)
    386     def __get_result(self):
    387         if self._exception:
--> 388             raise self._exception
    389         else:
    390             return self._result

~/anaconda/envs/pyAFQ/lib/python3.8/concurrent/futures/thread.py in run(self)
     55 
     56         try:
---> 57             result = self.fn(*self.args, **self.kwargs)
     58         except BaseException as exc:
     59             self.future.set_exception(exc)

~/anaconda/envs/pyAFQ/lib/python3.8/site-packages/cloudknot/cloudknot.py in set_dockerimage(knot_name, input_docker_image, func_, script_path, work_dir, base_image_, github_installs, username_, tags, repo_name_)
   1453                 else:
   1454                     # Create and build the docker image
-> 1455                     di = dockerimage.DockerImage(
   1456                         func=func_,
   1457                         script_path=script_path,

~/anaconda/envs/pyAFQ/lib/python3.8/site-packages/cloudknot/dockerimage.py in __init__(self, name, func, script_path, dir_name, base_image, github_installs, username, overwrite)
    129         # User must specify at least `func` or `script_path`
    130         if not any([name, func, script_path]):
--> 131             raise CloudknotInputError(
    132                 "You must suppy either `name`, `func` " "or `script_path`."
    133             )

CloudknotInputError: You must suppy either `name`, `func` or `script_path`.
bloomdt-uw commented 4 years ago

Cloudknot also fails if can't get all jobs

knot = ck.Knot(name="vis_rois-2020-10-20T12-39-24")

config file:

[knot vis_rois-2020-10-20T12-39-24]
region = us-west-2
profile = default
stack-id = arn:aws:cloudformation:us-west-2:454929164628:stack/vis-rois-2020-10-20T12-39-24-knot/6a526760-130d-11eb-b949-06df31711729
pars = vis-rois-2020-10-20T12-39-24
docker-image = vis-rois
docker-repo = cloudknot
job-definition = arn:aws:batch:us-west-2:454929164628:job-definition/vis_rois-2020-10-20T12-39-24-cloudknot-job-definition:1
compute-environment = arn:aws:batch:us-west-2:454929164628:compute-environment/vis_rois-2020-10-20T12-39-24-cloudknot-compute-environment
job-queue = arn:aws:batch:us-west-2:454929164628:job-queue/vis_rois-2020-10-20T12-39-24-cloudknot-job-queue
job_ids = afd2d5f1-1f2c-46eb-ada2-fbd52d3e17b6

Stacktrace:

---------------------------------------------------------------------------
ResourceDoesNotExistException             Traceback (most recent call last)
<ipython-input-11-759c29cdc470> in <module>
----> 1 knot = ck.Knot(name="vis_rois-2020-10-20T12-39-24")

~/anaconda/envs/pyAFQ/lib/python3.8/site-packages/cloudknot/cloudknot.py in __init__(self, name, pars, pars_policies, docker_image, base_image, func, image_script_path, image_work_dir, image_github_installs, username, repo_name, image_tags, job_definition_name, job_def_vcpus, memory, retries, compute_environment_name, instance_types, min_vcpus, max_vcpus, desired_vcpus, volume_size, image_id, ec2_key_pair, bid_percentage, job_queue_name, priority)
   1083 
   1084             self._job_ids = config.get(self._knot_name, "job_ids").split()
-> 1085             self._jobs = [aws.BatchJob(job_id=jid) for jid in self.job_ids]
   1086         else:
   1087             if pars and not isinstance(pars, Pars):

~/anaconda/envs/pyAFQ/lib/python3.8/site-packages/cloudknot/cloudknot.py in <listcomp>(.0)
   1083 
   1084             self._job_ids = config.get(self._knot_name, "job_ids").split()
-> 1085             self._jobs = [aws.BatchJob(job_id=jid) for jid in self.job_ids]
   1086         else:
   1087             if pars and not isinstance(pars, Pars):

~/anaconda/envs/pyAFQ/lib/python3.8/site-packages/cloudknot/aws/batch.py in __init__(self, job_id, name, job_queue, job_definition, input_, starmap, environment_variables, array_job)
    182             job = _exists_already(job_id=job_id)
    183             if not job.exists:
--> 184                 raise ResourceDoesNotExistException(
    185                     "jobId {id:s} does not exists".format(id=job_id), job_id
    186                 )

ResourceDoesNotExistException: jobId afd2d5f1-1f2c-46eb-ada2-fbd52d3e17b6 does not exists