douglasdavis
commented
3 years ago I've run into this, after digging around a bit with @martindurant we've noticed that dask.config.config is modified (and grows quite a bit with only default settings) upon importing dask_cloudprovider or dask_cloudprovider.{aws,gcp}. (reproducer below).
Would it be reasonable to drop the keys in the config dict that have defaults? (especially things that are in the config only to spin up cloud resources, since the cloud instances don't need to know how to start things on the cloud?)
In [1]: import dask.config
In [2]: dask.config.config
Out[2]:
{'temporary-directory': None,
'dataframe': {'shuffle-compression': None},
'array': {'svg': {'size': 120}, 'slicing': {'split-large-chunks': None}},
'optimization': {'fuse': {'active': True,
'ave-width': 1,
'max-width': None,
'max-height': inf,
'max-depth-new-edges': None,
'subgraphs': None,
'rename-keys': True}}}
In [3]: import dask_cloudprovider.aws
In [4]: dask.config.config
Out[4]:
{'temporary-directory': None,
'dataframe': {'shuffle-compression': None},
'array': {'svg': {'size': 120}, 'slicing': {'split-large-chunks': None}},
'optimization': {'fuse': {'active': True,
'ave-width': 1,
'max-width': None,
'max-height': inf,
'max-depth-new-edges': None,
'subgraphs': None,
'rename-keys': True}},
'cloudprovider': {'ecs': {'fargate_scheduler': False,
'fargate_workers': False,
'scheduler_cpu': 1024,
'scheduler_mem': 4096,
'worker_cpu': 4096,
'worker_mem': 16384,
'worker_gpu': 0,
'n_workers': 0,
'scheduler_timeout': '5 minutes',
'image': 'daskdev/dask:latest',
'gpu_image': 'rapidsai/rapidsai:latest',
'cluster_name_template': 'dask-{uuid}',
'cluster_arn': '',
'execution_role_arn': '',
'task_role_arn': '',
'task_role_policies': [],
'cloudwatch_logs_group': '',
'cloudwatch_logs_stream_prefix': '{cluster_name}',
'cloudwatch_logs_default_retention': 30,
'vpc': 'default',
'subnets': [],
'security_groups': [],
'tags': {},
'environment': {},
'find_address_timeout': 60,
'skip_cleanup': False},
'ec2': {'region': None,
'availability_zone': None,
'bootstrap': True,
'auto_shutdown': True,
'instance_type': 't2.micro',
'docker_image': 'daskdev/dask:latest',
'filesystem_size': 40,
'key_name': None,
'iam_instance_profile': {}},
'azure': {'location': None,
'resource_group': None,
'azurevm': {'vnet': None,
'security_group': None,
'public_ingress': True,
'vm_size': 'Standard_DS1_v2',
'disk_size': 50,
'scheduler_vm_size': None,
'docker_image': 'daskdev/dask:latest',
'vm_image': {'publisher': 'Canonical',
'offer': 'UbuntuServer',
'sku': '18.04-LTS',
'version': 'latest'},
'bootstrap': True,
'auto_shutdown': True}},
'digitalocean': {'token': None,
'region': 'nyc3',
'size': 's-1vcpu-1gb',
'image': 'ubuntu-20-04-x64'},
'gcp': {'source_image': 'projects/ubuntu-os-cloud/global/images/ubuntu-minimal-1804-bionic-v20201014',
'zone': 'us-east1-c',
'network': 'default',
'projectid': '',
'on_host_maintenance': 'TERMINATE',
'machine_type': 'n1-standard-1',
'filesystem_size': 50,
'ngpus': '',
'gpu_type': '',
'disk_type': 'pd-standard',
'docker_image': 'daskdev/dask:latest',
'auto_shutdown': True,
'public_ingress': True}},
'distributed': {'version': 2,
'scheduler': {'allowed-failures': 3,
'bandwidth': 100000000,
'blocked-handlers': [],
'default-data-size': '1kiB',
'events-cleanup-delay': '1h',
'idle-timeout': None,
'transition-log-length': 100000,
'events-log-length': 100000,
'work-stealing': True,
'work-stealing-interval': '100ms',
'worker-ttl': None,
'pickle': True,
'preload': [],
'preload-argv': [],
'unknown-task-duration': '500ms',
'default-task-durations': {'rechunk-split': '1us', 'shuffle-split': '1us'},
'validate': False,
'dashboard': {'status': {'task-stream-length': 1000},
'tasks': {'task-stream-length': 100000},
'tls': {'ca-file': None, 'key': None, 'cert': None},
'bokeh-application': {'allow_websocket_origin': ['*'],
'keep_alive_milliseconds': 500,
'check_unused_sessions_milliseconds': 500}},
'locks': {'lease-validation-interval': '10s', 'lease-timeout': '30s'},
'http': {'routes': ['distributed.http.scheduler.prometheus',
'distributed.http.scheduler.info',
'distributed.http.scheduler.json',
'distributed.http.health',
'distributed.http.proxy',
'distributed.http.statics']},
'allowed-imports': ['dask', 'distributed']},
'worker': {'blocked-handlers': [],
'multiprocessing-method': 'spawn',
'use-file-locking': True,
'connections': {'outgoing': 50, 'incoming': 10},
'preload': [],
'preload-argv': [],
'daemon': True,
'validate': False,
'resources': {},
'lifetime': {'duration': None, 'stagger': '0 seconds', 'restart': False},
'profile': {'interval': '10ms', 'cycle': '1000ms', 'low-level': False},
'memory': {'recent_to_old_time': '30s',
'target': 0.6,
'spill': 0.7,
'pause': 0.8,
'terminate': 0.95},
'http': {'routes': ['distributed.http.worker.prometheus',
'distributed.http.health',
'distributed.http.statics']}},
'nanny': {'preload': [], 'preload-argv': []},
'client': {'heartbeat': '5s', 'scheduler-info-interval': '2s'},
'deploy': {'lost-worker-timeout': '15s', 'cluster-repr-interval': '500ms'},
'adaptive': {'interval': '1s',
'target-duration': '5s',
'minimum': 0,
'maximum': inf,
'wait-count': 3},
'comm': {'retry': {'count': 0, 'delay': {'min': '1s', 'max': '20s'}},
'compression': 'auto',
'offload': '10MiB',
'default-scheme': 'tcp',
'socket-backlog': 2048,
'recent-messages-log-length': 0,
'zstd': {'level': 3, 'threads': 0},
'timeouts': {'connect': '10s', 'tcp': '30s'},
'require-encryption': None,
'tls': {'ciphers': None,
'ca-file': None,
'scheduler': {'cert': None, 'key': None},
'worker': {'key': None, 'cert': None},
'client': {'key': None, 'cert': None}}},
'dashboard': {'link': '{scheme}://{host}:{port}/status',
'export-tool': False,
'graph-max-items': 5000,
'prometheus': {'namespace': 'dask'}},
'admin': {'tick': {'interval': '20ms', 'limit': '3s'},
'max-error-length': 10000,
'log-length': 10000,
'log-format': '%(name)s - %(levelname)s - %(message)s',
'pdb-on-err': False,
'system-monitor': {'interval': '500ms'},
'event-loop': 'tornado'}},
'rmm': {'pool-size': None},
'ucx': {'cuda_copy': False,
'tcp': False,
'nvlink': False,
'infiniband': False,
'rdmacm': False,
'net-devices': None,
'reuse-endpoints': True}}
eric-valente
martindurant
jacobtomlinson
tiagoantao
shireenrao
filpano
It seems there is a 16kb limit on the amount of user_data that can be passed to an EC2 instance when starting up.
We serialize the local Dask config and pass it to the scheduler and workers via the user_data.
https://github.com/dask/dask-cloudprovider/blob/da454827b88c2f0f0d06af07e02d5d1580a4c366/dask_cloudprovider/generic/vmcluster.py#L33-L35
Depending on what config the user has locally this can tip us over the limit and result in the AWS API rejecting the instance creation call.