Scaling clusters in GCP causes segmentation fault

It looks like something is falling over when decompressing an API response with Gzip / zlib -- but would appreciate any insights from people who know more about Python builds / C APIs than I do!

This is all on Python 3.8.5 from conda-forge.

Traceback:

``` Fatal Python error: Segmentation fault -- Thread 0x00007efcdffff700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1099 in read File "/opt/conda/lib/python3.8/ssl.py", line 1241 in recv_into File "/opt/conda/lib/python3.8/socket.py", line 669 in readinto File "/opt/conda/lib/python3.8/http/client.py", line 515 in _read_next_chunk_size File "/opt/conda/lib/python3.8/http/client.py", line 555 in _get_chunk_left File "/opt/conda/lib/python3.8/http/client.py", line 572 in _readall_chunked File "/opt/conda/lib/python3.8/http/client.py", line 465 in read File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1405 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efcf4dfa700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1099 in read File "/opt/conda/lib/python3.8/ssl.py", line 1241 in recv_into File "/opt/conda/lib/python3.8/socket.py", line 669 in readinto File "/opt/conda/lib/python3.8/http/client.py", line 268 in _read_status File "/opt/conda/lib/python3.8/http/client.py", line 307 in begin File "/opt/conda/lib/python3.8/http/client.py", line 1347 in getresponse File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1376 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efcf57fb700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1099 in read File "/opt/conda/lib/python3.8/ssl.py", line 1241 in recv_into File "/opt/conda/lib/python3.8/socket.py", line 669 in readinto File "/opt/conda/lib/python3.8/http/client.py", line 268 in _read_status File "/opt/conda/lib/python3.8/http/client.py", line 307 in begin File "/opt/conda/lib/python3.8/http/client.py", line 1347 in getresponse File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1376 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 55 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efcf61fc700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1173 in send File "/opt/conda/lib/python3.8/ssl.py", line 1204 in sendall File "/opt/conda/lib/python3.8/http/client.py", line 971 in send File "/opt/conda/lib/python3.8/http/client.py", line 1010 in _send_output File "/opt/conda/lib/python3.8/http/client.py", line 1250 in endheaders File "/opt/conda/lib/python3.8/http/client.py", line 1301 in _send_request File "/opt/conda/lib/python3.8/http/client.py", line 1255 in request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1347 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Current thread 0x00007efcf6bfd700 (most recent call first): File "/opt/conda/lib/python3.8/gzip.py", line 487 in read File "/opt/conda/lib/python3.8/gzip.py", line 292 in read File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 358 in _decompressContent File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1408 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efcf75fe700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1099 in read File "/opt/conda/lib/python3.8/ssl.py", line 1241 in recv_into File "/opt/conda/lib/python3.8/socket.py", line 669 in readinto File "/opt/conda/lib/python3.8/http/client.py", line 612 in _safe_read File "/opt/conda/lib/python3.8/http/client.py", line 575 in _readall_chunked File "/opt/conda/lib/python3.8/http/client.py", line 465 in read File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1405 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 55 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efcf7fff700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1099 in read File "/opt/conda/lib/python3.8/ssl.py", line 1241 in recv_into File "/opt/conda/lib/python3.8/socket.py", line 669 in readinto File "/opt/conda/lib/python3.8/http/client.py", line 268 in _read_status File "/opt/conda/lib/python3.8/http/client.py", line 307 in begin File "/opt/conda/lib/python3.8/http/client.py", line 1347 in getresponse File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1376 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 55 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efcfcbfe700 (most recent call first): File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efcfd5ff700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 483 in __new__ File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 147 in _build_ssl_context File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1080 in __init__ File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1561 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd063be700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 483 in __new__ File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 147 in _build_ssl_context File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1080 in __init__ File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1561 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 55 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd0737f700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1099 in read File "/opt/conda/lib/python3.8/ssl.py", line 1241 in recv_into File "/opt/conda/lib/python3.8/socket.py", line 669 in readinto File "/opt/conda/lib/python3.8/http/client.py", line 612 in _safe_read File "/opt/conda/lib/python3.8/http/client.py", line 575 in _readall_chunked File "/opt/conda/lib/python3.8/http/client.py", line 465 in read File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1405 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd217fb700 (most recent call first): File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 169 in _build_ssl_context File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1080 in __init__ File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1561 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd221fc700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1309 in do_handshake File "/opt/conda/lib/python3.8/ssl.py", line 1040 in _create File "/opt/conda/lib/python3.8/ssl.py", line 500 in wrap_socket File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1138 in connect File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1346 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 54 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd34f3d700 (most recent call first): File "/opt/conda/lib/python3.8/ssl.py", line 1031 in _create File "/opt/conda/lib/python3.8/ssl.py", line 500 in wrap_socket File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1138 in connect File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1346 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 55 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd22bfd700 (most recent call first): File "/opt/conda/lib/python3.8/socket.py", line 231 in __init__ File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1132 in connect File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1346 in _conn_request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1424 in _request File "/opt/conda/lib/python3.8/site-packages/httplib2/__init__.py", line 1708 in request File "/opt/conda/lib/python3.8/site-packages/google_auth_httplib2.py", line 200 in request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 177 in _retry_request File "/opt/conda/lib/python3.8/site-packages/googleapiclient/http.py", line 900 in execute File "/opt/conda/lib/python3.8/site-packages/googleapiclient/_helpers.py", line 134 in positional_wrapper File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 29 in get_common_metadata File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/helpers.py", line 113 in get_metadata_item_value File "/opt/conda/lib/python3.8/site-packages/cloudbridge/providers/gcp/resources.py", line 483 in label File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 55 in File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 53 in filter_by File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/helpers.py", line 70 in generic_find File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/services.py", line 90 in find File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 110 in invoke File "/opt/conda/lib/python3.8/site-packages/cloudbridge/base/middleware.py", line 45 in wrap_exception File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 78 in invoke File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/events.py", line 218 in dispatch File "/opt/conda/lib/python3.8/site-packages/pyeventsystem/middleware.py", line 74 in wrapper File "./backends/cloudbridge/cloudbridge_utils.py", line 1516 in _get_fw File "./backends/cloudbridge/cloudbridge_utils.py", line 700 in add_worker File "./backends/cloudbridge/cloudbridge_utils.py", line 202 in time_wrapper File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd23fff700 (most recent call first): File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd235fe700 (most recent call first): File "/opt/conda/lib/python3.8/threading.py", line 306 in wait File "/opt/conda/lib/python3.8/threading.py", line 558 in wait File "/opt/conda/lib/python3.8/site-packages/distributed/utils.py", line 337 in sync File "./backends/core.py", line 147 in sync File "./backends/core.py", line 357 in scale File "./cloud/views/clusters.py", line 148 in update File "/opt/conda/lib/python3.8/site-packages/rest_framework/generics.py", line 229 in patch File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/rest_framework/views.py", line 502 in dispatch File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/views/generic/base.py", line 70 in view File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/views/decorators/csrf.py", line 54 in wrapped_view File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/base.py", line 181 in _get_response File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "./cloud/middleware/request_logging.py", line 66 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/utils/deprecation.py", line 114 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/utils/deprecation.py", line 114 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/utils/deprecation.py", line 114 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/utils/deprecation.py", line 114 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/utils/deprecation.py", line 114 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/utils/deprecation.py", line 114 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/whitenoise/middleware.py", line 59 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/utils/deprecation.py", line 114 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/utils/deprecation.py", line 114 in __call__ File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 259 in wrapped File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/exception.py", line 47 in inner File "/opt/conda/lib/python3.8/site-packages/django/core/handlers/base.py", line 130 in get_response File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 430 in traced_get_response File "/opt/conda/lib/python3.8/site-packages/ddtrace/contrib/django/patch.py", line 82 in wrapper File "/opt/conda/lib/python3.8/site-packages/channels/http.py", line 244 in handle File "/opt/conda/lib/python3.8/site-packages/asgiref/sync.py", line 334 in thread_handler File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 57 in run File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd35e7e700 (most recent call first): File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd3deff700 (most recent call first): File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd4bfff700 (most recent call first): File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd612bb700 (most recent call first): File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd621fc700 (most recent call first): File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd62bfd700 (most recent call first): File "/opt/conda/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd635fe700 (most recent call first): File "/opt/conda/lib/python3.8/threading.py", line 306 in wait File "/opt/conda/lib/python3.8/queue.py", line 179 in get File "/opt/conda/lib/python3.8/site-packages/distributed/threadpoolexecutor.py", line 51 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd63fff700 (most recent call first): File "/opt/conda/lib/python3.8/threading.py", line 306 in wait File "/opt/conda/lib/python3.8/queue.py", line 179 in get File "/opt/conda/lib/python3.8/site-packages/distributed/threadpoolexecutor.py", line 51 in _worker File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd6d045700 (most recent call first): File "/opt/conda/lib/python3.8/site-packages/distributed/profile.py", line 269 in _watch File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd74f02700 (most recent call first): File "./subscriptions/scheduler.py", line 36 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd75a43700 (most recent call first): File "/opt/conda/lib/python3.8/threading.py", line 306 in wait File "/opt/conda/lib/python3.8/threading.py", line 558 in wait File "/opt/conda/lib/python3.8/site-packages/ddtrace/_worker.py", line 68 in _target File "/opt/conda/lib/python3.8/threading.py", line 870 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efda3763700 (most recent call first): File "/opt/conda/lib/python3.8/threading.py", line 306 in wait File "/opt/conda/lib/python3.8/threading.py", line 558 in wait File "/opt/conda/lib/python3.8/site-packages/ddtrace/profiling/_periodic.py", line 45 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efda2d62700 (most recent call first): File "/opt/conda/lib/python3.8/site-packages/ddtrace/profiling/collector/memory.py", line 43 in File "/opt/conda/lib/python3.8/site-packages/ddtrace/profiling/collector/memory.py", line 43 in File "/opt/conda/lib/python3.8/site-packages/ddtrace/profiling/collector/memory.py", line 43 in _filter_profiler File "/opt/conda/lib/python3.8/site-packages/ddtrace/profiling/collector/memory.py", line 70 in collect File "/opt/conda/lib/python3.8/site-packages/ddtrace/profiling/collector/__init__.py", line 21 in periodic File "/opt/conda/lib/python3.8/site-packages/ddtrace/profiling/_periodic.py", line 46 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efd9a361700 (most recent call first): File "/opt/conda/lib/python3.8/threading.py", line 306 in wait File "/opt/conda/lib/python3.8/threading.py", line 558 in wait File "/opt/conda/lib/python3.8/site-packages/ddtrace/profiling/_periodic.py", line 45 in run File "/opt/conda/lib/python3.8/threading.py", line 932 in _bootstrap_inner File "/opt/conda/lib/python3.8/threading.py", line 890 in _bootstrap Thread 0x00007efda8fb6740 (most recent call first): File "/opt/conda/lib/python3.8/linecache.py", line 74 in checkcache File "/opt/conda/lib/python3.8/traceback.py", line 362 in extract File "/opt/conda/lib/python3.8/traceback.py", line 211 in extract_stack File "/opt/conda/lib/python3.8/site-packages/tornado/iostream.py", line 540 in write File "/opt/conda/lib/python3.8/site-packages/distributed/comm/tcp.py", line 248 in write File "/opt/conda/lib/python3.8/site-packages/distributed/core.py", line 642 in send_recv File "/opt/conda/lib/python3.8/site-packages/distributed/core.py", line 861 in send_recv_from_rpc File "/opt/conda/lib/python3.8/site-packages/distributed/client.py", line 1154 in _update_scheduler_info File "/opt/conda/lib/python3.8/asyncio/events.py", line 81 in _run File "/opt/conda/lib/python3.8/asyncio/base_events.py", line 1851 in _run_once File "/opt/conda/lib/python3.8/asyncio/base_events.py", line 570 in run_forever File "/opt/conda/lib/python3.8/site-packages/twisted/internet/asyncioreactor.py", line 267 in run File "/opt/conda/lib/python3.8/site-packages/daphne/server.py", line 140 in run File "/opt/conda/lib/python3.8/site-packages/daphne/cli.py", line 311 in run File "/opt/conda/lib/python3.8/site-packages/daphne/cli.py", line 191 in entrypoint File "/opt/conda/bin/daphne", line 8 in Segmentation fault (core dumped) ```

Core dumps:

coiled / feedback

Scaling clusters in GCP causes segmentation fault #117