Closed MichaelCoulter closed 2 years ago
So it looks like you're getting errors about invalid timestamps and an invalid token. Which group are you in? Are you able to write to kachery-cloud via the command line? Are others in the lab experiencing similar issues?
Ah okay, thanks. I'm in Loren Frank's group at UCSF. I don't know if others are getting this error. Yes I am normally able to write to kachery-cloud, but it sometimes the connection doesn't work. It happens on and off, but i haven't been able to find a specific action that always fails. Sorry I dont have more specific info. Thanks.
I consistently see this error message, but the code continues to run: Exception in subscribe loop: HTTP Client Error (403): {"message":"Token is expired.","error":true,"service":"Access Manager","status":403}
SSLEOFError Traceback (most recent call last)
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/urllib3/connectionpool.py:703, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
702 # Make the request on the httplib connection object.
--> 703 httplib_response = self._make_request(
704 conn,
705 method,
706 url,
707 timeout=timeout_obj,
708 body=body,
709 headers=headers,
710 chunked=chunked,
711 )
713 # If we're going to release the connection in finally:
, then
714 # the response doesn't need to know about the connection. Otherwise
715 # it will also try to release it and we'll have a double-release
716 # mess.
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/urllib3/connectionpool.py:386, in HTTPConnectionPool._make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw) 385 try: --> 386 self._validate_conn(conn) 387 except (SocketTimeout, BaseSSLError) as e: 388 # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout.
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/urllib3/connectionpool.py:1042, in HTTPSConnectionPool._validate_conn(self, conn)
1041 if not getattr(conn, "sock", None): # AppEngine might not have .sock
-> 1042 conn.connect()
1044 if not conn.is_verified:
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/urllib3/connection.py:414, in HTTPSConnection.connect(self) 412 context.load_default_certs() --> 414 self.sock = ssl_wrap_socket( 415 sock=conn, 416 keyfile=self.key_file, 417 certfile=self.cert_file, 418 key_password=self.key_password, 419 ca_certs=self.ca_certs, 420 ca_cert_dir=self.ca_cert_dir, 421 ca_cert_data=self.ca_cert_data, 422 server_hostname=server_hostname, 423 ssl_context=context, 424 tls_in_tls=tls_in_tls, 425 ) 427 # If we're using all defaults and the connection 428 # is TLSv1 or TLSv1.1 we throw a DeprecationWarning 429 # for the host.
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/urllib3/util/ssl_.py:449, in ssl_wrap_socket(sock, keyfile, certfile, cert_reqs, ca_certs, server_hostname, ssl_version, ciphers, ssl_context, ca_cert_dir, key_password, ca_cert_data, tls_in_tls) 448 if send_sni: --> 449 ssl_sock = _ssl_wrap_socket_impl( 450 sock, context, tls_in_tls, server_hostname=server_hostname 451 ) 452 else:
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/urllib3/util/ssl_.py:493, in _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname) 492 if server_hostname: --> 493 return ssl_context.wrap_socket(sock, server_hostname=server_hostname) 494 else:
File ~/anaconda3/envs/spyglass/lib/python3.9/ssl.py:501, in SSLContext.wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session) 495 def wrap_socket(self, sock, server_side=False, 496 do_handshake_on_connect=True, 497 suppress_ragged_eofs=True, 498 server_hostname=None, session=None): 499 # SSLSocket class handles server_hostname encoding before it calls 500 # ctx._wrap_socket() --> 501 return self.sslsocket_class._create( 502 sock=sock, 503 server_side=server_side, 504 do_handshake_on_connect=do_handshake_on_connect, 505 suppress_ragged_eofs=suppress_ragged_eofs, 506 server_hostname=server_hostname, 507 context=self, 508 session=session 509 )
File ~/anaconda3/envs/spyglass/lib/python3.9/ssl.py:1041, in SSLSocket._create(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session) 1040 raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") -> 1041 self.do_handshake() 1042 except (OSError, ValueError):
File ~/anaconda3/envs/spyglass/lib/python3.9/ssl.py:1310, in SSLSocket.do_handshake(self, block) 1309 self.settimeout(None) -> 1310 self._sslobj.do_handshake() 1311 finally:
SSLEOFError: EOF occurred in violation of protocol (_ssl.c:1129)
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last) File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/requests/adapters.py:489, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies) 488 if not chunked: --> 489 resp = conn.urlopen( 490 method=request.method, 491 url=url, 492 body=request.body, 493 headers=request.headers, 494 redirect=False, 495 assert_same_host=False, 496 preload_content=False, 497 decode_content=False, 498 retries=self.max_retries, 499 timeout=timeout, 500 ) 502 # Send the request. 503 else:
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/urllib3/connectionpool.py:787, in HTTPConnectionPool.urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw) 785 e = ProtocolError("Connection aborted.", e) --> 787 retries = retries.increment( 788 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] 789 ) 790 retries.sleep()
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/urllib3/util/retry.py:592, in Retry.increment(self, method, url, response, error, _pool, _stacktrace) 591 if new_retry.is_exhausted(): --> 592 raise MaxRetryError(_pool, url, error or ResponseError(cause)) 594 log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
MaxRetryError: HTTPSConnectionPool(host='cloud.kacheryhub.org', port=443): Max retries exceeded with url: /api/kacherycloud (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:1129)')))
During handling of the above exception, another exception occurred:
SSLError Traceback (most recent call last) Input In [4], in <cell line: 7>() 25 auto_curation_out_key = (ss.Curation & auto_curation_id).fetch1("KEY") 27 ss.SortingviewWorkspaceSelection.insert1(auto_curation_out_key, skip_duplicates=True) ---> 28 ss.SortingviewWorkspace.populate(auto_curation_out_key)
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/datajoint/autopopulate.py:230, in AutoPopulate.populate(self, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions) 226 if processes == 1: 227 for key in ( 228 tqdm(keys, desc=self.class.name) if display_progress else keys 229 ): --> 230 error = self._populate1(key, jobs, **populate_kwargs) 231 if error is not None: 232 error_list.append(error)
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/datajoint/autopopulate.py:281, in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs) 279 self.class._allow_insert = True 280 try: --> 281 make(dict(key), **(make_kwargs or {})) 282 except (KeyboardInterrupt, SystemExit, Exception) as error: 283 try:
File ~/spyglass/src/spyglass/spikesorting/sortingview.py:63, in SortingviewWorkspace.make(self, key) 60 google_user_ids.append(google_user_id[0]) 62 # do ---> 63 workspace_uri, recording_id, sorting_id = _create_spikesortingview_workspace(recording_path=recording_path, 64 sorting_path=sorting_path, 65 merge_groups=merge_groups, 66 workspace_label=workspace_label, 67 recording_label=recording_label, 68 sorting_label=sorting_label, 69 metrics=metrics, 70 curation_labels=curation_labels, 71 google_user_ids=google_user_ids) 73 # insert 74 key['workspace_uri'] = workspace_uri
File ~/spyglass/src/spyglass/spikesorting/sortingview_helper_fn.py:80, in _create_spikesortingview_workspace(recording_path, sorting_path, merge_groups, workspace_label, recording_label, sorting_label, metrics, google_user_ids, curation_labels, similarity_matrix, raster_plot_subsample_max_firing_rate, spike_amplitudes_subsample_max_firing_rate) 74 def _create_spikesortingview_workspace(recording_path: str, sorting_path: str, merge_groups: List[List[int]], 75 workspace_label: str, recording_label: str, sorting_label: str, 76 metrics: dict=None, google_user_ids: List=None, curation_labels: dict=None, 77 similarity_matrix: Union[List[List[float]], None]=None, 78 raster_plot_subsample_max_firing_rate = 50, spike_amplitudes_subsample_max_firing_rate = 50): ---> 80 workspace = sv.create_workspace(label=workspace_label) 82 recording = si.load_extractor(recording_path) 83 if recording.get_num_segments() > 1:
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/sortingview/workspace/create_workspace.py:7, in create_workspace(label) 6 def create_workspace(*, label: Union[str, None]): ----> 7 feed = kcl.create_feed() 8 feed_id = feed.feed_id 9 workspace_uri = f'sortingview-workspace:{feed_id}'
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/kachery_cloud/feeds/create_feed.py:6, in create_feed(project_id) 5 def create_feed(*, project_id: Union[str, None]=None): ----> 6 return Feed.create(project_id=project_id)
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/kachery_cloud/feeds/Feed.py:76, in Feed.create(project_id) 74 else: 75 payload['projectId'] = get_project_id() ---> 76 response = _kacherycloud_request(payload) 77 feed_id = response['feedId'] 78 project_id = response['projectId']
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/kachery_cloud/_kacherycloud_request.py:25, in _kacherycloud_request(request_payload) 21 raise Exception(f'Unable to perform operation in ephemeral mode: {payload["type"]}') 22 req = { 23 'payload': payload 24 } ---> 25 resp = requests.post(url, json=req) 26 if resp.status_code != 200: 27 raise Exception(f'Error in {payload["type"]} ({resp.status_code}) {resp.reason}: {resp.text}')
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/requests/api.py:115, in post(url, data, json, kwargs)
103 def post(url, data=None, json=None, kwargs):
104 r"""Sends a POST request.
105
106 :param url: URL for the new :class:Request
object.
(...)
112 :rtype: requests.Response
113 """
--> 115 return request("post", url, data=data, json=json, **kwargs)
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/requests/api.py:59, in request(method, url, kwargs) 55 # By using the 'with' statement we are sure the session is closed, thus we 56 # avoid leaving sockets open which can trigger a ResourceWarning in some 57 # cases, and look like a memory leak in others. 58 with sessions.Session() as session: ---> 59 return session.request(method=method, url=url, kwargs)
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/requests/sessions.py:587, in Session.request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json) 582 send_kwargs = { 583 "timeout": timeout, 584 "allow_redirects": allow_redirects, 585 } 586 send_kwargs.update(settings) --> 587 resp = self.send(prep, **send_kwargs) 589 return resp
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/requests/sessions.py:701, in Session.send(self, request, kwargs) 698 start = preferred_clock() 700 # Send the request --> 701 r = adapter.send(request, kwargs) 703 # Total elapsed time of the request (approximately) 704 elapsed = preferred_clock() - start
File ~/anaconda3/envs/spyglass/lib/python3.9/site-packages/requests/adapters.py:563, in HTTPAdapter.send(self, request, stream, timeout, verify, cert, proxies) 559 raise ProxyError(e, request=request) 561 if isinstance(e.reason, _SSLError): 562 # This branch is for urllib3 v1.22 and later. --> 563 raise SSLError(e, request=request) 565 raise ConnectionError(e, request=request) 567 except ClosedPoolError as e:
SSLError: HTTPSConnectionPool(host='cloud.kacheryhub.org', port=443): Max retries exceeded with url: /api/kacherycloud (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:1129)')))
Closing b/c I think it's no longer an issue. (Correct me if I'm wrong)
i'm having intermittent errors connecting to the server when calling the sortingviewworkspace.populate function in spyglass. here are two examples of the errors. thanks
Exception Traceback (most recent call last) Input In [3], in <cell line: 7>() 24 ## add to sortingview workspace for manual curation 25 ss.SortingviewWorkspaceSelection.insert1(auto_curation_out_key, skip_duplicates=True) ---> 26 ss.SortingviewWorkspace.populate(auto_curation_out_key)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/datajoint/autopopulate.py:230, in AutoPopulate.populate(self, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions) 226 if processes == 1: 227 for key in ( 228 tqdm(keys, desc=self.class.name) if display_progress else keys 229 ): --> 230 error = self._populate1(key, jobs, **populate_kwargs) 231 if error is not None: 232 error_list.append(error)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/datajoint/autopopulate.py:281, in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs) 279 self.class._allow_insert = True 280 try: --> 281 make(dict(key), **(make_kwargs or {})) 282 except (KeyboardInterrupt, SystemExit, Exception) as error: 283 try:
File ~/spyglass/src/spyglass/spikesorting/sortingview.py:63, in SortingviewWorkspace.make(self, key) 60 google_user_ids.append(google_user_id[0]) 62 # do ---> 63 workspace_uri, recording_id, sorting_id = _create_spikesortingview_workspace(recording_path=recording_path, 64 sorting_path=sorting_path, 65 merge_groups=merge_groups, 66 workspace_label=workspace_label, 67 recording_label=recording_label, 68 sorting_label=sorting_label, 69 metrics=metrics, 70 curation_labels=curation_labels, 71 google_user_ids=google_user_ids) 73 # insert 74 key['workspace_uri'] = workspace_uri
File ~/spyglass/src/spyglass/spikesorting/sortingview_helper_fn.py:185, in _create_spikesortingview_workspace(recording_path, sorting_path, merge_groups, workspace_label, recording_label, sorting_label, metrics, google_user_ids, curation_labels, similarity_matrix, raster_plot_subsample_max_firing_rate, spike_amplitudes_subsample_max_firing_rate) 138 view = vv.MountainLayout( 139 items=[ 140 vv.MountainLayoutItem( (...) 181 ] 182 ) 184 sorting_curation_uri = workspace.get_sorting_curation_uri(sorting_id) --> 185 url = view.url( 186 label=recording_label, 187 sorting_curation_uri=sorting_curation_uri 188 ) 190 print(f"figurl: {url}") 192 return workspace.uri, recording_id, sorting_id
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/sortingview/views/View.py:37, in View.url(self, label, sorting_curation_uri) 32 if self.is_layout: 33 all_views = self.get_descendant_views_including_self() 34 data = { 35 'type': 'SortingLayout', 36 'layout': self.to_dict(), ---> 37 'views': [ 38 { 39 'type': view.type, 40 'viewId': view.id, 41 'dataUri': _upload_data_and_return_uri(view.to_dict()) 42 } 43 for view in all_views if not view.is_layout 44 ] 45 } 46 if sorting_curation_uri is not None: 47 data['sortingCurationUri'] = sorting_curation_uri
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/sortingview/views/View.py:41, in(.0)
32 if self.is_layout:
33 all_views = self.get_descendant_views_including_self()
34 data = {
35 'type': 'SortingLayout',
36 'layout': self.to_dict(),
37 'views': [
38 {
39 'type': view.type,
40 'viewId': view.id,
---> 41 'dataUri': _upload_data_and_return_uri(view.to_dict())
42 }
43 for view in all_views if not view.is_layout
44 ]
45 }
46 if sorting_curation_uri is not None:
47 data['sortingCurationUri'] = sorting_curation_uri
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/sortingview/views/View.py:66, in _upload_data_and_return_uri(data) 65 def _upload_data_and_return_uri(data): ---> 66 return kcl.store_json(fig.serialize_data(data))
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/kachery_cloud/core.py:21, in store_json(x, separators, indent, label, cache_locally) 19 import simplejson 20 text = simplejson.dumps(x, separators=separators, indent=indent, allow_nan=False, sort_keys=True) ---> 21 return store_text(text, label=label, cache_locally=cache_locally)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/kachery_cloud/core.py:16, in store_text(text, label, cache_locally) 14 with open(fname, 'w') as f: 15 f.write(text) ---> 16 return store_file(fname, label=label, cache_locally=cache_locally)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/kachery_cloud/store_file.py:31, in store_file(filename, label, cache_locally) 29 timer = time.time() 30 while True: ---> 31 response: dict = _kacherycloud_request(payload) 32 already_exists = response.get('alreadyExists', False) 33 already_pending = response.get('alreadyPending', False)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/kachery_cloud/_kacherycloud_request.py:27, in _kacherycloud_request(request_payload) 25 resp = requests.post(url, json=req) 26 if resp.status_code != 200: ---> 27 raise Exception(f'Error in {payload["type"]} ({resp.status_code}) {resp.reason}: {resp.text}') 28 response = resp.json() 29 return response
Exception: Error in initiateFileUpload (500) Internal Server Error: Error: Invalid timestamp. 1660192522655 1660192564746 42091
Exception Traceback (most recent call last) Input In [11], in <cell line: 7>() 25 ## add to sortingview workspace for manual curation 26 ss.SortingviewWorkspaceSelection.insert1(auto_curation_out_key, skip_duplicates=True) ---> 27 ss.SortingviewWorkspace.populate(auto_curation_out_key)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/datajoint/autopopulate.py:230, in AutoPopulate.populate(self, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions) 226 if processes == 1: 227 for key in ( 228 tqdm(keys, desc=self.class.name) if display_progress else keys 229 ): --> 230 error = self._populate1(key, jobs, **populate_kwargs) 231 if error is not None: 232 error_list.append(error)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/datajoint/autopopulate.py:281, in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs) 279 self.class._allow_insert = True 280 try: --> 281 make(dict(key), **(make_kwargs or {})) 282 except (KeyboardInterrupt, SystemExit, Exception) as error: 283 try:
File ~/spyglass/src/spyglass/spikesorting/sortingview.py:63, in SortingviewWorkspace.make(self, key) 60 google_user_ids.append(google_user_id[0]) 62 # do ---> 63 workspace_uri, recording_id, sorting_id = _create_spikesortingview_workspace(recording_path=recording_path, 64 sorting_path=sorting_path, 65 merge_groups=merge_groups, 66 workspace_label=workspace_label, 67 recording_label=recording_label, 68 sorting_label=sorting_label, 69 metrics=metrics, 70 curation_labels=curation_labels, 71 google_user_ids=google_user_ids) 73 # insert 74 key['workspace_uri'] = workspace_uri
File ~/spyglass/src/spyglass/spikesorting/sortingview_helper_fn.py:86, in _create_spikesortingview_workspace(recording_path, sorting_path, merge_groups, workspace_label, recording_label, sorting_label, metrics, google_user_ids, curation_labels) 84 if len(merge_groups) != 0: 85 sorting = MergedSortingExtractor(parent_sorting=sorting, merge_groups=merge_groups) ---> 86 sorting_id = workspace.add_sorting(recording_id=recording_id, label=sorting_label, sorting=sorting) 88 if metrics is not None: 89 workspace = _add_metrics_to_sorting_in_workspace(workspace=workspace, metrics=metrics, 90 sortingview_sorting_id=sorting_id)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/sortingview/workspace/Workspace.py:105, in Workspace.add_sorting(self, recording_id, label, sorting) 100 raise Exception(f'Duplicate sorting ID: {sorting_id}') 101 sv_recording_record = self._recording_records[recording_id] 102 x = { 103 'sortingId': sorting_id, 104 'sortingLabel': label, --> 105 'sortingPath': kcl.store_json(sorting_object, label=f'{label}.json'), 106 'sortingObject': sorting_object, 107 108 'recordingId': recording_id, 109 'recordingPath': sv_recording_record['recordingPath'], 110 'recordingObject': sv_recording_record['recordingObject'], 111 112 'description': f'Imported from Python: {label}' 113 } 114 print(f'Adding sorting: {sorting_id}') 115 self._feed.append_message({ 116 'action': { 117 'type': 'ADD_SORTING', 118 'sorting': x 119 } 120 })
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/kachery_cloud/core.py:21, in store_json(x, separators, indent, label, cache_locally) 19 import simplejson 20 text = simplejson.dumps(x, separators=separators, indent=indent, allow_nan=False, sort_keys=True) ---> 21 return store_text(text, label=label, cache_locally=cache_locally)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/kachery_cloud/core.py:16, in store_text(text, label, cache_locally) 14 with open(fname, 'w') as f: 15 f.write(text) ---> 16 return store_file(fname, label=label, cache_locally=cache_locally)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/kachery_cloud/store_file.py:31, in store_file(filename, label, cache_locally) 29 timer = time.time() 30 while True: ---> 31 response: dict = _kacherycloud_request(payload) 32 already_exists = response.get('alreadyExists', False) 33 already_pending = response.get('alreadyPending', False)
File ~/anaconda3/envs/spyglass-test/lib/python3.8/site-packages/kachery_cloud/_kacherycloud_request.py:27, in _kacherycloud_request(request_payload) 25 resp = requests.post(url, json=req) 26 if resp.status_code != 200: ---> 27 raise Exception(f'Error in {payload["type"]} ({resp.status_code}) {resp.reason}: {resp.text}') 28 response = resp.json() 29 return response
Exception: Error in initiateFileUpload (500) Internal Server Error: Error: Invalid timestamp. 1660195557819 1660195591411 33592
Exception in subscribe loop: HTTP Client Error (403): {"message":"Token is expired.","error":true,"service":"Access Manager","status":403}