search

quanted / cts_app

cts repo
3 stars 2 forks source link

Envipath issues #206

Open popenc opened 1 year ago

popenc commented 1 year ago

Envipath doesn't seem to be running on any server.

envipath | WARNING:root:HTTPSConnectionPool(host='envipath.org', port=443): Max retries exceeded with url: /package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway/43ac0b09-dd78-43dd-8ffd-6e414d8d8e30 (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1123)'))) envipath | ERROR:waitress:Exception while serving /envipath/rest/run envipath | Traceback (most recent call last): envipath | File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 699, in urlopen envipath | httplib_response = self._make_request( envipath | File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 382, in _make_request envipath | self._validate_conn(conn) envipath | File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1010, in _validate_conn envipath | conn.connect() envipath | File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 411, in connect envipath | self.sock = ssl_wrapsocket( envipath | File "/usr/local/lib/python3.9/site-packages/urllib3/util/ssl.py", line 428, in ssl_wrap_socket envipath | ssl_sock = _ssl_wrap_socketimpl( envipath | File "/usr/local/lib/python3.9/site-packages/urllib3/util/ssl.py", line 472, in _ssl_wrap_socket_impl envipath | return ssl_context.wrap_socket(sock, server_hostname=server_hostname) envipath | File "/usr/local/lib/python3.9/ssl.py", line 500, in wrap_socket envipath | return self.sslsocket_class._create( envipath | File "/usr/local/lib/python3.9/ssl.py", line 1040, in _create envipath | self.do_handshake() envipath | File "/usr/local/lib/python3.9/ssl.py", line 1309, in do_handshake envipath | self._sslobj.do_handshake() envipath | ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1123) envipath | envipath | During handling of the above exception, another exception occurred: envipath | envipath | Traceback (most recent call last): envipath | File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 439, in send envipath | resp = conn.urlopen( envipath | File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 755, in urlopen envipath | retries = retries.increment( envipath | File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 574, in increment envipath | raise MaxRetryError(_pool, url, error or ResponseError(cause)) envipath | urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='envipath.org', port=443): Max retries exceeded with url: /package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway/43ac0b09-dd78-43dd-8ffd-6e414d8d8e30 (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1123)'))) envipath | envipath | During handling of the above exception, another exception occurred: envipath | envipath | Traceback (most recent call last): envipath | File "/src/cts_envipath.py", line 74, in get_envipath_tree envipath | json_retval = pw.get_json() envipath | File "/src/enviPath_python/objects.py", line 117, in get_json envipath | return self.requester.get_json(self.id) envipath | File "/src/enviPath_python/enviPath.py", line 266, in get_json envipath | return self.get_request(envipath_id).json() envipath | File "/src/enviPath_python/enviPath.py", line 225, in get_request envipath | return self._request('GET', url, params, payload, kwargs) envipath | File "/src/enviPath_python/enviPath.py", line 256, in _request envipath | response = self.session.request(method, url, params=params, data=payload, headers=self.header, kwargs) envipath | File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 542, in request envipath | resp = self.send(prep, send_kwargs) envipath | File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 655, in send envipath | r = adapter.send(request, kwargs) envipath | File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 514, in send envipath | raise SSLError(e, request=request) envipath | requests.exceptions.SSLError: HTTPSConnectionPool(host='envipath.org', port=443): Max retries exceeded with url: /package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway/43ac0b09-dd78-43dd-8ffd-6e414d8d8e30 (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1123)'))) envipath | envipath | During handling of the above exception, another exception occurred: envipath | envipath | Traceback (most recent call last): envipath | File "/src/cts_envipath.py", line 115, in get_envipath_tree envipath | return_val = json.dumps(err_msg) envipath | File "/usr/local/lib/python3.9/json/init.py", line 231, in dumps envipath | return _default_encoder.encode(obj) envipath | File "/usr/local/lib/python3.9/json/encoder.py", line 199, in encode envipath | chunks = self.iterencode(o, _one_shot=True) envipath | File "/usr/local/lib/python3.9/json/encoder.py", line 257, in iterencode envipath | return _iterencode(o, 0) envipath | File "/usr/local/lib/python3.9/json/encoder.py", line 179, in default envipath | raise TypeError(f'Object of type {o.class.name} ' envipath | TypeError: Object of type MaxRetryError is not JSON serializable envipath | envipath | During handling of the above exception, another exception occurred: envipath | envipath | Traceback (most recent call last): envipath | File "/usr/local/lib/python3.9/site-packages/waitress/channel.py", line 397, in service envipath | task.service() envipath | File "/usr/local/lib/python3.9/site-packages/waitress/task.py", line 168, in service envipath | self.execute() envipath | File "/usr/local/lib/python3.9/site-packages/waitress/task.py", line 434, in execute envipath | app_iter = self.channel.server.application(environ, start_response) envipath | File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 2088, in call envipath | return self.wsgi_app(environ, start_response) envipath | File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 2073, in wsgi_app envipath | response = self.handle_exception(e) envipath | File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 2070, in wsgi_app envipath | response = self.full_dispatch_request() envipath | File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 1515, in full_dispatch_request envipath | rv = self.handle_user_exception(e) envipath | File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 1513, in full_dispatch_request envipath | rv = self.dispatch_request() envipath | File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 1499, in dispatch_request envipath | return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args) envipath | File "/src/cts_envipath_flask.py", line 49, in run_envipath envipath | tree_dict = ctsenvipath.get_envipath_tree(smiles, gen_limit) envipath | File "/src/cts_envipath.py", line 118, in get_envipath_tree envipath | return return_val envipath | UnboundLocalError: local variable 'return_val' referenced before assignment

popenc commented 1 year ago

Added "verify=False" to enviPath.py _request() function to resolve "SSL: CERTIFICATE_VERIFY_FAILED," but now requests are taking forever with no end in sight. Here's a sample:

envipath    | DEBUG:urllib3.connectionpool:https://envipath.org:443 "GET /package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway/9cc12b1e-5a17-430a-a478-e6cec336ece7 HTTP/1.1" 200 None
envipath    | DEBUG:urllib3.connectionpool:Resetting dropped connection: envipath.org
envipath    | /usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py:1013: InsecureRequestWarning: Unverified HTTPS request is being made to host 'envipath.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
envipath    |   warnings.warn(
envipath    | DEBUG:urllib3.connectionpool:https://envipath.org:443 "GET /package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway/9cc12b1e-5a17-430a-a478-e6cec336ece7 HTTP/1.1" 200 None
envipath    | DEBUG:urllib3.connectionpool:Resetting dropped connection: envipath.org
envipath    | /usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py:1013: InsecureRequestWarning: Unverified HTTPS request is being made to host 'envipath.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
envipath    |   warnings.warn(
envipath    | DEBUG:urllib3.connectionpool:https://envipath.org:443 "GET /package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway/9cc12b1e-5a17-430a-a478-e6cec336ece7 HTTP/1.1" 200 None
envipath    | DEBUG:urllib3.connectionpool:Resetting dropped connection: envipath.org
envipath    | /usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py:1013: InsecureRequestWarning: Unverified HTTPS request is being made to host 'envipath.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
envipath    |   warnings.warn(
envipath    | DEBUG:urllib3.connectionpool:https://envipath.org:443 "GET /package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway/9cc12b1e-5a17-430a-a478-e6cec336ece7 HTTP/1.1" 200 None
envipath    | DEBUG:urllib3.connectionpool:Resetting dropped connection: envipath.org
envipath    | /usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py:1013: InsecureRequestWarning: Unverified HTTPS request is being made to host 'envipath.org'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
envipath    |   warnings.warn(
envipath    | DEBUG:urllib3.connectionpool:https://envipath.org:443 "GET /package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway/9cc12b1e-5a17-430a-a478-e6cec336ece7 HTTP/1.1" 200 None
popenc commented 1 year ago

Looks like envipath is still having issues.

popenc commented 1 year ago

Seems to still be an issue, at least still unable to get data from CTS calling envipath.

popenc commented 1 year ago

Looks like envipath's site is returning data now, so the next step is determining why data isn't coming back on ceam dev.

popenc commented 1 year ago

Although their website is returning data, CTS is getting an error response from the envipath API:

Incoming data to recurse: {'data': {'error': '400 Client Error: 400 for url: https://envipath.org/package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway'}, 'status': True}
worker_cts_1      | [2023-02-01 18:53:03,461: INFO/ForkPoolWorker-2] metabolites: 1
worker_cts_1      | [2023-02-01 18:53:03,461: WARNING/ForkPoolWorker-2] root: {'error': '400 Client Error: 400 for url: https://envipath.org/package/650babc9-9d68-4b73-9332-11972ca26f7b/pathway'}
worker_cts_1      | [2023-02-01 18:53:03,465: ERROR/ForkPoolWorker-2] Task tasks.cts_task[a41b7fdd-07be-43cb-8b98-82b342375eeb] raised unexpected: KeyError('smiles')
worker_cts_1      | Traceback (most recent call last):
worker_cts_1      |   File "/usr/local/lib/python3.9/site-packages/celery/app/trace.py", line 451, in trace_task
worker_cts_1      |     R = retval = fun(*args, **kwargs)
worker_cts_1      |   File "/usr/local/lib/python3.9/site-packages/celery/app/trace.py", line 734, in __protected_call__
worker_cts_1      |     return self.run(*args, **kwargs)
worker_cts_1      |   File "/src/tasks.py", line 68, in cts_task
worker_cts_1      |     task_obj.initiate_requests_parsing(request_post)
worker_cts_1      |   File "/src/tasks.py", line 229, in initiate_requests_parsing
worker_cts_1      |     self.parse_by_service(request_post.get('sessionid'), request_post)
worker_cts_1      |   File "/src/tasks.py", line 246, in parse_by_service
worker_cts_1      |     _results = self.envipath_calc.data_request_handler(request_post)
worker_cts_1      |   File "/src/cts_calcs/calculator_envipath.py", line 148, in data_request_handler
worker_cts_1      |     _results = self.recursive(json.loads(response.content), int(request_dict['gen_limit']), True)
worker_cts_1      |   File "/src/cts_calcs/calculator_envipath.py", line 58, in recursive
worker_cts_1      |     'tree': self.traverse(root, gen_limit, unranked),
worker_cts_1      |   File "/src/cts_calcs/calculator_envipath.py", line 87, in traverse
worker_cts_1      |     _parent = root['smiles']
worker_cts_1      | KeyError: 'smiles'
popenc commented 1 year ago

Kurt's getting the same error locally (400 Client Error) and is reaching out to the envipath folks.