Unable to add endpoint using https and this URL is Knox

[srujanattunuri]

Using Livy Latest Version. When i tried to do curl request and it's connecting to livy but through sparkmagic it's not going through. Even i tried changing the igone_ssl_errors in config.json

Added endpoint https://xxxxxxx:8443/gateway/sandbox/livy/v1/

HttpClientException Traceback (most recent call last) /opt/anaconda3/envs/p3env/lib/python3.6/site-packages/hdijupyterutils/ipywidgetfactory.py in submit_clicked(self, button) 63 64 def submit_clicked(self, button): ---> 65 self.parent_widget.run()

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/controllerwidget/addendpointwidget.py in run(self) 63 # We need to call the refresh method because drop down in Tab 2 for endpoints wouldn't refresh with the new 64 # value otherwise. ---> 65 self.refresh_method() 66 67 def _show_correct_endpoint_fields(self):

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/controllerwidget/magicscontrollerwidget.py in _refresh(self) 60 self.endpoints, self.endpoints_dropdown_widget, self._refresh) 61 self.manage_endpoint = ManageEndpointWidget(self.spark_controller, self.ipywidget_factory, self.ipython_display, ---> 62 self.endpoints, self._refresh) 63 64 self.tabs = self.ipywidget_factory.get_tab(children=[self.manage_session, self.create_session,

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/controllerwidget/manageendpointwidget.py in init(self, spark_controller, ipywidget_factory, ipython_display, endpoints, refresh_method) 15 self.refresh_method = refresh_method 16 ---> 17 self.children = self.get_existing_endpoint_widgets() 18 19 for child in self.children:

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/controllerwidget/manageendpointwidget.py in get_existing_endpoint_widgets(self) 35 for url, endpoint in self.endpoints.items(): 36 try: ---> 37 endpoint_widgets.append(self.get_endpoint_widget(url, endpoint)) 38 except HttpClientException: 39 # If we can't reach one of the default endpoints, just skip over it

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/controllerwidget/manageendpointwidget.py in get_endpoint_widget(self, url, endpoint) 58 hbox_outter_children = [] 59 try: ---> 60 vbox_left = self.get_endpoint_left(endpoint, url) 61 cleanup_w = self.get_cleanup_button_endpoint(url, endpoint) 62

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/controllerwidget/manageendpointwidget.py in get_endpoint_left(self, endpoint, url) 75 def get_endpoint_left(self, endpoint, url): 76 # 400 px ---> 77 info = self.get_info_endpoint_widget(endpoint, url) 78 delete_session_number = self.get_delete_session_endpoint_widget(url, endpoint) 79 vbox_left = self.ipywidget_factory.get_vbox(children=[info, delete_session_number], width="400px")

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/controllerwidget/manageendpointwidget.py in get_info_endpoint_widget(self, endpoint, url) 127 width = "400px" 128 --> 129 info_sessions = self.spark_controller.get_all_sessions_endpoint_info(endpoint) 130 131 if len(info_sessions) > 0:

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/livyclientlib/sparkcontroller.py in get_all_sessions_endpoint_info(self, endpoint) 51 52 def get_all_sessions_endpoint_info(self, endpoint): ---> 53 sessions = self.get_all_sessions_endpoint(endpoint) 54 return [str(s) for s in sessions] 55

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/livyclientlib/sparkcontroller.py in get_all_sessions_endpoint(self, endpoint) 42 def get_all_sessions_endpoint(self, endpoint): 43 http_client = self._http_client(endpoint) ---> 44 sessions = http_client.get_sessions()[u"sessions"] 45 session_list = [self._livy_session(http_client, {constants.LIVY_KIND_PARAM: s[constants.LIVY_KIND_PARAM]}, 46 self.ipython_display, s[u"id"])

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/livyclientlib/livyreliablehttpclient.py in get_sessions(self) 31 32 def get_sessions(self): ---> 33 return self._http_client.get("/sessions", [200]).json() 34 35 def post_session(self, properties):

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/livyclientlib/reliablehttpclient.py in get(self, relative_url, accepted_status_codes) 44 def get(self, relative_url, accepted_status_codes): 45 """Sends a get request. Returns a response.""" ---> 46 return self._send_request(relative_url, accepted_status_codes, requests.get) 47 48 def post(self, relative_url, accepted_status_codes, data):

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/livyclientlib/reliablehttpclient.py in _send_request(self, relative_url, accepted_status_codes, function, data) 55 56 def _send_request(self, relative_url, accepted_status_codes, function, data=None): ---> 57 return self._send_request_helper(self.compose_url(relative_url), accepted_status_codes, function, data, 0) 58 59 def _send_request_helper(self, url, accepted_status_codes, function, data, retry_count):

/opt/anaconda3/envs/p3env/lib/python3.6/site-packages/sparkmagic/livyclientlib/reliablehttpclient.py in _send_request_helper(self, url, accepted_status_codes, function, data, retry_count) 90 91 if error: ---> 92 raise HttpClientException(u"Error sending http request and maximum retry encountered.") 93 else: 94 raise HttpClientException(u"Invalid status code '{}' from {} with error payload: {}"

HttpClientException: Error sending http request and maximum retry encountered.

[srujanattunuri]

Sparkmagic version: 0.13.1 Livy : 0.6

[amacaskill]

Did you ever get this to work?

[srujanattunuri]

No, It didn't work and reported as bug

[NixTheMan]

Hi I am running into the same bug: I am running: sparkmagic 0.17.1 livy v0.7.0 pyspark 2.4.6