We did some testing this morning and ran into a couple of issues on end point discoverability.
In the following situations--
AML workspace has a managed end point that is tagged Seeq=true and no other end points
AML workspace has a managed end point that is tagged Seeq=true and one other AKS end point that is not tagged Seeq=true
We get the following stack trace--
KeyError: 'scoringUri'
Error found at line 2 in cell 2.
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
/tmp/ipykernel_59/3894584444.py in <module>
1 C = azureml.MlOperate(sdl_notebook_url="https://chevron.seeq.site/workbook/2E3DBF0C-F2DD-4C01-94C3-75F50496B400/worksheet/AF5028E7-711B-4BB2-BC01-C0EB8214722C")
----> 2 C.run()
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/_add_on_main.py in run(self)
93 def run(self):
94 display(self.app)
---> 95 self.populate_available_aml_endpoints()
96 clear_output()
97 return self.app
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/_add_on_main.py in populate_available_aml_endpoints(self)
114
115 try:
--> 116 self.inputs_provider = ModelInputsProvider(config_file=self.config_file)
117 except AzureMLException as e:
118 self.set_spinner_message(title="Azure Exception", message=str(e), status="ERROR")
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_seeq_inputs_provider.py in __init__(self, config_file)
56 workspace_name=_config.get('azure', 'WORKSPACE_NAME'))
57
---> 58 self.endpoints = self.get_endpoints()
59 self.deployment = None
60 self.asset_paths = None
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_seeq_inputs_provider.py in get_endpoints(self)
68
69 def get_endpoints(self):
---> 70 names = [x.name for x in self.endpoint_svc.list_online_endpoints()]
71 renames = _rename_duplicates(names)
72 return dict(zip(renames, self.endpoint_svc.list_online_endpoints()))
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_aml_online_endpoint_service.py in list_online_endpoints(self)
278 to each object
279 """
--> 280 oes = self._get_unmanaged_online_endpoints()
281 oes += self._get_managed_online_endpoints()
282 return oes
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_aml_online_endpoint_service.py in _get_unmanaged_online_endpoints(self)
229 message="Error getting ACI endpoints")
230
--> 231 oes = OnlineEndpoint.deserialize_unmanaged_endpoint_response(response.json())
232 for oe in oes:
233 self._add_keys_to_endpoint(oe)
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_aml_response_models.py in deserialize_unmanaged_endpoint_response(json)
269 oe.type = "Microsoft.MachineLearningServices/workspaces/onlineEndpoints"
270 oe.description = v['description']
--> 271 oe.scoringUri = v['scoringUri']
272 oe.authMode = v['authEnabled']
273 oe.kind = 'ACI'
KeyError: 'scoringUri'
The only way that we've been able to get past this error is by having one aks end point that is not tagged Seeq=true, in which case we get the UI to load but obviously can't do much since there are no discoverable models.
Additionally, when we we have one aks end pt without the tag, and another aks end point that is correctly tagged, we get a different stack trace--
KeyError: 'location'
Error found at line 2 in cell 2.
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
/tmp/ipykernel_147/3894584444.py in <module>
1 C = azureml.MlOperate(sdl_notebook_url="https://chevron.seeq.site/workbook/2E3DBF0C-F2DD-4C01-94C3-75F50496B400/worksheet/AF5028E7-711B-4BB2-BC01-C0EB8214722C")
----> 2 C.run()
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/_add_on_main.py in run(self)
93 def run(self):
94 display(self.app)
---> 95 self.populate_available_aml_endpoints()
96 clear_output()
97 return self.app
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/_add_on_main.py in populate_available_aml_endpoints(self)
114
115 try:
--> 116 self.inputs_provider = ModelInputsProvider(config_file=self.config_file)
117 except AzureMLException as e:
118 self.set_spinner_message(title="Azure Exception", message=str(e), status="ERROR")
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_seeq_inputs_provider.py in __init__(self, config_file)
56 workspace_name=_config.get('azure', 'WORKSPACE_NAME'))
57
---> 58 self.endpoints = self.get_endpoints()
59 self.deployment = None
60 self.asset_paths = None
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_seeq_inputs_provider.py in get_endpoints(self)
68
69 def get_endpoints(self):
---> 70 names = [x.name for x in self.endpoint_svc.list_online_endpoints()]
71 renames = _rename_duplicates(names)
72 return dict(zip(renames, self.endpoint_svc.list_online_endpoints()))
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_aml_online_endpoint_service.py in list_online_endpoints(self)
278 to each object
279 """
--> 280 oes = self._get_unmanaged_online_endpoints()
281 oes += self._get_managed_online_endpoints()
282 return oes
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_aml_online_endpoint_service.py in _get_unmanaged_online_endpoints(self)
229 message="Error getting ACI endpoints")
230
--> 231 oes = OnlineEndpoint.deserialize_unmanaged_endpoint_response(response.json())
232 for oe in oes:
233 self._add_keys_to_endpoint(oe)
~/.local/lib/python3.8/site-packages/seeq/addons/azureml/backend/_aml_response_models.py in deserialize_unmanaged_endpoint_response(json)
274 oe.createdBy = v['createdBy']['userName']
275 oe.lastModifiedAt = v['updatedTime']
--> 276 oe.location = v['location']
277 od = OnlineDeployment(oe.name, None, oe.type)
278 od.location = oe.location
KeyError: 'location'
@cherrera2001, happy to hop on a call if you'd like to take a look at our environments, or if you need any additional information to reproduce this behavior.
We did some testing this morning and ran into a couple of issues on end point discoverability.
In the following situations--
We get the following stack trace--
The only way that we've been able to get past this error is by having one aks end point that is not tagged Seeq=true, in which case we get the UI to load but obviously can't do much since there are no discoverable models.
Additionally, when we we have one aks end pt without the tag, and another aks end point that is correctly tagged, we get a different stack trace--
@cherrera2001, happy to hop on a call if you'd like to take a look at our environments, or if you need any additional information to reproduce this behavior.