Unable to run the pipeline creation ipynb file

[Sundragon1993]

Hello @flopach , thank you so much for the fantastic work. I'm trying to run the digits_recognizer_pipeline.ipynb as your instruction however I'm facing this error, do you have any idea on this matter? The multipass VM is being used to create for deploying the microk8s:

---

ApiException Traceback (most recent call last) Cell In [8], line 12 9 run_directly = 1 11 if (rundirectly == 1): **---> 12 client.create_run_from_pipeline_func(output_test,arguments=arguments,experimentname="test")** 13 else: 14 kfp.compiler.Compiler().compile(pipeline_func=output_test,package_path='output_test.yaml')

File /opt/conda/lib/python3.8/site-packages/kfp/_client.py:781, in Client.create_run_from_pipeline_func(self, pipeline_func, arguments, run_name, experiment_name, pipeline_conf, namespace, mode, launcher_image, pipeline_root) 775 pipeline_package_path = os.path.join(tmpdir, 'pipeline.yaml') 776 compiler.Compiler(mode=mode, launcher_image=launcher_image).compile( 777 pipeline_func=pipeline_func, 778 package_path=pipeline_package_path, 779 pipeline_conf=pipeline_conf) --> 781 return self.create_run_from_pipeline_package( 782 pipeline_file=pipeline_package_path, 783 arguments=arguments, 784 run_name=run_name, 785 experiment_name=experiment_name, 786 namespace=namespace, 787 pipeline_root=pipeline_root, 788 )

File /opt/conda/lib/python3.8/site-packages/kfp/_client.py:842, in Client.create_run_from_pipeline_package(self, pipeline_file, arguments, run_name, experiment_name, namespace, pipeline_root) 838 experiment_name = overridden_experiment_name or 'Default' 839 run_name = run_name or (pipeline_name + ' ' + 840 datetime.datetime.now().strftime( 841 '%Y-%m-%d %H-%M-%S')) --> 842 experiment = self.create_experiment(name=experiment_name, namespace=namespace) 843 run_info = self.run_pipeline( 844 experiment_id=experiment.id, 845 job_name=run_name, 846 pipeline_package_path=pipeline_file, 847 params=arguments, 848 pipeline_root=pipeline_root) 849 return RunPipelineResult(self, run_info)

File /opt/conda/lib/python3.8/site-packages/kfp/_client.py:366, in Client.create_experiment(self, name, description, namespace) 364 experiment = None 365 try: --> 366 experiment = self.get_experiment(experiment_name=name, namespace=namespace) 367 except ValueError as error: 368 # Ignore error if the experiment does not exist. 369 if not str(error).startswith('No experiment is found with name'):

File /opt/conda/lib/python3.8/site-packages/kfp/_client.py:478, in Client.get_experiment(self, experiment_id, experiment_name, namespace) 468 experiment_filter = json.dumps({ 469 "predicates": [ 470 { (...) 475 ] 476 }) 477 if namespace: --> 478 result = self._experiment_api.list_experiment( 479 filter=experiment_filter, 480 resource_reference_key_type=kfp_server_api.models.api_resource_type.ApiResourceType.NAMESPACE, 481 resource_reference_key_id=namespace) 482 else: 483 result = self._experiment_api.list_experiment(filter=experiment_filter)

File /opt/conda/lib/python3.8/site-packages/kfp_server_api/api/experiment_service_api.py:567, in ExperimentServiceApi.list_experiment(self, kwargs) 532 """Finds all experiments. Supports pagination, and sorting on certain fields. # noqa: E501 533 534 This method makes a synchronous HTTP request by default. To make an (...) 564 :rtype: ApiListExperimentsResponse 565 """ 566 kwargs['_return_http_data_only'] = True --> 567 return self.list_experiment_with_http_info(kwargs)

File /opt/conda/lib/python3.8/site-packages/kfp_server_api/api/experiment_service_api.py:668, in ExperimentServiceApi.list_experiment_with_http_info(self, **kwargs) 665 # Authentication setting 666 auth_settings = ['Bearer'] # noqa: E501 --> 668 return self.api_client.call_api( 669 '/apis/v1beta1/experiments', 'GET', 670 path_params, 671 query_params, 672 header_params, 673 body=body_params, 674 post_params=form_params, 675 files=local_var_files, 676 response_type='ApiListExperimentsResponse', # noqa: E501 677 auth_settings=auth_settings, 678 async_req=local_var_params.get('async_req'), 679 _return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501 680 _preload_content=local_var_params.get('_preload_content', True), 681 _request_timeout=local_var_params.get('_request_timeout'), 682 collection_formats=collection_formats)

File /opt/conda/lib/python3.8/site-packages/kfp_server_api/api_client.py:364, in ApiClient.call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, async_req, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host) 327 """Makes the HTTP request (synchronous) and returns deserialized data. 328 329 To make an async_req request, set the async_req parameter. (...) 361 then the method will return the response directly. 362 """ 363 if not async_req: --> 364 return self.__call_api(resource_path, method, 365 path_params, query_params, header_params, 366 body, post_params, files, 367 response_type, auth_settings, 368 _return_http_data_only, collection_formats, 369 _preload_content, _request_timeout, _host) 371 return self.pool.apply_async(self.__call_api, (resource_path, 372 method, path_params, 373 query_params, (...) 381 _request_timeout, 382 _host))

File /opt/conda/lib/python3.8/site-packages/kfp_server_api/api_client.py:188, in ApiClient.__call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host) 186 except ApiException as e: 187 e.body = e.body.decode('utf-8') if six.PY3 else e.body --> 188 raise e 190 content_type = response_data.getheader('content-type') 192 self.last_response = response_data

File /opt/conda/lib/python3.8/site-packages/kfp_server_api/api_client.py:181, in ApiClient.__call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host) 177 url = _host + resource_path 179 try: 180 # perform request and return response --> 181 response_data = self.request( 182 method, url, query_params=query_params, headers=header_params, 183 post_params=post_params, body=body, 184 _preload_content=_preload_content, 185 _request_timeout=_request_timeout) 186 except ApiException as e: 187 e.body = e.body.decode('utf-8') if six.PY3 else e.body

File /opt/conda/lib/python3.8/site-packages/kfp_server_api/api_client.py:389, in ApiClient.request(self, method, url, query_params, headers, post_params, body, _preload_content, _request_timeout) 387 """Makes the HTTP request using RESTClient.""" 388 if method == "GET": --> 389 return self.rest_client.GET(url, 390 query_params=query_params, 391 _preload_content=_preload_content, 392 _request_timeout=_request_timeout, 393 headers=headers) 394 elif method == "HEAD": 395 return self.rest_client.HEAD(url, 396 query_params=query_params, 397 _preload_content=_preload_content, 398 _request_timeout=_request_timeout, 399 headers=headers)

File /opt/conda/lib/python3.8/site-packages/kfp_server_api/rest.py:230, in RESTClientObject.GET(self, url, headers, query_params, _preload_content, _request_timeout) 228 def GET(self, url, headers=None, query_params=None, _preload_content=True, 229 _request_timeout=None): --> 230 return self.request("GET", url, 231 headers=headers, 232 _preload_content=_preload_content, 233 _request_timeout=_request_timeout, 234 query_params=query_params)

File /opt/conda/lib/python3.8/site-packages/kfp_server_api/rest.py:224, in RESTClientObject.request(self, method, url, query_params, headers, body, post_params, _preload_content, _request_timeout) 221 logger.debug("response body: %s", r.data) _223 if not 200 <= r.status <= 299: --> 224 raise ApiException(httpresp=r) 226 return r

ApiException: (500) Reason: Internal Server Error HTTP response headers: HTTPHeaderDict({'content-type': 'application/json', 'date': 'Tue, 08 Nov 2022 07:28:22 GMT', 'x-envoy-upstream-service-time': '2', 'server': 'envoy', 'transfer-encoding': 'chunked'}) HTTP response body: {"error":"Internal error: Unauthenticated: Request header error: there is no user identity header.: Request header error: there is no user identity header.\nFailed to authorize with API resource references\ngithub.com/kubeflow/pipelines/backend/src/common/util.Wrap\n\t/go/src/github.com/kubeflow/pipelines/backend/src/common/util/error.go:287\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).canAccessExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:249\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).ListExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:148\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler.func1\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1089\nmain.apiServerInterceptor\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/interceptor.go:30\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1091\ngoogle.golang.org/grpc.(Server).processUnaryRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1282\ngoogle.golang.org/grpc.(Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1616\ngoogle.golang.org/grpc.(Server).serveStreams.func1.2\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:921\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581\nFailed to authorize with API resource references\ngithub.com/kubeflow/pipelines/backend/src/common/util.Wrap\n\t/go/src/github.com/kubeflow/pipelines/backend/src/common/util/error.go:287\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).ListExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:150\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler.func1\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1089\nmain.apiServerInterceptor\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/interceptor.go:30\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1091\ngoogle.golang.org/grpc.(Server).processUnaryRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1282\ngoogle.golang.org/grpc.(Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1616\ngoogle.golang.org/grpc.(Server).serveStreams.func1.2\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:921\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581","code":13,"message":"Internal error: Unauthenticated: Request header error: there is no user identity header.: Request header error: there is no user identity header.\nFailed to authorize with API resource references\ngithub.com/kubeflow/pipelines/backend/src/common/util.Wrap\n\t/go/src/github.com/kubeflow/pipelines/backend/src/common/util/error.go:287\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).canAccessExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:249\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).ListExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:148\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler.func1\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1089\nmain.apiServerInterceptor\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/interceptor.go:30\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1091\ngoogle.golang.org/grpc.(Server).processUnaryRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1282\ngoogle.golang.org/grpc.(Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1616\ngoogle.golang.org/grpc.(Server).serveStreams.func1.2\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:921\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581\nFailed to authorize with API resource references\ngithub.com/kubeflow/pipelines/backend/src/common/util.Wrap\n\t/go/src/github.com/kubeflow/pipelines/backend/src/common/util/error.go:287\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).ListExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:150\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler.func1\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1089\nmain.apiServerInterceptor\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/interceptor.go:30\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1091\ngoogle.golang.org/grpc.(Server).processUnaryRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1282\ngoogle.golang.org/grpc.(Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1616\ngoogle.golang.org/grpc.(Server).serveStreams.func1.2\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:921\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581","details":[{"@type":"type.googleapis.com/api.Error","error_message":"Internal error: Unauthenticated: Request header error: there is no user identity header.: Request header error: there is no user identity header.\nFailed to authorize with API resource references\ngithub.com/kubeflow/pipelines/backend/src/common/util.Wrap\n\t/go/src/github.com/kubeflow/pipelines/backend/src/common/util/error.go:287\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).canAccessExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:249\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).ListExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:148\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler.func1\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1089\nmain.apiServerInterceptor\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/interceptor.go:30\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1091\ngoogle.golang.org/grpc.(Server).processUnaryRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1282\ngoogle.golang.org/grpc.(Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1616\ngoogle.golang.org/grpc.(Server).serveStreams.func1.2\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:921\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581\nFailed to authorize with API resource references\ngithub.com/kubeflow/pipelines/backend/src/common/util.Wrap\n\t/go/src/github.com/kubeflow/pipelines/backend/src/common/util/error.go:287\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).ListExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:150\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler.func1\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1089\nmain.apiServerInterceptor\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/interceptor.go:30\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1091\ngoogle.golang.org/grpc.(Server).processUnaryRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1282\ngoogle.golang.org/grpc.(Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1616\ngoogle.golang.org/grpc.(Server).serveStreams.func1.2\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:921\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581","error_details":"Internal error: Unauthenticated: Request header error: there is no user identity header.: Request header error: there is no user identity header.\nFailed to authorize with API resource references\ngithub.com/kubeflow/pipelines/backend/src/common/util.Wrap\n\t/go/src/github.com/kubeflow/pipelines/backend/src/common/util/error.go:287\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).canAccessExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:249\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).ListExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:148\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler.func1\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1089\nmain.apiServerInterceptor\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/interceptor.go:30\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1091\ngoogle.golang.org/grpc.(Server).processUnaryRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1282\ngoogle.golang.org/grpc.(Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1616\ngoogle.golang.org/grpc.(Server).serveStreams.func1.2\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:921\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581\nFailed to authorize with API resource references\ngithub.com/kubeflow/pipelines/backend/src/common/util.Wrap\n\t/go/src/github.com/kubeflow/pipelines/backend/src/common/util/error.go:287\ngithub.com/kubeflow/pipelines/backend/src/apiserver/server.(ExperimentServer).ListExperiment\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/server/experiment_server.go:150\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler.func1\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1089\nmain.apiServerInterceptor\n\t/go/src/github.com/kubeflow/pipelines/backend/src/apiserver/interceptor.go:30\ngithub.com/kubeflow/pipelines/backend/api/go_client._ExperimentService_ListExperiment_Handler\n\t/go/src/github.com/kubeflow/pipelines/backend/api/go_client/experiment.pb.go:1091\ngoogle.golang.org/grpc.(Server).processUnaryRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1282\ngoogle.golang.org/grpc.(Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:1616\ngoogle.golang.org/grpc.(Server).serveStreams.func1.2\n\t/go/pkg/mod/google.golang.org/grpc@v1.44.0/server.go:921\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581"}]}

[flopach]

Hi, I would check if all Kubeflow related python libraries are up to date and if you actually can access the Kubeflow Pipeline APIs from the Jupyter Notebook.

[Sundragon1993]

@flopach Hi, thank you for the instruction, I've solved the issue by myself. The pipeline was successfully created and compiled, however, I'm stuck with the Kserve. When I tried to show all InferenceService by this command, it returned nothing:

I tried to debug and watching the events by:

microk8s kubectl get pod -n admin NAME READY STATUS RESTARTS AGE ml-pipeline-ui-artifact-6c696b6499-s2fms 2/2 Running 0 9d ml-pipeline-visualizationserver-795f7db965-2w2dv 2/2 Running 0 9d facadeimmortal2-0 2/2 Running 0 159m

`microk8s kubectl get events -n admin LAST SEEN TYPE REASON OBJECT MESSAGE

11m Warning InternalError inferenceservice/digits-recognizer-2022-11-11--06-21-49 No ServingRuntimes or ClusterServingRuntimes with the name: kserve-tensorflow-serving

9m16s Warning InternalError inferenceservice/digits-recognizer-2022-11-14--02-00-41 No ServingRuntimes or ClusterServingRuntimes with the name: kserve-tensorflow-serving `

Do you have any ideas?

[flopach]

@Sundragon1993, yeah you need to be careful there. I would check with this Python script if the Kserve instance actually works (from the Jupyter Notebook): https://github.com/flopach/digits-recognizer-kubeflow/blob/master/kserve_python_test.ipynb

Here is the Kserve troubleshoot guide which helps als well: https://kserve.github.io/website/developer/debug/

Double check if your model (Tensorflow saved model, not the h5 file!) gets picked up from minio and the version name (e.g. "/1/" is set like I did in my script.

[Sundragon1993]

@flopach Thank you for the comments, I've made the inference service online now, however, I couldn't use it as the request to the isvc is always not responding...

ubuntu@master-node-11:~$ **microk8s kubectl describe inferenceservice digits-recognizer-svc -n admin**
Name:         digits-recognizer-svc
Namespace:    admin
Labels:       <none>
Annotations:  sidecar.istio.io/inject: false
API Version:  serving.kserve.io/v1beta1
Kind:         InferenceService
Metadata:
  Creation Timestamp:  2022-11-18T09:57:16Z
  Finalizers:
    inferenceservice.finalizers
  Generation:  1
  Managed Fields:
    API Version:  serving.kserve.io/v1beta1
    Fields Type:  FieldsV1
    fieldsV1:
      f:metadata:
        f:annotations:
          .:
          f:sidecar.istio.io/inject:
      f:spec:
        .:
        f:predictor:
          .:
          f:minReplicas:
          f:tensorflow:
            .:
            f:name:
            f:resources:
              .:
              f:limits:
                .:
                f:cpu:
                f:memory:
              f:requests:
                .:
                f:cpu:
                f:memory:
            f:storageUri:
    Manager:      OpenAPI-Generator
    Operation:    Update
    Time:         2022-11-18T09:57:14Z
    API Version:  serving.kserve.io/v1beta1
    Fields Type:  FieldsV1
    fieldsV1:
      f:metadata:
        f:finalizers:
          .:
          v:"inferenceservice.finalizers":
      f:status:
        .:
        f:address:
          .:
          f:url:
        f:components:
          .:
          f:predictor:
            .:
            f:latestCreatedRevision:
            f:url:
        f:conditions:
        f:url:
    Manager:         manager
    Operation:       Update
    Time:            2022-11-18T09:57:59Z
  Resource Version:  627143
  Self Link:         /apis/serving.kserve.io/v1beta1/namespaces/admin/inferenceservices/digits-recognizer-svc
  UID:               e9df91e9-6e3f-4628-b2c4-9eaf9ecde5a1
Spec:
  Predictor:
    Min Replicas:  1
    Model:
      Model Format:
        Name:            tensorflow
      Name:              
      Protocol Version:  v1
      Resources:
        Limits:
          Cpu:     100m
          Memory:  256Mi
        Requests:
          Cpu:      10m
          Memory:   128Mi
      Runtime:      kserve-tensorflow-serving
      Storage Uri:  gs://kfserving-examples/models/tensorflow/flowers
Status:
  Address:
    URL:  http://digits-recognizer-svc.admin.svc.cluster.local
  Components:
    Predictor:
      Latest Created Revision:  1
      URL:                      http://digits-recognizer-svc-predictor-default-admin.example.com
  Conditions:
    Last Transition Time:  2022-11-18T09:57:59Z
    Status:                True
    Type:                  IngressReady
    Last Transition Time:  2022-11-18T09:57:59Z
    Status:                True
    Type:                  PredictorReady
    Last Transition Time:  2022-11-18T09:57:59Z
    Status:                True
    Type:                  Ready
  URL:                     http://digits-recognizer-svc-admin.example.com
Events:                    <none>

ubuntu@master-node-11:~$ **microk8s kubectl get inferenceservice -n admin**
NAME                    URL                                                                          READY   PREV   LATEST   PREVROLL      AGE
digits-recognizer-svc   http://digits-recognizer-svc-admin.example.com   True      
```                                                            2d16h

ubuntu@master-node-11:~$ microk8s kubectl get ingress --all-namespaces NAMESPACE NAME CLASS HOSTS ADDRESS PORTS AGE admin digits-recognizer-svc digits-recognizer-svc-admin.example.com,digits-recognizer-svc-predictor-default-admin.example.com 80 2d16h

I've tried to access to the host both by **curl and code:**

def predict_str(service_name, input_json, protocol_version="v1",
                version=constants.KSERVE_V1BETA1_VERSION, model_name=None):
        kfs_client = KServeClient()
        isvc = kfs_client.get(
            service_name,
            namespace=KSERVE_NAMESPACE,
            version=version,
        )
        print("+++++++++++GOT the isvc++++++++++++")
        # temporary sleep until this is fixed https://github.com/kserve/kserve/issues/604
        time.sleep(10)

        host = urlparse(isvc["status"]["url"]).netloc
        headers = {"Host": host, "Content-Type": "application/json"}

        print(f"+++++++++++GOT the host {host}++++++++++++")

        if model_name is None:
            model_name = service_name

        cluster_ip = get_cluster_ip()
        print("+++++++++++GOT the CLUSTER IP++++++++++++")

        url = f"http://{cluster_ip}/v1/models/{model_name}:predict"
        if protocol_version == "v2":
            url = f"http://{cluster_ip}/v2/models/{model_name}/infer"

        logging.info("Sending Header = %s", headers)
        logging.info("Sending url = %s", url)
        logging.info("Sending request data: %s", input_json)
        response = requests.post(url, input_json, headers=headers)
        logging.info("Got response code %s, content %s", response.status_code, response.content)
        preds = json.loads(response.content.decode("utf-8"))
        return preds

by curl:

ubuntu@master-node-11:~$ curl -v http://digits-recognizer-svc-predictor-default-admin.example.com:predict -d flower.json
* Closing connection -1
curl: (3) URL using bad/illegal format or missing URL
ubuntu@master-node-11:~$ ping digits-recognizer-svc-predictor-default-admin.example.com
ping: digits-recognizer-svc-predictor-default-admin.example.com: Name or service not known
ubuntu@master-node-11:~$ curl -v http://digits-recognizer-svc.default/v1/models/digits-recognizer-svc:predict -d flower.json
* Could not resolve host: digits-recognizer-svc.default
* Closing connection 0
curl: (6) Could not resolve host: digits-recognizer-svc.default
ubuntu@master-node-11:~$ curl -v http://digits-recognizer-svc-predictor.default/v1/models/digits-recognizer-svc:predict -d flower.json
* Could not resolve host: digits-recognizer-svc-predictor.default
* Closing connection 0
curl: (6) Could not resolve host: digits-recognizer-svc-predictor.default
ubuntu@master-node-11:~$ curl -v http://digits-recognizer-svc-predictor.admin/v1/models/digits-recognizer-svc:predict -d flower.json
* Could not resolve host: digits-recognizer-svc-predictor.admin
* Closing connection 0
curl: (6) Could not resolve host: digits-recognizer-svc-predictor.admin
ubuntu@master-node-11:~$ curl -v http://digits-recognizer-svc-predictor.admin/v1/models/digits-recognizer-svc-predictor:predict -d flower.json
* Could not resolve host: digits-recognizer-svc-predictor.admin
* Closing connection 0
curl: (6) Could not resolve host: digits-recognizer-svc-predictor.admin
ubuntu@master-node-11:~$ curl -v http://10.152.183.90/v1/models//digits-recognizer-svc-predictor:predict -d flower.json
*   Trying 10.152.183.90:80...
^C
ubuntu@master-node-11:~$ curl -v http://10.152.183.90/v1/models/digits-recognizer-svc-predictor:predict -d flower.json
*   Trying 10.152.183.90:80...
^C

Your feedback is highly appreciated...