search

jupyterhub / kubespawner

Kubernetes spawner for JupyterHub
https://jupyterhub-kubespawner.readthedocs.io
BSD 3-Clause "New" or "Revised" License
531 stars 299 forks source link

certificate verify failed: self signed certificate #799

Closed devenami closed 8 months ago

devenami commented 8 months ago

Bug description

k8s client connect to apiserver error. certificate verify failed: self signed certificate

Can we verify_ssl a verify attribute to skip certificate checking in clients.py？

# clients.py
@lru_cache()
def load_config(host=None, ssl_ca_cert=None, verify_ssl=None):
    """
    Loads global configuration for the Python client we use to communicate with
    a Kubernetes API server, and optionally tweaks that configuration based on
    specific settings on the passed caller object.

    This needs to be called before creating a kubernetes client, so practically
    before the shared_client function is called.
    """
    try:
        kubernetes_asyncio.config.load_incluster_config()
    except kubernetes_asyncio.config.ConfigException:
        # avoid making this async just for load-config
        # run async load_kube_config in a background thread,
        # blocking this thread until it's done
        with ThreadPoolExecutor(1) as pool:
            load_sync = lambda: asyncio.run(
                kubernetes_asyncio.config.load_kube_config()
            )
            future = pool.submit(load_sync)
            # blocking wait for load to complete
            future.result()

    if ssl_ca_cert:
        global_conf = Configuration.get_default_copy()
        global_conf.ssl_ca_cert = ssl_ca_cert
        Configuration.set_default(global_conf)
    if host:
        global_conf = Configuration.get_default_copy()
        global_conf.host = host
        Configuration.set_default(global_conf)
    if not verify_ssl:
        global_conf = Configuration.get_default_copy()
        global_conf.verify_ssl = verify_ssl
        Configuration.set_default(global_conf)

How to reproduce

  1. install jhub by helm helm install -n jhub hub
  2. get svc ip kubectl get svc -n jhub
  3. access hub page
  4. See error

Expected behaviour

redirect to home page and go to server

Actual behaviour

image

Your personal set up

Full environment ``` # paste output of `pip freeze` or `conda list` here ```
Configuration ```python # jupyterhub_config.py # load the config object (satisfies linters) c = get_config() # noqa import glob import os import re import sys from jupyterhub.utils import url_path_join from kubernetes_asyncio import client from kubernetes_asyncio.client import Configuration from tornado.httpclient import AsyncHTTPClient from kubespawner.clients import load_config # Make sure that modules placed in the same directory as the jupyterhub config are added to the pythonpath configuration_directory = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, configuration_directory) from z2jh import ( get_config, get_name, get_name_env, get_secret_value, set_config_if_not_none, ) def camelCaseify(s): """convert snake_case to camelCase For the common case where some_value is set from someValue so we don't have to specify the name twice. """ return re.sub(r"_([a-z])", lambda m: m.group(1).upper(), s) # Configure JupyterHub to use the curl backend for making HTTP requests, # rather than the pure-python implementations. The default one starts # being too slow to make a large number of requests to the proxy API # at the rate required. AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient") c.JupyterHub.spawner_class = "kubespawner.KubeSpawner" # Connect to a proxy running in a different pod. Note that *_SERVICE_* # environment variables are set by Kubernetes for Services c.ConfigurableHTTPProxy.api_url = ( f'http://{get_name("proxy-api")}:{get_name_env("proxy-api", "_SERVICE_PORT")}' ) c.ConfigurableHTTPProxy.should_start = False # Do not shut down user pods when hub is restarted c.JupyterHub.cleanup_servers = False # Check that the proxy has routes appropriately setup c.JupyterHub.last_activity_interval = 60 # Don't wait at all before redirecting a spawning user to the progress page c.JupyterHub.tornado_settings = { "slow_spawn_timeout": 0, } # configure the hub db connection db_type = get_config("hub.db.type") if db_type == "sqlite-pvc": c.JupyterHub.db_url = "sqlite:///jupyterhub.sqlite" elif db_type == "sqlite-memory": c.JupyterHub.db_url = "sqlite://" else: set_config_if_not_none(c.JupyterHub, "db_url", "hub.db.url") db_password = get_secret_value("hub.db.password", None) if db_password is not None: if db_type == "mysql": os.environ["MYSQL_PWD"] = db_password elif db_type == "postgres": os.environ["PGPASSWORD"] = db_password else: print(f"Warning: hub.db.password is ignored for hub.db.type={db_type}") # c.JupyterHub configuration from Helm chart's configmap for trait, cfg_key in ( ("concurrent_spawn_limit", None), ("active_server_limit", None), ("base_url", None), ("allow_named_servers", None), ("named_server_limit_per_user", None), ("authenticate_prometheus", None), ("redirect_to_server", None), ("shutdown_on_logout", None), ("template_paths", None), ("template_vars", None), ): if cfg_key is None: cfg_key = camelCaseify(trait) set_config_if_not_none(c.JupyterHub, trait, "hub." + cfg_key) # hub_bind_url configures what the JupyterHub process within the hub pod's # container should listen to. hub_container_port = 8081 c.JupyterHub.hub_bind_url = f"http://:{hub_container_port}" # hub_connect_url is the URL for connecting to the hub for use by external # JupyterHub services such as the proxy. Note that *_SERVICE_* environment # variables are set by Kubernetes for Services. c.JupyterHub.hub_connect_url = ( f'http://{get_name("hub")}:{get_name_env("hub", "_SERVICE_PORT")}' ) # implement common labels # this duplicates the jupyterhub.commonLabels helper common_labels = c.KubeSpawner.common_labels = {} common_labels["app"] = get_config( "nameOverride", default=get_config("Chart.Name", "jupyterhub"), ) common_labels["heritage"] = "jupyterhub" chart_name = get_config("Chart.Name") chart_version = get_config("Chart.Version") if chart_name and chart_version: common_labels["chart"] = "{}-{}".format( chart_name, chart_version.replace("+", "_"), ) release = get_config("Release.Name") if release: common_labels["release"] = release c.KubeSpawner.namespace = os.environ.get("POD_NAMESPACE", "default") # Max number of consecutive failures before the Hub restarts itself # requires jupyterhub 0.9.2 set_config_if_not_none( c.Spawner, "consecutive_failure_limit", "hub.consecutiveFailureLimit", ) for trait, cfg_key in ( ("pod_name_template", None), ("start_timeout", None), ("image_pull_policy", "image.pullPolicy"), # ('image_pull_secrets', 'image.pullSecrets'), # Managed manually below ("events_enabled", "events"), ("extra_labels", None), ("extra_annotations", None), # ("allow_privilege_escalation", None), # Managed manually below ("uid", None), ("fs_gid", None), ("service_account", "serviceAccountName"), ("storage_extra_labels", "storage.extraLabels"), # ("tolerations", "extraTolerations"), # Managed manually below ("node_selector", None), ("node_affinity_required", "extraNodeAffinity.required"), ("node_affinity_preferred", "extraNodeAffinity.preferred"), ("pod_affinity_required", "extraPodAffinity.required"), ("pod_affinity_preferred", "extraPodAffinity.preferred"), ("pod_anti_affinity_required", "extraPodAntiAffinity.required"), ("pod_anti_affinity_preferred", "extraPodAntiAffinity.preferred"), ("lifecycle_hooks", None), ("init_containers", None), ("extra_containers", None), ("mem_limit", "memory.limit"), ("mem_guarantee", "memory.guarantee"), ("cpu_limit", "cpu.limit"), ("cpu_guarantee", "cpu.guarantee"), ("extra_resource_limits", "extraResource.limits"), ("extra_resource_guarantees", "extraResource.guarantees"), ("environment", "extraEnv"), ("profile_list", None), ("extra_pod_config", None), ): if cfg_key is None: cfg_key = camelCaseify(trait) set_config_if_not_none(c.KubeSpawner, trait, "singleuser." + cfg_key) image = get_config("singleuser.image.name") if image: tag = get_config("singleuser.image.tag") if tag: image = f"{image}:{tag}" c.KubeSpawner.image = image # allow_privilege_escalation defaults to False in KubeSpawner 2+. Since its a # property where None, False, and True all are valid values that users of the # Helm chart may want to set, we can't use the set_config_if_not_none helper # function as someone may want to override the default False value to None. # c.KubeSpawner.allow_privilege_escalation = get_config( "singleuser.allowPrivilegeEscalation" ) # Combine imagePullSecret.create (single), imagePullSecrets (list), and # singleuser.image.pullSecrets (list). image_pull_secrets = [] if get_config("imagePullSecret.automaticReferenceInjection") and get_config( "imagePullSecret.create" ): image_pull_secrets.append(get_name("image-pull-secret")) if get_config("imagePullSecrets"): image_pull_secrets.extend(get_config("imagePullSecrets")) if get_config("singleuser.image.pullSecrets"): image_pull_secrets.extend(get_config("singleuser.image.pullSecrets")) if image_pull_secrets: c.KubeSpawner.image_pull_secrets = image_pull_secrets # scheduling: if get_config("scheduling.userScheduler.enabled"): c.KubeSpawner.scheduler_name = get_name("user-scheduler") if get_config("scheduling.podPriority.enabled"): c.KubeSpawner.priority_class_name = get_name("priority") # add node-purpose affinity match_node_purpose = get_config("scheduling.userPods.nodeAffinity.matchNodePurpose") if match_node_purpose: node_selector = dict( matchExpressions=[ dict( key="hub.jupyter.org/node-purpose", operator="In", values=["user"], ) ], ) if match_node_purpose == "prefer": c.KubeSpawner.node_affinity_preferred.append( dict( weight=100, preference=node_selector, ), ) elif match_node_purpose == "require": c.KubeSpawner.node_affinity_required.append(node_selector) elif match_node_purpose == "ignore": pass else: raise ValueError( f"Unrecognized value for matchNodePurpose: {match_node_purpose}" ) # Combine the common tolerations for user pods with singleuser tolerations scheduling_user_pods_tolerations = get_config("scheduling.userPods.tolerations", []) singleuser_extra_tolerations = get_config("singleuser.extraTolerations", []) tolerations = scheduling_user_pods_tolerations + singleuser_extra_tolerations if tolerations: c.KubeSpawner.tolerations = tolerations # Configure dynamically provisioning pvc storage_type = get_config("singleuser.storage.type") if storage_type == "dynamic": pvc_name_template = get_config("singleuser.storage.dynamic.pvcNameTemplate") c.KubeSpawner.pvc_name_template = pvc_name_template volume_name_template = get_config("singleuser.storage.dynamic.volumeNameTemplate") c.KubeSpawner.storage_pvc_ensure = True set_config_if_not_none( c.KubeSpawner, "storage_class", "singleuser.storage.dynamic.storageClass" ) set_config_if_not_none( c.KubeSpawner, "storage_access_modes", "singleuser.storage.dynamic.storageAccessModes", ) set_config_if_not_none( c.KubeSpawner, "storage_capacity", "singleuser.storage.capacity" ) # Add volumes to singleuser pods c.KubeSpawner.volumes = [ { "name": volume_name_template, "persistentVolumeClaim": {"claimName": pvc_name_template}, } ] c.KubeSpawner.volume_mounts = [ { "mountPath": get_config("singleuser.storage.homeMountPath"), "name": volume_name_template, } ] elif storage_type == "static": pvc_claim_name = get_config("singleuser.storage.static.pvcName") c.KubeSpawner.volumes = [ {"name": "home", "persistentVolumeClaim": {"claimName": pvc_claim_name}} ] c.KubeSpawner.volume_mounts = [ { "mountPath": get_config("singleuser.storage.homeMountPath"), "name": "home", "subPath": get_config("singleuser.storage.static.subPath"), } ] # Inject singleuser.extraFiles as volumes and volumeMounts with data loaded from # the dedicated k8s Secret prepared to hold the extraFiles actual content. extra_files = get_config("singleuser.extraFiles", {}) if extra_files: volume = { "name": "files", } items = [] for file_key, file_details in extra_files.items(): # Each item is a mapping of a key in the k8s Secret to a path in this # abstract volume, the goal is to enable us to set the mode / # permissions only though so we don't change the mapping. item = { "key": file_key, "path": file_key, } if "mode" in file_details: item["mode"] = file_details["mode"] items.append(item) volume["secret"] = { "secretName": get_name("singleuser"), "items": items, } c.KubeSpawner.volumes.append(volume) volume_mounts = [] for file_key, file_details in extra_files.items(): volume_mounts.append( { "mountPath": file_details["mountPath"], "subPath": file_key, "name": "files", } ) c.KubeSpawner.volume_mounts.extend(volume_mounts) # Inject extraVolumes / extraVolumeMounts c.KubeSpawner.volumes.extend(get_config("singleuser.storage.extraVolumes", [])) c.KubeSpawner.volume_mounts.extend( get_config("singleuser.storage.extraVolumeMounts", []) ) c.JupyterHub.services = [] c.JupyterHub.load_roles = [] # jupyterhub-idle-culler's permissions are scoped to what it needs only, see # https://github.com/jupyterhub/jupyterhub-idle-culler#permissions. # if get_config("cull.enabled", False): jupyterhub_idle_culler_role = { "name": "jupyterhub-idle-culler", "scopes": [ "list:users", "read:users:activity", "read:servers", "delete:servers", # "admin:users", # dynamically added if --cull-users is passed ], # assign the role to a jupyterhub service, so it gains these permissions "services": ["jupyterhub-idle-culler"], } cull_cmd = ["python3", "-m", "jupyterhub_idle_culler"] base_url = c.JupyterHub.get("base_url", "/") cull_cmd.append("--url=http://localhost:8081" + url_path_join(base_url, "hub/api")) cull_timeout = get_config("cull.timeout") if cull_timeout: cull_cmd.append(f"--timeout={cull_timeout}") cull_every = get_config("cull.every") if cull_every: cull_cmd.append(f"--cull-every={cull_every}") cull_concurrency = get_config("cull.concurrency") if cull_concurrency: cull_cmd.append(f"--concurrency={cull_concurrency}") if get_config("cull.users"): cull_cmd.append("--cull-users") jupyterhub_idle_culler_role["scopes"].append("admin:users") if not get_config("cull.adminUsers"): cull_cmd.append("--cull-admin-users=false") if get_config("cull.removeNamedServers"): cull_cmd.append("--remove-named-servers") cull_max_age = get_config("cull.maxAge") if cull_max_age: cull_cmd.append(f"--max-age={cull_max_age}") c.JupyterHub.services.append( { "name": "jupyterhub-idle-culler", "command": cull_cmd, } ) c.JupyterHub.load_roles.append(jupyterhub_idle_culler_role) for key, service in get_config("hub.services", {}).items(): # c.JupyterHub.services is a list of dicts, but # hub.services is a dict of dicts to make the config mergable service.setdefault("name", key) # As the api_token could be exposed in hub.existingSecret, we need to read # it it from there or fall back to the chart managed k8s Secret's value. service.pop("apiToken", None) service["api_token"] = get_secret_value(f"hub.services.{key}.apiToken") c.JupyterHub.services.append(service) for key, role in get_config("hub.loadRoles", {}).items(): # c.JupyterHub.load_roles is a list of dicts, but # hub.loadRoles is a dict of dicts to make the config mergable role.setdefault("name", key) c.JupyterHub.load_roles.append(role) # respect explicit null command (distinct from unspecified) # this avoids relying on KubeSpawner.cmd's default being None _unspecified = object() specified_cmd = get_config("singleuser.cmd", _unspecified) if specified_cmd is not _unspecified: c.Spawner.cmd = specified_cmd set_config_if_not_none(c.Spawner, "default_url", "singleuser.defaultUrl") cloud_metadata = get_config("singleuser.cloudMetadata") if cloud_metadata.get("blockWithIptables") == True: # Use iptables to block access to cloud metadata by default network_tools_image_name = get_config("singleuser.networkTools.image.name") network_tools_image_tag = get_config("singleuser.networkTools.image.tag") network_tools_resources = get_config("singleuser.networkTools.resources") ip = cloud_metadata["ip"] ip_block_container = client.V1Container( name="block-cloud-metadata", image=f"{network_tools_image_name}:{network_tools_image_tag}", command=[ "iptables", "--append", "OUTPUT", "--protocol", "tcp", "--destination", ip, "--destination-port", "80", "--jump", "DROP", ], security_context=client.V1SecurityContext( privileged=True, run_as_user=0, capabilities=client.V1Capabilities(add=["NET_ADMIN"]), ), resources=network_tools_resources, ) c.KubeSpawner.init_containers.append(ip_block_container) if get_config("debug.enabled", False): c.JupyterHub.log_level = "DEBUG" c.Spawner.debug = True # load potentially seeded secrets # # NOTE: ConfigurableHTTPProxy.auth_token is set through an environment variable # that is set using the chart managed secret. c.JupyterHub.cookie_secret = get_secret_value("hub.config.JupyterHub.cookie_secret") # NOTE: CryptKeeper.keys should be a list of strings, but we have encoded as a # single string joined with ; in the k8s Secret. # c.CryptKeeper.keys = get_secret_value("hub.config.CryptKeeper.keys").split(";") # load hub.config values, except potentially seeded secrets already loaded for app, cfg in get_config("hub.config", {}).items(): if app == "JupyterHub": cfg.pop("proxy_auth_token", None) cfg.pop("cookie_secret", None) cfg.pop("services", None) elif app == "ConfigurableHTTPProxy": cfg.pop("auth_token", None) elif app == "CryptKeeper": cfg.pop("keys", None) c[app].update(cfg) # load /usr/local/etc/jupyterhub/jupyterhub_config.d config files config_dir = "/usr/local/etc/jupyterhub/jupyterhub_config.d" if os.path.isdir(config_dir): for file_path in sorted(glob.glob(f"{config_dir}/*.py")): file_name = os.path.basename(file_path) print(f"Loading {config_dir} config: {file_name}") with open(file_path) as f: file_content = f.read() # compiling makes debugging easier: https://stackoverflow.com/a/437857 exec(compile(source=file_content, filename=file_name, mode="exec")) # execute hub.extraConfig entries for key, config_py in sorted(get_config("hub.extraConfig", {}).items()): print(f"Loading extra config: {key}") exec(config_py) ```
Logs ``` # paste relevant logs here, if any Loading /usr/local/etc/jupyterhub/secret/values.yaml No config at /usr/local/etc/jupyterhub/existing-secret/values.yaml [I 2023-10-20 08:50:32.711 JupyterHub app:2859] Running JupyterHub version 4.0.2 [I 2023-10-20 08:50:32.711 JupyterHub app:2889] Using Authenticator: jupyterhub.auth.DummyAuthenticator-4.0.2 [I 2023-10-20 08:50:32.711 JupyterHub app:2889] Using Spawner: kubespawner.spawner.KubeSpawner-6.1.0 [I 2023-10-20 08:50:32.711 JupyterHub app:2889] Using Proxy: jupyterhub.proxy.ConfigurableHTTPProxy-4.0.2 [I 2023-10-20 08:50:32.740 alembic.runtime.migration migration:213] Context impl SQLiteImpl. [I 2023-10-20 08:50:32.740 alembic.runtime.migration migration:216] Will assume non-transactional DDL. [I 2023-10-20 08:50:32.745 alembic.runtime.migration migration:619] Running stamp_revision -> 0eee8c825d24 [I 2023-10-20 08:50:32.831 JupyterHub roles:172] Role jupyterhub-idle-culler added to database [I 2023-10-20 08:50:32.839 JupyterHub roles:238] Adding role admin for User: taotao.zhou [I 2023-10-20 08:50:32.841 JupyterHub roles:238] Adding role user for User: taotao.zhou [I 2023-10-20 08:50:32.842 JupyterHub app:1984] Not using allowed_users. Any authenticated user will be allowed. [I 2023-10-20 08:50:32.885 JupyterHub app:2928] Initialized 0 spawners in 0.003 seconds [I 2023-10-20 08:50:32.894 JupyterHub metrics:278] Found 0 active users in the last ActiveUserPeriods.twenty_four_hours [I 2023-10-20 08:50:32.895 JupyterHub metrics:278] Found 0 active users in the last ActiveUserPeriods.seven_days [I 2023-10-20 08:50:32.895 JupyterHub metrics:278] Found 0 active users in the last ActiveUserPeriods.thirty_days [I 2023-10-20 08:50:32.895 JupyterHub app:3142] Not starting proxy [I 2023-10-20 08:50:32.905 JupyterHub app:3178] Hub API listening on http://:8081/hub/ [I 2023-10-20 08:50:32.905 JupyterHub app:3180] Private Hub API connect url http://hub:8081/hub/ [I 2023-10-20 08:50:32.905 JupyterHub app:3189] Starting managed service jupyterhub-idle-culler [I 2023-10-20 08:50:32.905 JupyterHub service:385] Starting service 'jupyterhub-idle-culler': ['python3', '-m', 'jupyterhub_idle_culler', '--url=http://localhost:8081/hub/api', '--timeout=36000', '--cull-every=600', '--concurrency=10', '--cull-users'] [I 2023-10-20 08:50:32.907 JupyterHub service:133] Spawning python3 -m jupyterhub_idle_culler --url=http://localhost:8081/hub/api --timeout=36000 --cull-every=600 --concurrency=10 --cull-users [I 2023-10-20 08:50:32.911 JupyterHub proxy:477] Adding route for Hub: / => http://hub:8081 [I 2023-10-20 08:50:32.914 JupyterHub app:3247] JupyterHub is now running, internal Hub API at http://hub:8081/hub/ [I 2023-10-20 08:50:33.699 JupyterHub log:191] 200 GET /hub/api/ (jupyterhub-idle-culler@127.0.0.1) 9.20ms [I 2023-10-20 08:50:33.719 JupyterHub log:191] 200 GET /hub/api/users?state=[secret] (jupyterhub-idle-culler@127.0.0.1) 18.08ms [I 2023-10-20 08:50:33.729 JupyterHub log:191] 200 GET /hub/api/users?state=[secret] (jupyterhub-idle-culler@127.0.0.1) 6.56ms [W 2023-10-20 08:50:46.118 JupyterHub base:422] Invalid cookie token [I 2023-10-20 08:50:46.120 JupyterHub log:191] 302 GET /hub/spawn-pending/taotao.zhou -> /hub/login?next=%2Fhub%2Fspawn-pending%2Ftaotao.zhou (@::ffff:10.176.206.224) 4.58ms [I 2023-10-20 08:50:46.186 JupyterHub log:191] 200 GET /hub/login?next=%2Fhub%2Fspawn-pending%2Ftaotao.zhou (@::ffff:10.176.206.225) 43.55ms [I 2023-10-20 08:50:48.872 JupyterHub base:837] User logged in: taotao.zhou [I 2023-10-20 08:50:48.873 JupyterHub log:191] 302 POST /hub/login?next=%2Fhub%2Fspawn-pending%2Ftaotao.zhou -> /hub/spawn-pending/taotao.zhou (taotao.zhou@::ffff:10.176.206.174) 8.54ms [I 2023-10-20 08:50:48.923 JupyterHub log:191] 200 GET /hub/spawn-pending/taotao.zhou (taotao.zhou@::ffff:10.176.206.184) 27.39ms [I 2023-10-20 08:50:56.460 JupyterHub provider:659] Creating oauth client jupyterhub-user-taotao.zhou [I 2023-10-20 08:50:56.473 JupyterHub log:191] 302 GET /hub/spawn/taotao.zhou -> /hub/spawn-pending/taotao.zhou (taotao.zhou@::ffff:10.176.206.146) 35.56ms [E 2023-10-20 08:50:56.478 JupyterHub reflector:397] Initial list of pods failed Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/kubespawner/reflector.py", line 395, in start await self._list_and_update() File "/usr/local/lib/python3.11/site-packages/kubespawner/reflector.py", line 231, in _list_and_update initial_resources_raw = await list_method(**kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/api_client.py", line 185, in __call_api response_data = await self.request( ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/rest.py", line 196, in GET return (await self.request("GET", url, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/rest.py", line 180, in request r = await self.pool_manager.request(**args) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/client.py", line 536, in _request conn = await self._connector.connect( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 540, in connect proto = await self._create_connection(req, traces, timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 901, in _create_connection _, proto = await self._create_direct_connection(req, traces, timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 1209, in _create_direct_connection raise last_exc File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 1178, in _create_direct_connection transp, proto = await self._wrap_create_connection( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 982, in _wrap_create_connection raise ClientConnectorCertificateError(req.connection_key, exc) from exc aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host 10.xxx.xxx.1:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate (_ssl.c:1006)')] [E 2023-10-20 08:50:56.484 JupyterHub spawner:2407] Reflector with key ('pods', 'jhub') failed to start. Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 2405, in catch_reflector_start await func File "/usr/local/lib/python3.11/site-packages/kubespawner/reflector.py", line 395, in start await self._list_and_update() File "/usr/local/lib/python3.11/site-packages/kubespawner/reflector.py", line 231, in _list_and_update initial_resources_raw = await list_method(**kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/api_client.py", line 185, in __call_api response_data = await self.request( ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/rest.py", line 196, in GET return (await self.request("GET", url, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/rest.py", line 180, in request r = await self.pool_manager.request(**args) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/client.py", line 536, in _request conn = await self._connector.connect( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 540, in connect proto = await self._create_connection(req, traces, timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 901, in _create_connection _, proto = await self._create_direct_connection(req, traces, timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 1209, in _create_direct_connection raise last_exc File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 1178, in _create_direct_connection transp, proto = await self._wrap_create_connection( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 982, in _wrap_create_connection raise ClientConnectorCertificateError(req.connection_key, exc) from exc aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host 10.xxx.xxx.1:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate (_ssl.c:1006)')] [E 2023-10-20 08:50:56.489 JupyterHub reflector:397] Initial list of events failed Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/kubespawner/reflector.py", line 395, in start await self._list_and_update() File "/usr/local/lib/python3.11/site-packages/kubespawner/reflector.py", line 231, in _list_and_update initial_resources_raw = await list_method(**kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/api_client.py", line 185, in __call_api response_data = await self.request( ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/rest.py", line 196, in GET return (await self.request("GET", url, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/rest.py", line 180, in request r = await self.pool_manager.request(**args) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/client.py", line 536, in _request conn = await self._connector.connect( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 540, in connect proto = await self._create_connection(req, traces, timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 901, in _create_connection _, proto = await self._create_direct_connection(req, traces, timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 1209, in _create_direct_connection raise last_exc File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 1178, in _create_direct_connection transp, proto = await self._wrap_create_connection( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 982, in _wrap_create_connection raise ClientConnectorCertificateError(req.connection_key, exc) from exc aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host 10.xxx.xxx.1:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate (_ssl.c:1006)')] [E 2023-10-20 08:50:56.491 JupyterHub spawner:2407] Reflector with key ('events', 'jhub') failed to start. Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 2405, in catch_reflector_start await func File "/usr/local/lib/python3.11/site-packages/kubespawner/reflector.py", line 395, in start await self._list_and_update() File "/usr/local/lib/python3.11/site-packages/kubespawner/reflector.py", line 231, in _list_and_update initial_resources_raw = await list_method(**kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/api_client.py", line 185, in __call_api response_data = await self.request( ^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/rest.py", line 196, in GET return (await self.request("GET", url, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubernetes_asyncio/client/rest.py", line 180, in request r = await self.pool_manager.request(**args) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/client.py", line 536, in _request conn = await self._connector.connect( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 540, in connect proto = await self._create_connection(req, traces, timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 901, in _create_connection _, proto = await self._create_direct_connection(req, traces, timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 1209, in _create_direct_connection raise last_exc File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 1178, in _create_direct_connection transp, proto = await self._wrap_create_connection( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/aiohttp/connector.py", line 982, in _wrap_create_connection raise ClientConnectorCertificateError(req.connection_key, exc) from exc aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host 10.xxx.xxx.1:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate (_ssl.c:1006)')] [I 2023-10-20 08:50:56.497 JupyterHub pages:398] taotao.zhou is pending spawn [I 2023-10-20 08:50:56.502 JupyterHub log:191] 200 GET /hub/spawn-pending/taotao.zhou (taotao.zhou@::ffff:10.176.206.132) 8.02ms [E 2023-10-20 08:50:56.502 JupyterHub user:884] Unhandled error starting taotao.zhou's server: 'coroutine' object has no attribute 'cancel' Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/jupyterhub/user.py", line 798, in spawn url = await gen.with_timeout(timedelta(seconds=spawner.start_timeout), f) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 2669, in _start future.cancel() ^^^^^^^^^^^^^ AttributeError: 'coroutine' object has no attribute 'cancel' [E 2023-10-20 08:50:56.530 JupyterHub gen:630] Exception in Future .finish_user_spawn() done, defined at /usr/local/lib/python3.11/site-packages/jupyterhub/handlers/base.py:981> exception=AttributeError("'coroutine' object has no attribute 'cancel'")> after timeout Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/tornado/gen.py", line 625, in error_callback future.result() File "/usr/local/lib/python3.11/site-packages/jupyterhub/handlers/base.py", line 988, in finish_user_spawn await spawn_future File "/usr/local/lib/python3.11/site-packages/jupyterhub/user.py", line 902, in spawn raise e File "/usr/local/lib/python3.11/site-packages/jupyterhub/user.py", line 798, in spawn url = await gen.with_timeout(timedelta(seconds=spawner.start_timeout), f) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 2669, in _start future.cancel() ^^^^^^^^^^^^^ AttributeError: 'coroutine' object has no attribute 'cancel' [I 2023-10-20 08:50:56.567 JupyterHub log:191] 200 GET /hub/api/users/taotao.zhou/server/progress?_xsrf=[secret] (taotao.zhou@::ffff:10.176.206.185) 4.67ms [I 2023-10-20 09:00:33.737 JupyterHub log:191] 200 GET /hub/api/ (jupyterhub-idle-culler@127.0.0.1) 5.80ms [I 2023-10-20 09:00:33.745 JupyterHub log:191] 200 GET /hub/api/users?state=[secret] (jupyterhub-idle-culler@127.0.0.1) 4.39ms [I 2023-10-20 09:00:33.752 JupyterHub log:191] 200 GET /hub/api/users?state=[secret] (jupyterhub-idle-culler@127.0.0.1) 3.45ms [I 2023-10-20 09:10:33.726 JupyterHub log:191] 200 GET /hub/api/ (jupyterhub-idle-culler@127.0.0.1) 6.39ms [I 2023-10-20 09:10:33.733 JupyterHub log:191] 200 GET /hub/api/users?state=[secret] (jupyterhub-idle-culler@127.0.0.1) 3.71ms [I 2023-10-20 09:10:33.743 JupyterHub log:191] 200 GET /hub/api/users?state=[secret] (jupyterhub-idle-culler@127.0.0.1) 7.05ms ```
welcome[bot] commented 8 months ago

Thank you for opening your first issue in this project! Engagement like this is essential for open source projects! :hugs:
If you haven't done so already, check out Jupyter's Code of Conduct. Also, please try to follow the issue template as it helps other other community members to contribute more effectively. welcome You can meet the other Jovyans by joining our Discourse forum. There is also an intro thread there where you can stop by and say Hi! :wave:
Welcome to the Jupyter community! :tada:

consideRatio commented 8 months ago

Do you wish to be able to configure kubespawner to not verify certificates when communicating with the k8s api-server?

This is where kubespawner use the load_clients function, but is hardcoded to not pass a verify argument

https://github.com/jupyterhub/kubespawner/blob/34a1f66b0ab3a5d5805834b7b260a65ff37475a4/kubespawner/spawner.py#L204

consideRatio commented 8 months ago

Seems like there is a bug in the load_config function as well, where it should be "if verify_ssl is not None" instead of "if not verify_ssl" i think

devenami commented 8 months ago

Do you wish to be able to configure kubespawner to not verify certificates when communicating with the k8s api-server?

This is where kubespawner use the load_clients function, but is hardcoded to not pass a verify argument

https://github.com/jupyterhub/kubespawner/blob/34a1f66b0ab3a5d5805834b7b260a65ff37475a4/kubespawner/spawner.py#L204

Thank you very much for your reply.

I guess it's because I used a self signed certificate, and Python's SSL library is unable to verify the validity of the self signed certificate. This issue does not occur when using lower versions of Kubespawner, but it does occur in higher versions. Therefore, I prefer to skip the method of certificate verification.

The code changed:

# clients.py
@lru_cache()
def load_config(host=None, ssl_ca_cert=None, verify_ssl=None):
    """
    Loads global configuration for the Python client we use to communicate with
    a Kubernetes API server, and optionally tweaks that configuration based on
    specific settings on the passed caller object.

    This needs to be called before creating a kubernetes client, so practically
    before the shared_client function is called.
    """
    try:
        kubernetes_asyncio.config.load_incluster_config()
    except kubernetes_asyncio.config.ConfigException:
        # avoid making this async just for load-config
        # run async load_kube_config in a background thread,
        # blocking this thread until it's done
        with ThreadPoolExecutor(1) as pool:
            load_sync = lambda: asyncio.run(
                kubernetes_asyncio.config.load_kube_config()
            )
            future = pool.submit(load_sync)
            # blocking wait for load to complete
            future.result()

    if ssl_ca_cert:
        global_conf = Configuration.get_default_copy()
        global_conf.ssl_ca_cert = ssl_ca_cert
        Configuration.set_default(global_conf)
    if host:
        global_conf = Configuration.get_default_copy()
        global_conf.host = host
        Configuration.set_default(global_conf)
    if verify_ssl is not None:
        global_conf = Configuration.get_default_copy()
        global_conf.verify_ssl = verify_ssl
        Configuration.set_default(global_conf)