undefined method `get_namespaces' for k8s client

#<Thread:0x00007f033e6d71b0 run> terminated with exception (report_on_exception is true):
/fluentd/vendor/bundle/ruby/3.1.0/gems/fluent-plugin-kubernetes_metadata_filter-2.13.0/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb:119:in `rescue in start_namespace_watch': start_namespace_watch: Exception encountered setting up namespace watch from Kubernetes API v1 endpoint https://10.96.0.1:443/api: undefined method `get_namespaces' for #<Kubeclient::Client:0x00007f033e784ba8 @api_endpoint=#<URI::HTTPS https://10.96.0.1/api>, @api_group="", @entities={"resource_quota"=>#<OpenStruct entity_type="ResourceQuota", resource_name="resourcequotas", method_names=["resource_quota", "resource_quotas"]>, "secret"=>#<OpenStruct entity_type="Secret", resource_name="secrets", method_names=["secret", "secrets"]>, "service_account"=>#<OpenStruct entity_type="ServiceAccount", resource_name="serviceaccounts", method_names=["service_account", "service_accounts"]>, "service"=>#<OpenStruct entity_type="Service", resource_name="services", method_names=["service", "services"]>}, @discovered=true, @api_version="v1", @headers={:Authorization=>"Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjVlSWFoWmhjLW9sUnFHZ1NzN294ZzNPM21jZVBkTkM4SUdRNG5UMFpCaGsifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLml3cGxhdGZvcm0ubG9jYWwiXSwiZXhwIjoxNzEzNjE1MDU0LCJpYXQiOjE2ODIwNzkwNTQsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5pd3BsYXRmb3JtLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJpbmZvd2F0Y2giLCJwb2QiOnsibmFtZSI6ImZsdWVudGQtd25kZjUiLCJ1aWQiOiIwYTU4MGZiNy05ZmZmLTQ2ZTgtYTIxMS00N2JkOTkwZjhlZjkifSwic2VydmljZWFjY291bnQiOnsibmFtZSI6ImZsdWVudGQiLCJ1aWQiOiI4YzBkMzMwNS0yMjIxLTRjYTAtOWVjYS00MDAzZjFhMGVjMGIifSwid2FybmFmdGVyIjoxNjgyMDgyNjYxfSwibmJmIjoxNjgyMDc5MDU0LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW5mb3dhdGNoOmZsdWVudGQifQ.Fk-3Xacu_aC9bXw8mhC8XuKAhLBCQyuLSWCdjryHXg-wggaSHIrVjIjvdwytwBQt9uHBJQLVstiJGKNG7FGIXgTG7hkF5i9ISxN64xdOhWvWUtpp8DJEhgzcuU5UMaycQ0nerCz6g7R1WFKK4mewuYf0dMm9p5fviztPm2TKrh5sK1U35eZZsXx2inAla_YAjJ5jC0F0Jp6QgqMgibU6xAOKFuRK5SwCeR9kZJE-MxaLj7dLp8xd1W324qi7d35KNIeCTalarbp8xrlQe0at3JzUCLyJcSbxqipzMqbFD9xHmsYcirNhsrpYlx80XFp950FgZ6bf-4bt7tURxUgljw"}, @ssl_options={:client_cert=>nil, :client_key=>nil, :ca_file=>"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", :verify_ssl=>1}, @auth_options={:bearer_token_file=>"/var/run/secrets/kubernetes.io/serviceaccount/token"}, @socket_options={:socket_class=>nil, :ssl_socket_class=>nil}, @timeouts={:open=>60, :read=>60}, @http_proxy_uri=nil, @http_max_redirects=10, @as=:parsed_symbolized, @rest_client=#<RestClient::Resource:0x00007f033e6edac8 @url="https://10.96.0.1/api/v1", @block=nil, @options={:ssl_ca_file=>"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", :ssl_cert_store=>nil, :verify_ssl=>1, :ssl_client_cert=>nil, :ssl_client_key=>nil, :proxy=>nil, :max_redirects=>10, :user=>nil, :password=>nil, :open_timeout=>60, :read_timeout=>60}>> (Fluent::ConfigError)
Did you mean?  get_secrets
    from /fluentd/vendor/bundle/ruby/3.1.0/gems/fluent-plugin-kubernetes_metadata_filter-2.13.0/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb:110:in `start_namespace_watch'
    from /fluentd/vendor/bundle/ruby/3.1.0/gems/fluent-plugin-kubernetes_metadata_filter-2.13.0/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb:32:in `set_up_namespace_thread'
2023-04-21 12:34:38 +0000 [info]: adding match pattern="kubernetes.**" type="relabel"
2023-04-21 12:34:38 +0000 [info]: adding source type="tail"
/fluentd/vendor/bundle/ruby/3.1.0/gems/kubeclient-4.11.0/lib/kubeclient/common.rb:103:in `method_missing': undefined method `get_namespaces' for #<Kubeclient::Client:0x00007f033e784ba8 @api_endpoint=#<URI::HTTPS https://10.96.0.1/api>, @api_group="", @entities={"resource_quota"=>#<OpenStruct entity_type="ResourceQuota", resource_name="resourcequotas", method_names=["resource_quota", "resource_quotas"]>, "secret"=>#<OpenStruct entity_type="Secret", resource_name="secrets", method_names=["secret", "secrets"]>, "service_account"=>#<OpenStruct entity_type="ServiceAccount", resource_name="serviceaccounts", method_names=["service_account", "service_accounts"]>, "service"=>#<OpenStruct entity_type="Service", resource_name="services", method_names=["service", "services"]>}, @discovered=true, @api_version="v1", @headers={:Authorization=>"Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjVlSWFoWmhjLW9sUnFHZ1NzN294ZzNPM21jZVBkTkM4SUdRNG5UMFpCaGsifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLml3cGxhdGZvcm0ubG9jYWwiXSwiZXhwIjoxNzEzNjE1MDU0LCJpYXQiOjE2ODIwNzkwNTQsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5pd3BsYXRmb3JtLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJpbmZvd2F0Y2giLCJwb2QiOnsibmFtZSI6ImZsdWVudGQtd25kZjUiLCJ1aWQiOiIwYTU4MGZiNy05ZmZmLTQ2ZTgtYTIxMS00N2JkOTkwZjhlZjkifSwic2VydmljZWFjY291bnQiOnsibmFtZSI6ImZsdWVudGQiLCJ1aWQiOiI4YzBkMzMwNS0yMjIxLTRjYTAtOWVjYS00MDAzZjFhMGVjMGIifSwid2FybmFmdGVyIjoxNjgyMDgyNjYxfSwibmJmIjoxNjgyMDc5MDU0LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW5mb3dhdGNoOmZsdWVudGQifQ.Fk-3Xacu_aC9bXw8mhC8XuKAhLBCQyuLSWCdjryHXg-wggaSHIrVjIjvdwytwBQt9uHBJQLVstiJGKNG7FGIXgTG7hkF5i9ISxN64xdOhWvWUtpp8DJEhgzcuU5UMaycQ0nerCz6g7R1WFKK4mewuYf0dMm9p5fviztPm2TKrh5sK1U35eZZsXx2inAla_YAjJ5jC0F0Jp6QgqMgibU6xAOKFuRK5SwCeR9kZJE-MxaLj7dLp8xd1W324qi7d35KNIeCTalarbp8xrlQe0at3JzUCLyJcSbxqipzMqbFD9xHmsYcirNhsrpYlx80XFp950FgZ6bf-4bt7tURxUgljw"}, @ssl_options={:client_cert=>nil, :client_key=>nil, :ca_file=>"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", :verify_ssl=>1}, @auth_options={:bearer_token_file=>"/var/run/secrets/kubernetes.io/serviceaccount/token"}, @socket_options={:socket_class=>nil, :ssl_socket_class=>nil}, @timeouts={:open=>60, :read=>60}, @http_proxy_uri=nil, @http_max_redirects=10, @as=:parsed_symbolized, @rest_client=#<RestClient::Resource:0x00007f033e6edac8 @url="https://10.96.0.1/api/v1", @block=nil, @options={:ssl_ca_file=>"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", :ssl_cert_store=>nil, :verify_ssl=>1, :ssl_client_cert=>nil, :ssl_client_key=>nil, :proxy=>nil, :max_redirects=>10, :user=>nil, :password=>nil, :open_timeout=>60, :read_timeout=>60}>> (NoMethodError)
Did you mean?  get_secrets
    from /fluentd/vendor/bundle/ruby/3.1.0/gems/kubeclient-4.11.0/lib/kubeclient/common.rb:101:in `method_missing'
    from /fluentd/vendor/bundle/ruby/3.1.0/gems/fluent-plugin-kubernetes_metadata_filter-2.13.0/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb:128:in `get_namespaces_and_start_watcher'
    from /fluentd/vendor/bundle/ruby/3.1.0/gems/fluent-plugin-kubernetes_metadata_filter-2.13.0/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb:111:in `start_namespace_watch'
    from /fluentd/vendor/bundle/ruby/3.1.0/gems/fluent-plugin-kubernetes_metadata_filter-2.13.0/lib/fluent/plugin/kubernetes_metadata_watch_namespaces.rb:32:in `set_up_namespace_thread'
2023-04-21 12:34:38 +0000 [error]: #0 config error file="/root/fluent/fluent.conf" error_class=Fluent::ConfigError error="start_namespace_watch: Exception encountered setting up namespace watch from Kubernetes API v1 endpoint https://10.96.0.1:443/api: undefined method `get_namespaces' for #<Kubeclient::Client:0x00007f033e784ba8 @api_endpoint=#<URI::HTTPS https://10.96.0.1/api>, @api_group=\"\", @entities={\"resource_quota\"=>#<OpenStruct entity_type=\"ResourceQuota\", resource_name=\"resourcequotas\", method_names=[\"resource_quota\", \"resource_quotas\"]>, \"secret\"=>#<OpenStruct entity_type=\"Secret\", resource_name=\"secrets\", method_names=[\"secret\", \"secrets\"]>, \"service_account\"=>#<OpenStruct entity_type=\"ServiceAccount\", resource_name=\"serviceaccounts\", method_names=[\"service_account\", \"service_accounts\"]>, \"service\"=>#<OpenStruct entity_type=\"Service\", resource_name=\"services\", method_names=[\"service\", \"services\"]>}, @discovered=true, @api_version=\"v1\", @headers={:Authorization=>\"Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjVlSWFoWmhjLW9sUnFHZ1NzN294ZzNPM21jZVBkTkM4SUdRNG5UMFpCaGsifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLml3cGxhdGZvcm0ubG9jYWwiXSwiZXhwIjoxNzEzNjE1MDU0LCJpYXQiOjE2ODIwNzkwNTQsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5pd3BsYXRmb3JtLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJpbmZvd2F0Y2giLCJwb2QiOnsibmFtZSI6ImZsdWVudGQtd25kZjUiLCJ1aWQiOiIwYTU4MGZiNy05ZmZmLTQ2ZTgtYTIxMS00N2JkOTkwZjhlZjkifSwic2VydmljZWFjY291bnQiOnsibmFtZSI6ImZsdWVudGQiLCJ1aWQiOiI4YzBkMzMwNS0yMjIxLTRjYTAtOWVjYS00MDAzZjFhMGVjMGIifSwid2FybmFmdGVyIjoxNjgyMDgyNjYxfSwibmJmIjoxNjgyMDc5MDU0LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6aW5mb3dhdGNoOmZsdWVudGQifQ.Fk-3Xacu_aC9bXw8mhC8XuKAhLBCQyuLSWCdjryHXg-wggaSHIrVjIjvdwytwBQt9uHBJQLVstiJGKNG7FGIXgTG7hkF5i9ISxN64xdOhWvWUtpp8DJEhgzcuU5UMaycQ0nerCz6g7R1WFKK4mewuYf0dMm9p5fviztPm2TKrh5sK1U35eZZsXx2inAla_YAjJ5jC0F0Jp6QgqMgibU6xAOKFuRK5SwCeR9kZJE-MxaLj7dLp8xd1W324qi7d35KNIeCTalarbp8xrlQe0at3JzUCLyJcSbxqipzMqbFD9xHmsYcirNhsrpYlx80XFp950FgZ6bf-4bt7tURxUgljw\"}, @ssl_options={:client_cert=>nil, :client_key=>nil, :ca_file=>\"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\", :verify_ssl=>1}, @auth_options={:bearer_token_file=>\"/var/run/secrets/kubernetes.io/serviceaccount/token\"}, @socket_options={:socket_class=>nil, :ssl_socket_class=>nil}, @timeouts={:open=>60, :read=>60}, @http_proxy_uri=nil, @http_max_redirects=10, @as=:parsed_symbolized, @rest_client=#<RestClient::Resource:0x00007f033e6edac8 @url=\"https://10.96.0.1/api/v1\", @block=nil, @options={:ssl_ca_file=>\"/var/run/secrets/kubernetes.io/serviceaccount/ca.crt\", :ssl_cert_store=>nil, :verify_ssl=>1, :ssl_client_cert=>nil, :ssl_client_key=>nil, :proxy=>nil, :max_redirects=>10, :user=>nil, :password=>nil, :open_timeout=>60, :read_timeout=>60}>>\nDid you mean?  get_secrets"
2023-04-21 12:34:38 +0000 [error]: Worker 0 exited unexpectedly with status 2
Sometimes i got such error. Seems like my kubernetes_metadata_filter not support my kubernetes version or something not good with k8s cluster. Also it is maybe because timeouts?? Stange that fluentd works with other logs and some data was parsed and sended to out (es), before error
I use:
daemonset with docker.io/fluent/fluentd-kubernetes-daemonset:v1.15.3-debian-elasticsearch7-amd64-1.1
fluentd 1.15.3
fluent-plugin-kubernetes_metadata_filter (2.13.0)
k8s versions: Client Version: v1.25.2, Kustomize Version: v4.5.7, Server Version: v1.25.2
fabric8io / fluent-plugin-kubernetes_metadata_filter

undefined method `get_namespaces' for k8s client #369
