search

elastic / kibana

Your window into the Elastic Stack
https://www.elastic.co/products/kibana
Other
19.48k stars 8.04k forks source link

Cant use preconfigured integration using ECK #186381

Open gaimes410 opened 2 weeks ago

gaimes410 commented 2 weeks ago

Kibana version: 8.14.1

Elasticsearch version: 8.14.1

Server OS version: Ubuntu 20.04.6 LTS (Focal Fossa)

Original install method (e.g. download page, yum, from source, etc.): ECK on prem kubernetes cluster

Describe the bug: When I'm trying to start a kibana instance with the additions of additional inputs in the package policies I'm getting the error

FATAL Error: [config validation of [xpack.fleet].agentPolicies.1.package_policies.0.inputs]: expected value of type [array] but got [Object]

Steps to reproduce:

  1. install the eck-operator using the helm chart from the official website
  2. install the entire logging-stack using the quickstart from the site
  3. insert in the kibana the exact sample of xpack.fleet.agentPolicies from the site https://www.elastic.co/guide/en/kibana/current/fleet-settings-kb.html

Expected behavior: Kibana starting normally with preconfigured policies

Provide logs and/or server output (if relevant): kubectl logs kibana-quickstart-kb-744978ddd-4csl6 -f Defaulted container "kibana" out of: kibana, elastic-internal-init-config (init) {"log.level":"info","@timestamp":"2024-06-18T13:08:09.702Z","log.logger":"elastic-apm-node","ecs.version":"8.10.0","agentVersion":"4.4.0","env":{"pid":7,"proctitle":"/usr/share/kibana/bin/../node/bin/node","os":"linux 5.4.0-77-generic","arch":"x64","host":"kibana-quickstart-kb-744978ddd-4csl6","timezone":"UTC+00","runtime":"Node.js v20.12.2"},"config":{"active":{"source":"start","value":true},"breakdownMetrics":{"source":"start","value":false},"captureBody":{"source":"start","value":"off","commonName":"capture_body"},"captureHeaders":{"source":"start","value":false},"centralConfig":{"source":"start","value":false},"contextPropagationOnly":{"source":"start","value":true},"environment":{"source":"start","value":"production"},"globalLabels":{"source":"start","value":[["git_rev","f5dc24d1969f80e4aa3ced7cc375dd00554f8c0c"]],"sourceValue":{"git_rev":"f5dc24d1969f80e4aa3ced7cc375dd00554f8c0c"}},"logLevel":{"source":"default","value":"info","commonName":"log_level"},"metricsInterval":{"source":"start","value":120,"sourceValue":"120s"},"serverUrl":{"source":"start","value":"https://kibana-cloud-apm.apm.us-east-1.aws.found.io/","commonName":"server_url"},"transactionSampleRate":{"source":"start","value":0.1,"commonName":"transaction_sample_rate"},"captureSpanStackTraces":{"source":"start","sourceValue":false},"secretToken":{"source":"start","value":"[REDACTED]","commonName":"secret_token"},"serviceName":{"source":"start","value":"kibana","commonName":"service_name"},"serviceVersion":{"source":"start","value":"8.13.4","commonName":"service_version"}},"activationMethod":"require","message":"Elastic APM Node.js Agent v4.4.0"} Native global console methods have been overridden in production environment. [2024-06-18T13:08:11.554+00:00][INFO ][root] Kibana is starting [2024-06-18T13:08:11.659+00:00][INFO ][node] Kibana process configured with roles: [background_tasks, ui] [2024-06-18T13:08:21.788+00:00][INFO ][root] Kibana is shutting down [2024-06-18T13:08:21.793+00:00][FATAL][root] Reason: [config validation of [xpack.fleet].agentPolicies.1.package_policies.0.inputs]: expected value of type [array] but got [Object] Error: [config validation of [xpack.fleet].agentPolicies.1.package_policies.0.inputs]: expected value of type [array] but got [Object] at ObjectType.validate (/usr/share/kibana/node_modules/@kbn/config-schema/src/types/type.js:91:13) at ConfigService.validateAtPath (/usr/share/kibana/node_modules/@kbn/config/src/config_service.js:252:19) at /usr/share/kibana/node_modules/@kbn/config/src/config_service.js:262:204 at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/map.js:10:37 at OperatorSubscriber._this._next (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/OperatorSubscriber.js:33:21) at OperatorSubscriber.Subscriber.next (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Subscriber.js:51:18) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/distinctUntilChanged.js:18:28 at OperatorSubscriber._this._next (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/OperatorSubscriber.js:33:21) at OperatorSubscriber.Subscriber.next (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Subscriber.js:51:18) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/map.js:10:24 at OperatorSubscriber._this._next (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/OperatorSubscriber.js:33:21) at OperatorSubscriber.Subscriber.next (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Subscriber.js:51:18) at ReplaySubject._subscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/ReplaySubject.js:54:24) at ReplaySubject.Observable._trySubscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:41:25) at ReplaySubject.Subject._trySubscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Subject.js:123:47) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:35:31 at Object.errorContext (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/errorContext.js:22:9) at ReplaySubject.Observable.subscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:26:24) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/share.js:65:18 at OperatorSubscriber. (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/lift.js:14:28) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:30:30 at Object.errorContext (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/errorContext.js:22:9) at Observable.subscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:26:24) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/map.js:9:16 at OperatorSubscriber. (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/lift.js:14:28) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:30:30 at Object.errorContext (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/errorContext.js:22:9) at Observable.subscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:26:24) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/distinctUntilChanged.js:13:16 at OperatorSubscriber. (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/lift.js:14:28) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:30:30 at Object.errorContext (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/errorContext.js:22:9) at Observable.subscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:26:24) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/map.js:9:16 at OperatorSubscriber. (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/lift.js:14:28) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:30:30 at Object.errorContext (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/errorContext.js:22:9) at Observable.subscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:26:24) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/take.js:13:20 at OperatorSubscriber. (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/lift.js:14:28) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:30:30 at Object.errorContext (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/errorContext.js:22:9) at Observable.subscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:26:24) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/operators/throwIfEmpty.js:11:16 at SafeSubscriber. (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/lift.js:14:28) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:30:30 at Object.errorContext (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/util/errorContext.js:22:9) at Observable.subscribe (/usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:26:24) at /usr/share/kibana/node_modules/rxjs/dist/cjs/internal/Observable.js:86:19 at new Promise ()

FATAL Error: [config validation of [xpack.fleet].agentPolicies.1.package_policies.0.inputs]: expected value of type [array] but got [Object]

elasticmachine commented 2 weeks ago

Pinging @elastic/fleet (Team:Fleet)

kpollich commented 1 week ago

It seems like our example preconfiguration was updated to use the key/value syntax instead of the array based syntax for inputs, which should be supported as of https://github.com/elastic/kibana/pull/179087.

I am not able to reproduce the validation error on latest Kibana, so this looks to be working as intended as far as I can tell.

@gaimes410 can you confirm that there's no extra indentation or anything in your kibana.yml file that's causing the YML to be parsed incorrectly?

nchaulet commented 1 week ago

@gaimes410 @kpollich Will be great to confirm the kibana version too, from the logs it may be on 8.13.4 and we only support this from 8.14.0

gaimes410 commented 1 week ago

Hi, I tested it again and there is no extra indentation in the kibana.yml file, the packages are being installed just fine with the same indentation

I have noticed that the version was too old in this specific run but even on later runs where kibana version is set to 8.14.1 I get the following logs:

Defaulted container "kibana" out of: kibana, elastic-internal-init-config (init) {"log.level":"info","@timestamp":"2024-06-22T20:34:32.356Z","log.logger":"elastic-apm-node","ecs.version":"8.10.0","agentVersion":"4.5.0","env":{"pid":7,"proctitle":"/usr/share/kibana/bin/../node/bin/node","os":"linux 5.4.0-77-generic","arch":"x64","host":"kibana-quickstart-kb-64b889f646-xdqt6","timezone":"UTC+00","runtime":"Node.js v20.13.1"},"config":{"active":{"source":"start","value":true},"breakdownMetrics":{"source":"start","value":false},"captureBody":{"source":"start","value":"off","commonName":"capture_body"},"captureHeaders":{"source":"start","value":false},"centralConfig":{"source":"start","value":false},"contextPropagationOnly":{"source":"start","value":true},"environment":{"source":"start","value":"production"},"globalLabels":{"source":"start","value":[["git_rev","afbd904e868f2a48a2bbeb8ff20baee8d4aeb468"]],"sourceValue":{"git_rev":"afbd904e868f2a48a2bbeb8ff20baee8d4aeb468"}},"logLevel":{"source":"default","value":"info","commonName":"log_level"},"metricsInterval":{"source":"start","value":120,"sourceValue":"120s"},"serverUrl":{"source":"start","value":"https://kibana-cloud-apm.apm.us-east-1.aws.found.io/","commonName":"server_url"},"transactionSampleRate":{"source":"start","value":0.1,"commonName":"transaction_sample_rate"},"captureSpanStackTraces":{"source":"start","sourceValue":false},"secretToken":{"source":"start","value":"[REDACTED]","commonName":"secret_token"},"serviceName":{"source":"start","value":"kibana","commonName":"service_name"},"serviceVersion":{"source":"start","value":"8.14.1","commonName":"service_version"}},"activationMethod":"require","message":"Elastic APM Node.js Agent v4.5.0"} Native global console methods have been overridden in production environment. [2024-06-22T20:34:33.850+00:00][INFO ][root] Kibana is starting [2024-06-22T20:34:33.945+00:00][INFO ][node] Kibana process configured with roles: [background_tasks, ui] [2024-06-22T20:34:42.453+00:00][INFO ][root] Kibana is shutting down [2024-06-22T20:34:42.456+00:00][FATAL][root] Reason: [config validation of [xpack.fleet].agentPolicies.0.package_policies.0]: types that failed validation:

@kpollich may I ask for your sample kibana.yml file?