search

redpanda-data / helm-charts

Redpanda Helm Chart
http://redpanda.com
Apache License 2.0
77 stars 96 forks source link

Users not created from mTLS certificates #857

Open JakeSCahill opened 1 year ago

JakeSCahill commented 1 year ago

What happened?

When you enable mTLS authentication in the chart, the Redpanda logs are flooded with errors, which suggest that users are not correctly created from the default self-signed certificates:

ERROR 2023-11-08 12:03:45,814 [shard 0] kafka - server.cc:156 - Error[applying protocol] remote address: 10.244.2.9:59544 - std::__1::system_error (error GnuTLS:-112, Certificate is required.)
WARN  2023-11-08 12:03:50,079 [shard 0] request_auth - request_auth.cc:113 - Client auth failure: user '{}' not found
WARN  2023-11-08 12:03:50,084 [shard 0] request_auth - request_auth.cc:113 - Client auth failure: user '{}' not found

What did you expect to happen?

These errors should not appear.

How can we reproduce it (as minimally and precisely as possible)?. Please include values file.

helm upgrade --install redpanda redpanda/redpanda \
  --namespace redpanda \
  --create-namespace \
  --set external.domain=${DOMAIN} \
  --set statefulset.initContainers.setDataDirOwnership.enabled=true --set statefulset.sideCars.controllers.enabled=true --set rbac.enabled=true --set statefulset.sideCars.controllers.image.tag=v23.2.14 --set statefulset.replicas=5 --set listeners.kafka.authenticationMethod=mtls_identity --set listeners.kafka.tls.requireClientAuth=true --set auth.sasl.enabled=true --set auth.sasl.secretRef=redpanda-superusers
```console $ helm get values -n --all COMPUTED VALUES: affinity: {} auth: sasl: enabled: true mechanism: SCRAM-SHA-512 secretRef: redpanda-superusers users: [] clusterDomain: cluster.local commonLabels: {} config: cluster: default_topic_replications: 3 node: crash_loop_limit: 5 pandaproxy_client: {} rpk: {} schema_registry_client: {} tunable: compacted_log_segment_size: 67108864 group_topic_partitions: 16 kafka_batch_max_bytes: 1048576 kafka_connection_rate_limit: 1000 log_segment_size: 134217728 log_segment_size_max: 268435456 log_segment_size_min: 16777216 max_compacted_log_segment_size: 536870912 topic_partitions_per_shard: 1000 connectors: auth: sasl: enabled: false mechanism: scram-sha-512 secretRef: "" userName: "" commonLabels: {} connectors: additionalConfiguration: "" bootstrapServers: "" brokerTLS: ca: secretNameOverwrite: "" secretRef: "" cert: secretNameOverwrite: "" secretRef: "" enabled: false key: secretNameOverwrite: "" secretRef: "" groupID: connectors-cluster producerBatchSize: 131072 producerLingerMS: 1 restPort: 8083 schemaRegistryURL: "" secretManager: connectorsPrefix: "" consolePrefix: "" enabled: false region: "" storage: remote: read: config: false offset: false status: false write: config: false offset: false status: false replicationFactor: config: -1 offset: -1 status: -1 topic: config: _internal_connectors_configs offset: _internal_connectors_offsets status: _internal_connectors_status container: javaGCLogEnabled: "false" resources: javaMaxHeapSize: 2G limits: cpu: 1 memory: 2350Mi request: cpu: 1 memory: 2350Mi securityContext: allowPrivilegeEscalation: false deployment: annotations: {} budget: maxUnavailable: 1 create: false extraEnv: [] livenessProbe: failureThreshold: 3 initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 nodeAffinity: {} nodeSelector: {} podAffinity: {} podAntiAffinity: custom: {} topologyKey: kubernetes.io/hostname type: hard weight: 100 priorityClassName: "" progressDeadlineSeconds: 600 readinessProbe: failureThreshold: 2 initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 3 timeoutSeconds: 5 restartPolicy: Always revisionHistoryLimit: 10 schedulerName: "" securityContext: fsGroup: 101 fsGroupChangePolicy: OnRootMismatch runAsUser: 101 strategy: type: RollingUpdate terminationGracePeriodSeconds: 30 tolerations: [] topologySpreadConstraints: - maxSkew: 1 topologyKey: topology.kubernetes.io/zone whenUnsatisfiable: ScheduleAnyway updateStrategy: type: RollingUpdate enabled: false fullnameOverride: "" global: {} image: pullPolicy: IfNotPresent repository: docker.redpanda.com/redpandadata/connectors tag: "" imagePullSecrets: [] logging: level: warn monitoring: annotations: {} enabled: false labels: {} namespaceSelector: any: true scrapeInterval: 30s nameOverride: "" service: annotations: {} name: "" ports: - name: prometheus port: 9404 serviceAccount: annotations: {} create: false name: "" storage: volume: - emptyDir: medium: Memory sizeLimit: 5Mi name: rp-connect-tmp volumeMounts: - mountPath: /tmp name: rp-connect-tmp test: create: false tolerations: [] console: affinity: {} annotations: {} autoscaling: enabled: false maxReplicas: 100 minReplicas: 1 targetCPUUtilizationPercentage: 80 config: {} configmap: create: false console: config: {} deployment: create: false enabled: true enterprise: licenseSecretRef: key: "" name: "" extraContainers: [] extraEnv: [] extraEnvFrom: [] extraVolumeMounts: [] extraVolumes: [] fullnameOverride: "" global: {} image: pullPolicy: IfNotPresent registry: docker.redpanda.com repository: redpandadata/console tag: "" imagePullSecrets: [] ingress: annotations: {} className: "" enabled: false hosts: - host: chart-example.local paths: - path: / pathType: ImplementationSpecific tls: [] initContainers: extraInitContainers: "" livenessProbe: failureThreshold: 3 initialDelaySeconds: 0 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 nameOverride: "" nodeSelector: {} podAnnotations: {} podLabels: {} podSecurityContext: fsGroup: 99 runAsUser: 99 priorityClassName: "" readinessProbe: failureThreshold: 3 initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 1 replicaCount: 1 resources: {} secret: create: false enterprise: {} kafka: {} login: github: {} google: {} jwtSecret: "" oidc: {} okta: {} redpanda: adminApi: {} secretMounts: [] securityContext: runAsNonRoot: true service: annotations: {} port: 8080 type: ClusterIP serviceAccount: annotations: {} create: true name: "" tolerations: [] topologySpreadConstraints: {} enterprise: license: "" licenseSecretRef: {} external: domain: customredpandadomain.local enabled: true service: enabled: true type: NodePort fullnameOverride: "" image: pullPolicy: IfNotPresent repository: docker.redpanda.com/redpandadata/redpanda tag: "" imagePullSecrets: [] license_key: "" license_secret_ref: {} listeners: admin: external: default: advertisedPorts: - 31644 port: 9645 tls: cert: external port: 9644 tls: cert: default requireClientAuth: false http: authenticationMethod: null enabled: true external: default: advertisedPorts: - 30082 authenticationMethod: null port: 8083 tls: cert: external requireClientAuth: false kafkaEndpoint: default port: 8082 tls: cert: default requireClientAuth: false kafka: authenticationMethod: mtls_identity external: default: advertisedPorts: - 31092 authenticationMethod: null port: 9094 tls: cert: external port: 9093 tls: cert: default requireClientAuth: true rpc: port: 33145 tls: cert: default requireClientAuth: false schemaRegistry: authenticationMethod: null enabled: true external: default: advertisedPorts: - 30081 authenticationMethod: null port: 8084 tls: cert: external requireClientAuth: false kafkaEndpoint: default port: 8081 tls: cert: default requireClientAuth: false logging: logLevel: info usageStats: enabled: true monitoring: enabled: false labels: {} scrapeInterval: 30s tlsConfig: {} nameOverride: "" nodeSelector: {} post_install_job: affinity: {} enabled: true post_upgrade_job: affinity: {} enabled: true rackAwareness: enabled: false nodeAnnotation: topology.kubernetes.io/zone rbac: annotations: {} enabled: true resources: cpu: cores: 1 memory: container: max: 2.5Gi serviceAccount: annotations: {} create: false name: "" statefulset: additionalRedpandaCmdFlags: [] annotations: {} budget: maxUnavailable: 1 extraVolumeMounts: "" extraVolumes: "" initContainerImage: repository: busybox tag: latest initContainers: configurator: extraVolumeMounts: "" resources: {} extraInitContainers: "" setDataDirOwnership: enabled: true extraVolumeMounts: "" resources: {} setTieredStorageCacheDirOwnership: extraVolumeMounts: "" resources: {} tuning: extraVolumeMounts: "" resources: {} livenessProbe: failureThreshold: 3 initialDelaySeconds: 10 periodSeconds: 10 nodeSelector: {} podAffinity: {} podAntiAffinity: custom: {} topologyKey: kubernetes.io/hostname type: hard weight: 100 priorityClassName: "" readinessProbe: failureThreshold: 3 initialDelaySeconds: 1 periodSeconds: 10 successThreshold: 1 replicas: 5 securityContext: fsGroup: 101 fsGroupChangePolicy: OnRootMismatch runAsUser: 101 sideCars: configWatcher: enabled: true extraVolumeMounts: "" resources: {} securityContext: {} controllers: createRBAC: true enabled: true healthProbeAddress: :8085 image: repository: docker.redpanda.com/redpandadata/redpanda-operator tag: v23.2.14 metricsAddress: :9082 resources: {} run: - all securityContext: {} startupProbe: failureThreshold: 120 initialDelaySeconds: 1 periodSeconds: 10 terminationGracePeriodSeconds: 90 tolerations: [] topologySpreadConstraints: - maxSkew: 1 topologyKey: topology.kubernetes.io/zone whenUnsatisfiable: ScheduleAnyway updateStrategy: type: RollingUpdate storage: hostPath: "" persistentVolume: annotations: {} enabled: true labels: {} size: 20Gi storageClass: "" tiered: config: cloud_storage_access_key: "" cloud_storage_api_endpoint: "" cloud_storage_azure_container: null cloud_storage_azure_shared_key: null cloud_storage_azure_storage_account: null cloud_storage_bucket: "" cloud_storage_cache_size: 5368709120 cloud_storage_credentials_source: config_file cloud_storage_enable_remote_read: true cloud_storage_enable_remote_write: true cloud_storage_enabled: false cloud_storage_region: "" cloud_storage_secret_key: "" hostPath: "" mountType: emptyDir persistentVolume: annotations: {} labels: {} storageClass: "" tls: certs: default: caEnabled: true external: caEnabled: true enabled: true tolerations: [] tuning: tune_aio_events: true ```

Anything else we need to know?

No response

Which are the affected charts?

No response

Chart Version(s)

```console $ helm -n list # paste output here ```

Cloud provider

JIRA Link: K8S-74

alejandroEsc commented 1 year ago

@JakeSCahill are you creating users from a list or is there a secret named 'redpanda-superusers'?

JakeSCahill commented 1 year ago

There’s a Secret with the super user credentials

JakeSCahill commented 3 months ago

Tested this again. Can someone confirm if this config should be valid?

helm repo update
helm install redpanda redpanda/redpanda \
  --version 5.9.0 \
  --namespace jake \
  --create-namespace \
  --set external.domain=customredpandadomain.local \
  --set statefulset.initContainers.setDataDirOwnership.enabled=true --set "auth.sasl.users[0].name=superuser" --set auth.sasl.enabled=true --set "auth.sasl.users[0].password=secretpassword" --set "auth.sasl.users[0].mechanism=SCRAM-SHA-512" --set listeners.kafka.authenticationMethod=mtls_identity --set listeners.kafka.tls.requireClientAuth=true

When I try to use the Kafka API, rpk seems not to have the correct certs:

kubectl exec redpanda-0 -n jake -- rpk topic create test -X user=superuser -X pass=secretpassword
Defaulted container "redpanda" out of: redpanda, config-watcher, tuning (init), set-datadir-ownership (init), redpanda-configurator (init)
unable to initialize kafka client: unable to read cert at "/etc/tls/certs/redpanda-client/tls.crt": open /etc/tls/certs/redpanda-client/tls.crt: no such file or directory
command terminated with exit code 1

And the Redpanda logs complain too:

INFO  2024-08-09 14:03:37,998 [shard 0:main] tx - [{kafka/_internal_connectors_offsets/11}] - rm_stm.cc:106 - Setting bootstrap committed offset to: 2
INFO  2024-08-09 14:03:37,998 [shard 0:main] raft - [group_id:13, {kafka/_internal_connectors_offsets/11}] vote_stm.cc:433 - became the leader term: 3
INFO  2024-08-09 14:03:38,000 [shard 0:main] raft - [group_id:44, {kafka/__consumer_offsets/11}] vote_stm.cc:433 - became the leader term: 4
INFO  2024-08-09 14:03:38,088 [shard 0:main] tx - [{kafka/_internal_connectors_offsets/9}] - rm_stm.cc:106 - Setting bootstrap committed offset to: 3
INFO  2024-08-09 14:03:38,088 [shard 0:main] tx - [{kafka/_internal_connectors_offsets/17}] - rm_stm.cc:106 - Setting bootstrap committed offset to: 2
WARN  2024-08-09 14:03:38,477 [shard 0:admi] request_auth - request_auth.cc:123 - Client auth failure: user '' not found
WARN  2024-08-09 14:03:38,480 [shard 0:admi] request_auth - request_auth.cc:123 - Client auth failure: user '' not found
WARN  2024-08-09 14:03:38,485 [shard 0:admi] request_auth - request_auth.cc:123 - Client auth failure: user '' not found
INFO  2024-08-09 14:03:38,491 [shard 0:main] cluster - health_monitor_backend.cc:424 - received node 1 health report, marking node as up
WARN  2024-08-09 14:03:38,492 [shard 0:main] cluster - metadata_dissemination_service.cc:436 - Error sending metadata update rpc::errc::exponential_backoff to 2
INFO  2024-08-09 14:03:38,492 [shard 0:main] cluster - partition_balancer_planner.cc:334 - node 2 is unresponsive, time since last status reply: 3042 ms
INFO  2024-08-09 14:03:38,492 [shard 0:main] cluster - partition_balancer_planner.cc:365 - node 2: no disk report
INFO  2024-08-09 14:03:38,724 [shard 0:main] cluster - members_manager.cc:411 - applying maintenance_mode_cmd, offset: 740, node id: 2, enabled: false
INFO  2024-08-09 14:03:38,724 [shard 0:main] cluster - members_table.cc:210 - marking node 2 not in maintenance state
JakeSCahill commented 3 months ago

redpanda configmap:

Name:         redpanda
Namespace:    jake
Labels:       app.kubernetes.io/component=redpanda
              app.kubernetes.io/instance=redpanda
              app.kubernetes.io/managed-by=Helm
              app.kubernetes.io/name=redpanda
              helm.sh/chart=redpanda-5.9.0
Annotations:  meta.helm.sh/release-name: redpanda
              meta.helm.sh/release-namespace: jake

Data
====
bootstrap.yaml:
----
audit_enabled: false
compacted_log_segment_size: 67108864
default_topic_replications: 3
enable_rack_awareness: false
enable_sasl: true
group_topic_partitions: 16
kafka_batch_max_bytes: 1048576
kafka_connection_rate_limit: 1000
kafka_enable_authorization: true
log_segment_size: 134217728
log_segment_size_max: 268435456
log_segment_size_min: 16777216
max_compacted_log_segment_size: 536870912
storage_min_free_bytes: 1073741824
superusers:
- superuser
topic_partitions_per_shard: 1000
redpanda.yaml:
----
config_file: /etc/redpanda/redpanda.yaml
pandaproxy:
  pandaproxy_api:
  - address: 0.0.0.0
    authentication_method: http_basic
    name: internal
    port: 8082
  - address: 0.0.0.0
    authentication_method: http_basic
    name: default
    port: 8083
  pandaproxy_api_tls:
  - cert_file: /etc/tls/certs/default/tls.crt
    enabled: true
    key_file: /etc/tls/certs/default/tls.key
    name: internal
    require_client_auth: false
    truststore_file: /etc/tls/certs/default/ca.crt
  - cert_file: /etc/tls/certs/external/tls.crt
    enabled: true
    key_file: /etc/tls/certs/external/tls.key
    name: default
    require_client_auth: false
    truststore_file: /etc/tls/certs/external/ca.crt
pandaproxy_client:
  broker_tls:
    cert_file: /etc/tls/certs/default/tls.crt
    enabled: true
    key_file: /etc/tls/certs/default/tls.key
    require_client_auth: true
    truststore_file: /etc/tls/certs/default/ca.crt
  brokers:
  - address: redpanda-0.redpanda.jake.svc.cluster.local.
    port: 9093
  - address: redpanda-1.redpanda.jake.svc.cluster.local.
    port: 9093
  - address: redpanda-2.redpanda.jake.svc.cluster.local.
    port: 9093
redpanda:
  admin:
  - address: 0.0.0.0
    name: internal
    port: 9644
  - address: 0.0.0.0
    name: default
    port: 9645
  admin_api_tls:
  - cert_file: /etc/tls/certs/default/tls.crt
    enabled: true
    key_file: /etc/tls/certs/default/tls.key
    name: internal
    require_client_auth: false
    truststore_file: /etc/tls/certs/default/ca.crt
  - cert_file: /etc/tls/certs/external/tls.crt
    enabled: true
    key_file: /etc/tls/certs/external/tls.key
    name: default
    require_client_auth: false
    truststore_file: /etc/tls/certs/external/ca.crt
  audit_enabled: false
  compacted_log_segment_size: 67108864
  crash_loop_limit: 5
  default_topic_replications: 3
  empty_seed_starts_cluster: false
  enable_sasl: true
  group_topic_partitions: 16
  kafka_api:
  - address: 0.0.0.0
    authentication_method: mtls_identity
    name: internal
    port: 9093
  - address: 0.0.0.0
    authentication_method: sasl
    name: default
    port: 9094
  kafka_api_tls:
  - cert_file: /etc/tls/certs/default/tls.crt
    enabled: true
    key_file: /etc/tls/certs/default/tls.key
    name: internal
    require_client_auth: true
    truststore_file: /etc/tls/certs/default/ca.crt
  - cert_file: /etc/tls/certs/external/tls.crt
    enabled: true
    key_file: /etc/tls/certs/external/tls.key
    name: default
    require_client_auth: false
    truststore_file: /etc/tls/certs/external/ca.crt
  kafka_batch_max_bytes: 1048576
  kafka_connection_rate_limit: 1000
  kafka_enable_authorization: true
  log_segment_size: 134217728
  log_segment_size_max: 268435456
  log_segment_size_min: 16777216
  max_compacted_log_segment_size: 536870912
  rpc_server:
    address: 0.0.0.0
    port: 33145
  rpc_server_tls:
    cert_file: /etc/tls/certs/default/tls.crt
    enabled: true
    key_file: /etc/tls/certs/default/tls.key
    require_client_auth: false
    truststore_file: /etc/tls/certs/default/ca.crt
  seed_servers:
  - host:
      address: redpanda-0.redpanda.jake.svc.cluster.local.
      port: 33145
  - host:
      address: redpanda-1.redpanda.jake.svc.cluster.local.
      port: 33145
  - host:
      address: redpanda-2.redpanda.jake.svc.cluster.local.
      port: 33145
  storage_min_free_bytes: 1073741824
  superusers:
  - superuser
  topic_partitions_per_shard: 1000
rpk:
  additional_start_flags:
  - --default-log-level=info
  - --memory=2048M
  - --reserve-memory=205M
  - --smp=1
  admin_api:
    addresses:
    - redpanda-0.redpanda.jake.svc.cluster.local.:9644
    - redpanda-1.redpanda.jake.svc.cluster.local.:9644
    - redpanda-2.redpanda.jake.svc.cluster.local.:9644
    tls:
      truststore_file: /etc/tls/certs/default/ca.crt
  enable_memory_locking: false
  kafka_api:
    brokers:
    - redpanda-0.redpanda.jake.svc.cluster.local.:9093
    - redpanda-1.redpanda.jake.svc.cluster.local.:9093
    - redpanda-2.redpanda.jake.svc.cluster.local.:9093
    tls:
      cert_file: /etc/tls/certs/redpanda-client/tls.crt
      key_file: /etc/tls/certs/redpanda-client/tls.key
      truststore_file: /etc/tls/certs/default/ca.crt
  overprovisioned: false
  tune_aio_events: true
schema_registry:
  schema_registry_api:
  - address: 0.0.0.0
    authentication_method: http_basic
    name: internal
    port: 8081
  - address: 0.0.0.0
    authentication_method: http_basic
    name: default
    port: 8084
  schema_registry_api_tls:
  - cert_file: /etc/tls/certs/default/tls.crt
    enabled: true
    key_file: /etc/tls/certs/default/tls.key
    name: internal
    require_client_auth: false
    truststore_file: /etc/tls/certs/default/ca.crt
  - cert_file: /etc/tls/certs/external/tls.crt
    enabled: true
    key_file: /etc/tls/certs/external/tls.key
    name: default
    require_client_auth: false
    truststore_file: /etc/tls/certs/external/ca.crt
schema_registry_client:
  broker_tls:
    cert_file: /etc/tls/certs/default/tls.crt
    enabled: true
    key_file: /etc/tls/certs/default/tls.key
    require_client_auth: true
    truststore_file: /etc/tls/certs/default/ca.crt
  brokers:
  - address: redpanda-0.redpanda.jake.svc.cluster.local.
    port: 9093
  - address: redpanda-1.redpanda.jake.svc.cluster.local.
    port: 9093
  - address: redpanda-2.redpanda.jake.svc.cluster.local.
    port: 9093

BinaryData
====