search

elastic / helm-charts

You know, for Kubernetes
Apache License 2.0
1.89k stars 1.93k forks source link

UPGRADE FAILED: timed out waiting for the condition #1434

Closed dcvtruong closed 2 years ago

dcvtruong commented 3 years ago

What could be the condition causing the pods to remained stuck in Readiness probe failed: Waiting for elasticsearch cluster to become ready (request params: "wait_for_status=green&timeout=1s" )?

Chart version: 7.13.4 Kubernetes version: 1.20.7 Kubernetes provider: E.g. GKE (Google Kubernetes Engine) AKS Helm Version: 3.4

helm get release output

NAME: helm-es-security
LAST DEPLOYED: Thu Oct 28 10:07:45 2021
NAMESPACE: elk
STATUS: failed
REVISION: 10
USER-SUPPLIED VALUES:
clusterName: security
esConfig:
  elasticsearch.yml: |
    xpack.security.enabled: true
    xpack.security.transport.ssl.enabled: true
    xpack.security.transport.ssl.verification_mode: certificate
    xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.http.ssl.enabled: true
    xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
esJavaOpts: -Xmx7g -Xms7g
extraEnvs:
- name: ELASTIC_PASSWORD
  valueFrom:
    secretKeyRef:
      key: password
      name: elastic-credentials
- name: ELASTIC_USERNAME
  valueFrom:
    secretKeyRef:
      key: username
      name: elastic-credentials
keystore:
- secretName: sift-snapshot-credentials
nodeGroup: master
protocol: https
resources:
  limits:
    cpu: 4000m
    memory: 15Gi
  requests:
    cpu: 4000m
    memory: 15Gi
roles:
  data: "true"
  ingest: "true"
  master: "true"
secretMounts:
- name: elastic-certificates
  path: /usr/share/elasticsearch/config/certs
  secretName: elastic-certificates
service:
  nodePort: "30997"
  type: LoadBalancer
volumeClaimTemplate:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 290Gi

COMPUTED VALUES:
antiAffinity: hard
antiAffinityTopologyKey: kubernetes.io/hostname
clusterHealthCheckParams: wait_for_status=green&timeout=1s
clusterName: security
enableServiceLinks: true
envFrom: []
esConfig:
  elasticsearch.yml: |
    xpack.security.enabled: true
    xpack.security.transport.ssl.enabled: true
    xpack.security.transport.ssl.verification_mode: certificate
    xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.http.ssl.enabled: true
    xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
esJavaOpts: -Xmx7g -Xms7g
esMajorVersion: ""
extraContainers: []
extraEnvs:
- name: ELASTIC_PASSWORD
  valueFrom:
    secretKeyRef:
      key: password
      name: elastic-credentials
- name: ELASTIC_USERNAME
  valueFrom:
    secretKeyRef:
      key: username
      name: elastic-credentials
extraInitContainers:
- command:
  - sh
  - -c
  - bin/elasticsearch-plugin install --batch repository-azure
  image: docker.elastic.co/elasticsearch/elasticsearch:7.13.3
  imagePullPolicy: IfNotPresent
  name: install-plugins
  volumeMounts:
  - mountPath: /usr/share/elasticsearch/plugins
    name: plugins
extraVolumeMounts:
- mountPath: /usr/share/elasticsearch/plugins
  name: plugins
  readOnly: false
extraVolumes:
- emptyDir: {}
  name: plugins
fsGroup: ""
fullnameOverride: ""
healthNameOverride: ""
hostAliases: []
httpPort: 9200
image: docker.elastic.co/elasticsearch/elasticsearch
imagePullPolicy: IfNotPresent
imagePullSecrets: []
imageTag: 7.13.3
ingress:
  annotations: {}
  enabled: false
  hosts:
  - host: chart-example.local
    paths:
    - path: /
  tls: []
initResources:
  limits:
    cpu: 100m
    memory: 128Mi
  requests:
    cpu: 100m
    memory: 128Mi
keystore:
- secretName: sift-snapshot-credentials
labels: {}
lifecycle: {}
masterService: ""
maxUnavailable: 1
minimumMasterNodes: 2
nameOverride: ""
networkHost: 0.0.0.0
networkPolicy:
  http:
    enabled: false
  transport:
    enabled: false
nodeAffinity: {}
nodeGroup: master
nodeSelector: {}
persistence:
  annotations: {}
  enabled: true
  labels:
    enabled: false
podAnnotations: {}
podManagementPolicy: Parallel
podSecurityContext:
  fsGroup: 1000
  runAsUser: 1000
podSecurityPolicy:
  create: false
  name: ""
  spec:
    fsGroup:
      rule: RunAsAny
    privileged: true
    runAsUser:
      rule: RunAsAny
    seLinux:
      rule: RunAsAny
    supplementalGroups:
      rule: RunAsAny
    volumes:
    - secret
    - configMap
    - persistentVolumeClaim
    - emptyDir
priorityClassName: ""
protocol: https
rbac:
  create: false
  serviceAccountAnnotations: {}
  serviceAccountName: ""
readinessProbe:
  failureThreshold: 3
  initialDelaySeconds: 10
  periodSeconds: 10
  successThreshold: 3
  timeoutSeconds: 5
replicas: 3
resources:
  limits:
    cpu: 4000m
    memory: 15Gi
  requests:
    cpu: 4000m
    memory: 15Gi
roles:
  data: "true"
  ingest: "true"
  master: "true"
  ml: "true"
  remote_cluster_client: "true"
schedulerName: ""
secretMounts:
- name: elastic-certificates
  path: /usr/share/elasticsearch/config/certs
  secretName: elastic-certificates
securityContext:
  capabilities:
    drop:
    - ALL
  runAsNonRoot: true
  runAsUser: 1000
service:
  annotations: {}
  externalTrafficPolicy: ""
  httpPortName: http
  labels: {}
  labelsHeadless: {}
  loadBalancerIP: ""
  loadBalancerSourceRanges: []
  nodePort: "30997"
  transportPortName: transport
  type: LoadBalancer
sysctlInitContainer:
  enabled: true
sysctlVmMaxMapCount: 262144
terminationGracePeriod: 120
tolerations: []
transportPort: 9300
updateStrategy: RollingUpdate
volumeClaimTemplate:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 290Gi

HOOKS:
---
# Source: elasticsearch/templates/test/test-elasticsearch-health.yaml
apiVersion: v1
kind: Pod
metadata:
  name: "helm-es-security-zpmpe-test"
  annotations:
    "helm.sh/hook": test
    "helm.sh/hook-delete-policy": hook-succeeded
spec:
  securityContext:
    fsGroup: 1000
    runAsUser: 1000
  containers:
  - name: "helm-es-security-ximqr-test"
    image: "docker.elastic.co/elasticsearch/elasticsearch:7.13.3"
    imagePullPolicy: "IfNotPresent"
    command:
      - "sh"
      - "-c"
      - |
        #!/usr/bin/env bash -e
        curl -XGET --fail 'security-master:9200/_cluster/health?wait_for_status=green&timeout=1s'
  restartPolicy: Never
MANIFEST:
---
# Source: elasticsearch/templates/poddisruptionbudget.yaml
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: "security-master-pdb"
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      app: "security-master"
---
# Source: elasticsearch/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: security-master-config
  labels:
    heritage: "Helm"
    release: "helm-es-security"
    chart: "elasticsearch"
    app: "security-master"
data:
  elasticsearch.yml: |
    xpack.security.enabled: true
    xpack.security.transport.ssl.enabled: true
    xpack.security.transport.ssl.verification_mode: certificate
    xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.http.ssl.enabled: true
    xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
    xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12
---
# Source: elasticsearch/templates/service.yaml
kind: Service
apiVersion: v1
metadata:
  name: security-master
  labels:
    heritage: "Helm"
    release: "helm-es-security"
    chart: "elasticsearch"
    app: "security-master"
  annotations:
    {}
spec:
  type: LoadBalancer
  selector:
    release: "helm-es-security"
    chart: "elasticsearch"
    app: "security-master"
  ports:
  - name: http
    protocol: TCP
    port: 9200
    nodePort: 30997
  - name: transport
    protocol: TCP
    port: 9300
---
# Source: elasticsearch/templates/service.yaml
kind: Service
apiVersion: v1
metadata:
  name: security-master-headless
  labels:
    heritage: "Helm"
    release: "helm-es-security"
    chart: "elasticsearch"
    app: "security-master"
  annotations:
    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
spec:
  clusterIP: None # This is needed for statefulset hostnames like elasticsearch-0 to resolve
  # Create endpoints also if the related pod isn't ready
  publishNotReadyAddresses: true
  selector:
    app: "security-master"
  ports:
  - name: http
    port: 9200
  - name: transport
    port: 9300
---
# Source: elasticsearch/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: security-master
  labels:
    heritage: "Helm"
    release: "helm-es-security"
    chart: "elasticsearch"
    app: "security-master"
  annotations:
    esMajorVersion: "7"
spec:
  serviceName: security-master-headless
  selector:
    matchLabels:
      app: "security-master"
  replicas: 3
  podManagementPolicy: Parallel
  updateStrategy:
    type: RollingUpdate
  volumeClaimTemplates:
  - metadata:
      name: security-master
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 290Gi
  template:
    metadata:
      name: "security-master"
      labels:
        release: "helm-es-security"
        chart: "elasticsearch"
        app: "security-master"
      annotations:

        configchecksum: 00648dea96973bbf51ed7a7b54a2ce319f9c9f638f32cbcd6b2b21ed31078d4
    spec:
      securityContext:
        fsGroup: 1000
        runAsUser: 1000
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - "security-master"
            topologyKey: kubernetes.io/hostname
      terminationGracePeriodSeconds: 120
      volumes:
        - name: elastic-certificates
          secret:
            secretName: elastic-certificates
        - name: esconfig
          configMap:
            name: security-master-config
        - name: keystore
          emptyDir: {}
        - name: keystore-sift-snapshot-credentials
          secret: 
            secretName: sift-snapshot-credentials

      # Currently some extra blocks accept strings
      # to continue with backwards compatibility this is being kept
      # whilst also allowing for yaml to be specified too.
        - emptyDir: {}
          name: plugins
      enableServiceLinks: true
      initContainers:
      - name: configure-sysctl
        securityContext:
          runAsUser: 0
          privileged: true
        image: "docker.elastic.co/elasticsearch/elasticsearch:7.13.3"
        imagePullPolicy: "IfNotPresent"
        command: ["sysctl", "-w", "vm.max_map_count=262144"]
        resources:
          limits:
            cpu: 100m
            memory: 128Mi
          requests:
            cpu: 100m
            memory: 128Mi

      - name: keystore
        image: "docker.elastic.co/elasticsearch/elasticsearch:7.13.3"
        imagePullPolicy: "IfNotPresent"
        command:
        - sh
        - -c
        - |
          #!/usr/bin/env bash
          set -euo pipefail

          elasticsearch-keystore create

          for i in /tmp/keystoreSecrets/*/*; do
            key=$(basename $i)
            echo "Adding file $i to keystore key $key"
            elasticsearch-keystore add-file "$key" "$i"
          done

          # Add the bootstrap password since otherwise the Elasticsearch entrypoint tries to do this on startup
          if [ ! -z ${ELASTIC_PASSWORD+x} ]; then
            echo 'Adding env $ELASTIC_PASSWORD to keystore as key bootstrap.password'
            echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x bootstrap.password
          fi

          cp -a /usr/share/elasticsearch/config/elasticsearch.keystore /tmp/keystore/
        env: 
          - name: ELASTIC_PASSWORD
            valueFrom:
              secretKeyRef:
                key: password
                name: elastic-credentials
          - name: ELASTIC_USERNAME
            valueFrom:
              secretKeyRef:
                key: username
                name: elastic-credentials
        envFrom: 
          []
        resources: 
          limits:
            cpu: 100m
            memory: 128Mi
          requests:
            cpu: 100m
            memory: 128Mi
        volumeMounts:
          - name: keystore
            mountPath: /tmp/keystore
          - name: keystore-sift-snapshot-credentials
            mountPath: /tmp/keystoreSecrets/sift-snapshot-credentials

      # Currently some extra blocks accept strings
      # to continue with backwards compatibility this is being kept
      # whilst also allowing for yaml to be specified too.
      - command:
        - sh
        - -c
        - bin/elasticsearch-plugin install --batch repository-azure
        image: docker.elastic.co/elasticsearch/elasticsearch:7.13.3
        imagePullPolicy: IfNotPresent
        name: install-plugins
        volumeMounts:
        - mountPath: /usr/share/elasticsearch/plugins
          name: plugins
      containers:
      - name: "elasticsearch"
        securityContext:
          capabilities:
            drop:
            - ALL
          runAsNonRoot: true
          runAsUser: 1000
        image: "docker.elastic.co/elasticsearch/elasticsearch:7.13.3"
        imagePullPolicy: "IfNotPresent"
        readinessProbe:
          exec:
            command:
              - sh
              - -c
              - |
                #!/usr/bin/env bash -e
                # If the node is starting up wait for the cluster to be ready (request params: "wait_for_status=green&timeout=1s" )
                # Once it has started only check that the node itself is responding
                START_FILE=/tmp/.es_start_file

                # Disable nss cache to avoid filling dentry cache when calling curl
                # This is required with Elasticsearch Docker using nss < 3.52
                export NSS_SDB_USE_CACHE=no

                http () {
                  local path="${1}"
                  local args="${2}"
                  set -- -XGET -s

                  if [ "$args" != "" ]; then
                    set -- "$@" $args
                  fi

                  if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then
                    set -- "$@" -u "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}"
                  fi

                  curl --output /dev/null -k "$@" "https://127.0.0.1:9200${path}"
                }

                if [ -f "${START_FILE}" ]; then
                  echo 'Elasticsearch is already running, lets check the node is healthy'
                  HTTP_CODE=$(http "/" "-w %{http_code}")
                  RC=$?
                  if [[ ${RC} -ne 0 ]]; then
                    echo "curl --output /dev/null -k -XGET -s -w '%{http_code}' \${BASIC_AUTH} https://127.0.0.1:9200/ failed with RC ${RC}"
                    exit ${RC}
                  fi
                  # ready if HTTP code 200, 503 is tolerable if ES version is 6.x
                  if [[ ${HTTP_CODE} == "200" ]]; then
                    exit 0
                  elif [[ ${HTTP_CODE} == "503" && "7" == "6" ]]; then
                    exit 0
                  else
                    echo "curl --output /dev/null -k -XGET -s -w '%{http_code}' \${BASIC_AUTH} https://127.0.0.1:9200/ failed with HTTP code ${HTTP_CODE}"
                    exit 1
                  fi

                else
                  echo 'Waiting for elasticsearch cluster to become ready (request params: "wait_for_status=green&timeout=1s" )'
                  if http "/_cluster/health?wait_for_status=green&timeout=1s" "--fail" ; then
                    touch ${START_FILE}
                    exit 0
                  else
                    echo 'Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )'
                    exit 1
                  fi
                fi
          failureThreshold: 3
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 3
          timeoutSeconds: 5
        ports:
        - name: http
          containerPort: 9200
        - name: transport
          containerPort: 9300
        resources:
          limits:
            cpu: 4000m
            memory: 15Gi
          requests:
            cpu: 4000m
            memory: 15Gi
        env:
          - name: node.name
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: cluster.initial_master_nodes
            value: "security-master-0,security-master-1,security-master-2,"
          - name: discovery.seed_hosts
            value: "security-master-headless"
          - name: cluster.name
            value: "security"
          - name: network.host
            value: "0.0.0.0"
          - name: ES_JAVA_OPTS
            value: "-Xmx7g -Xms7g"
          - name: node.data
            value: "true"
          - name: node.ingest
            value: "true"
          - name: node.master
            value: "true"
          - name: node.ml
            value: "true"
          - name: node.remote_cluster_client
            value: "true"
          - name: ELASTIC_PASSWORD
            valueFrom:
              secretKeyRef:
                key: password
                name: elastic-credentials
          - name: ELASTIC_USERNAME
            valueFrom:
              secretKeyRef:
                key: username
                name: elastic-credentials
        volumeMounts:
          - name: "security-master"
            mountPath: /usr/share/elasticsearch/data

          - name: keystore
            mountPath: /usr/share/elasticsearch/config/elasticsearch.keystore
            subPath: elasticsearch.keystore

          - name: elastic-certificates
            mountPath: /usr/share/elasticsearch/config/certs
          - name: esconfig
            mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
            subPath: elasticsearch.yml
        # Currently some extra blocks accept strings
        # to continue with backwards compatibility this is being kept
        # whilst also allowing for yaml to be specified too.
          - mountPath: /usr/share/elasticsearch/plugins
            name: plugins
            readOnly: false

Describe the bug: Run the upgrade command and all three ES pods stuck in 'unhealthy' state

NAME                                           READY   STATUS    RESTARTS   AGE   IP            NODE                                NOMINATED NODE   READINESS GATES
helm-kibana-security-kibana-5fcf4c7647-xtdf5   0/1     Running   0          86m   10.244.3.22   aks-agentpool-93073037-vmss000003   <none>           <none>
security-master-0                              0/1     Running   0          36m   10.244.0.21   aks-agentpool-93073037-vmss000000   <none>           <none>
security-master-1                              0/1     Running   0          31m   10.244.1.15   aks-agentpool-93073037-vmss000002   <none>           <none>
security-master-2                              0/1     Running   0          41m   10.244.2.22   aks-agentpool-93073037-vmss000001   <none>           <none>

Steps to reproduce:

helm upgrade --wait --timeout=598s --install --namespace elk --values values.yaml helm-es-security ../../
Error: UPGRADE FAILED: timed out waiting for the condition

Expected behavior: ES cluster turn green after upgrade Provide logs and/or server output (if relevant):

Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )
30m         Normal    SuccessfulDelete       statefulset/security-master                               delete Pod security-master-2 in StatefulSet security-master successful
30m         Normal    SuccessfulCreate       statefulset/security-master                               create Pod security-master-2 in StatefulSet security-master successful
30m         Normal    Started                pod/security-master-2                                     Started container configure-sysctl
30m         Normal    Started                pod/security-master-2                                     Started container keystore
30m         Normal    Created                pod/security-master-2                                     Created container keystore
30m         Normal    Pulled                 pod/security-master-2                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
30m         Normal    Pulled                 pod/security-master-2                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
30m         Normal    Created                pod/security-master-2                                     Created container configure-sysctl
28m         Normal    Started                pod/security-master-2                                     Started container install-plugins
28m         Normal    Created                pod/security-master-2                                     Created container install-plugins
28m         Normal    Pulled                 pod/security-master-2                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
28m         Normal    Created                pod/security-master-2                                     Created container elasticsearch
28m         Normal    Pulled                 pod/security-master-2                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
28m         Normal    Started                pod/security-master-2                                     Started container elasticsearch
27m         Warning   Unhealthy              pod/security-master-0                                     Readiness probe failed: Waiting for elasticsearch cluster to become ready (request params: "wait_for_status=green&timeout=1s" )
Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )
25m         Normal    SuccessfulCreate         statefulset/security-master                               create Pod security-master-0 in StatefulSet security-master successful
25m         Normal    SuccessfulAttachVolume   pod/security-master-0                                     AttachVolume.Attach succeeded for volume "pvc-7857620e-c403-40ee-8a5d-97eb97129118"
25m         Normal    Pulled                   pod/security-master-0                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
25m         Normal    Started                  pod/security-master-0                                     Started container keystore
25m         Normal    Pulled                   pod/security-master-0                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
25m         Normal    Created                  pod/security-master-0                                     Created container keystore
25m         Normal    Started                  pod/security-master-0                                     Started container configure-sysctl
25m         Normal    Created                  pod/security-master-0                                     Created container configure-sysctl
24m         Warning   Unhealthy                pod/security-master-1                                     Readiness probe failed: Waiting for elasticsearch cluster to become ready (request params: "wait_for_status=green&timeout=1s" )
Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )
23m         Normal    Pulled                   pod/security-master-0                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
23m         Normal    Created                  pod/security-master-0                                     Created container install-plugins
23m         Normal    Started                  pod/security-master-0                                     Started container install-plugins
23m         Normal    Started                  pod/security-master-0                                     Started container elasticsearch
23m         Normal    Pulled                   pod/security-master-0                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
23m         Normal    Created                  pod/security-master-0                                     Created container elasticsearch
20m         Normal    SuccessfulCreate         statefulset/security-master                               create Pod security-master-1 in StatefulSet security-master successful
19m         Normal    SuccessfulAttachVolume   pod/security-master-1                                     AttachVolume.Attach succeeded for volume "pvc-9eca7023-4465-46fd-bb84-1a193320b716"
19m         Normal    Started                  pod/security-master-1                                     Started container configure-sysctl
19m         Normal    Created                  pod/security-master-1                                     Created container configure-sysctl
19m         Normal    Pulled                   pod/security-master-1                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
19m         Normal    Pulled                   pod/security-master-1                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
19m         Normal    Created                  pod/security-master-1                                     Created container keystore
19m         Normal    Started                  pod/security-master-1                                     Started container keystore
18m         Normal    Pulled                   pod/security-master-1                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
18m         Normal    Created                  pod/security-master-1                                     Created container install-plugins
18m         Normal    Started                  pod/security-master-1                                     Started container install-plugins
18m         Normal    Created                  pod/security-master-1                                     Created container elasticsearch
18m         Normal    Pulled                   pod/security-master-1                                     Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
18m         Normal    Started                  pod/security-master-1                                     Started container elasticsearch
4m58s       Warning   Unhealthy                pod/security-master-0                                     Readiness probe failed: Waiting for elasticsearch cluster to become ready (request params: "wait_for_status=green&timeout=1s" )
Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )
4m50s       Warning   Unhealthy                pod/helm-kibana-security-kibana-5fcf4c7647-xtdf5          Readiness probe failed: Error: Got HTTP code 503 but expected a 200
4m34s       Warning   Unhealthy                pod/security-master-1                                     Readiness probe failed: Waiting for elasticsearch cluster to become ready (request params: "wait_for_status=green&timeout=1s" )
Cluster is not yet ready (request params: "wait_for_status=green&timeout=1s" )
17s         Normal    Pulled                   pod/helm-es-security-zpmpe-test                           Container image "docker.elastic.co/elasticsearch/elasticsearch:7.13.3" already present on machine
5s          Warning   Unhealthy                pod/security-master-2                                     Readiness probe failed: Waiting for elasticsearch cluster to become ready (request params: "wait_for_status=green&timeout=1s" )

Logs: security-master-0:

{"type": "server", "timestamp": "2021-10-28T15:14:52,470Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-0", "message": "version[7.13.3], pid[6], build[default/docker/5d21bea28db1e89ecc1f66311ebdec9dc3aa7d64/2021-07-02T12:06:10.804015202Z], OS[Linux/5.4.0-1055-azure/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/16/16+36]" }
{"type": "server", "timestamp": "2021-10-28T15:14:52,472Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-0", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2021-10-28T15:14:52,472Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-0", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-11885240649581177606, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xmx7g, -Xms7g, -XX:MaxDirectMemorySize=3758096384, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,265Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-0", "message": "SLF4J: Failed to load class \"org.slf4j.impl.StaticLoggerBinder\"." }
{"type": "server", "timestamp": "2021-10-28T15:14:54,267Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-0", "message": "SLF4J: Defaulting to no-operation (NOP) logger implementation" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,267Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-0", "message": "SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details." }
{"type": "server", "timestamp": "2021-10-28T15:14:54,418Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,418Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,418Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,418Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,419Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,420Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,420Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,420Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,420Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,420Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,420Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,420Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,421Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,421Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,421Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,422Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,422Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,422Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,422Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,422Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,422Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,422Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,422Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,423Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,423Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,423Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,423Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,423Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,423Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,423Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,423Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,424Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,424Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,424Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,424Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,424Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,424Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,425Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,425Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,425Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,425Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,425Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,425Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,426Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,426Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,426Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,426Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,426Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,426Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,426Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,427Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,427Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-0", "message": "loaded plugin [repository-azure]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:14:54,430Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-0", "message": "[node.ml] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:14:54,446Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-0", "message": "[node.data] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-10-28T15:14:54,459Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "security", "node.name": "security-master-0", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sdc)]], net usable_space [268.4gb], net total_space [284.4gb], types [ext4]" }
{"type": "server", "timestamp": "2021-10-28T15:14:54,459Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "security", "node.name": "security-master-0", "message": "heap size [7gb], compressed ordinary object pointers [true]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:14:54,530Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-0", "message": "[node.master] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:14:54,534Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-0", "message": "[node.remote_cluster_client] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:14:54,535Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-0", "message": "[node.ingest] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-10-28T15:14:54,668Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-0", "message": "node name [security-master-0], node ID [aCrYyxJTSAae4NIqlRaa5Q], cluster name [security], roles [transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:14:54,669Z", "level": "DEPRECATION", "component": "o.e.d.n.Node", "cluster.name": "security", "node.name": "security-master-0", "message": "legacy role settings [node.data, node.remote_cluster_client, node.ingest, node.master, node.ml] are deprecated, use [node.roles=[transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]]" }
{"type": "server", "timestamp": "2021-10-28T15:14:58,109Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "security", "node.name": "security-master-0", "message": "[controller/359] [Main.cc@117] controller (64 bit): Version 7.13.3 (Build d74de47cfda9c3) Copyright (c) 2021 Elasticsearch BV" }
{"type": "server", "timestamp": "2021-10-28T15:14:58,405Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "security", "node.name": "security-master-0", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2021-10-28T15:14:58,887Z", "level": "INFO", "component": "o.e.i.g.LocalDatabases", "cluster.name": "security", "node.name": "security-master-0", "message": "initialized default databases [[GeoLite2-Country.mmdb, GeoLite2-City.mmdb, GeoLite2-ASN.mmdb]], config databases [[]] and watching [/usr/share/elasticsearch/config/ingest-geoip] for changes" }
{"type": "server", "timestamp": "2021-10-28T15:14:58,889Z", "level": "INFO", "component": "o.e.i.g.DatabaseRegistry", "cluster.name": "security", "node.name": "security-master-0", "message": "initialized database registry, using geoip-databases directory [/tmp/elasticsearch-11885240649581177606/geoip-databases/aCrYyxJTSAae4NIqlRaa5Q]" }
{"type": "server", "timestamp": "2021-10-28T15:14:59,424Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "security", "node.name": "security-master-0", "message": "creating NettyAllocator with the following configs: [name=elasticsearch_configured, chunk_size=1mb, suggested_max_allocation_size=1mb, factors={es.unsafe.use_netty_default_chunk_and_page_size=false, g1gc_enabled=true, g1gc_region_size=4mb}]" }
{"type": "server", "timestamp": "2021-10-28T15:14:59,480Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "security", "node.name": "security-master-0", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2021-10-28T15:14:59,831Z", "level": "INFO", "component": "o.e.g.DanglingIndicesState", "cluster.name": "security", "node.name": "security-master-0", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
{"type": "server", "timestamp": "2021-10-28T15:15:00,185Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-0", "message": "initialized" }
{"type": "server", "timestamp": "2021-10-28T15:15:00,185Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-0", "message": "starting ..." }
{"type": "server", "timestamp": "2021-10-28T15:15:00,226Z", "level": "INFO", "component": "o.e.x.s.c.f.PersistentCache", "cluster.name": "security", "node.name": "security-master-0", "message": "persistent cache index loaded" }
{"type": "server", "timestamp": "2021-10-28T15:15:00,303Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "security", "node.name": "security-master-0", "message": "publish_address {10.244.0.21:9300}, bound_addresses {[::]:9300}" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:15:00,597Z", "level": "DEPRECATION", "component": "o.e.d.c.j.Joda", "cluster.name": "security", "node.name": "security-master-0", "message": "'y' year should be replaced with 'u'. Use 'y' for year-of-era. Use new java.time date format specifiers." }
{"type": "server", "timestamp": "2021-10-28T15:15:01,487Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "security", "node.name": "security-master-0", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2021-10-28T15:15:01,489Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "security", "node.name": "security-master-0", "message": "cluster UUID [B5tIJoMcQ7ee3ybA-Hg5UA]" }
{"type": "server", "timestamp": "2021-10-28T15:15:02,859Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-0", "message": "master node changed {previous [], current [{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}]}, added {{security-master-2}{FfFqbAGDR0aYafc8UNVnMg}{yB7tniVIQsaMQ7N6iMBAIw}{10.244.2.22}{10.244.2.22:9300}{cdfhilmrstw}, {security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}}, term: 8, version: 2421, reason: ApplyCommitRequest{term=8, version=2421, sourceNode={security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}}" }
{"type": "server", "timestamp": "2021-10-28T15:15:02,869Z", "level": "INFO", "component": "o.e.c.s.ClusterSettings", "cluster.name": "security", "node.name": "security-master-0", "message": "updating [xpack.monitoring.collection.enabled] from [false] to [true]" }
{"type": "server", "timestamp": "2021-10-28T15:15:03,049Z", "level": "INFO", "component": "o.e.x.s.a.TokenService", "cluster.name": "security", "node.name": "security-master-0", "message": "refresh keys" }
{"type": "server", "timestamp": "2021-10-28T15:15:03,215Z", "level": "INFO", "component": "o.e.x.s.a.TokenService", "cluster.name": "security", "node.name": "security-master-0", "message": "refreshed keys" }
{"type": "server", "timestamp": "2021-10-28T15:15:03,259Z", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "security", "node.name": "security-master-0", "message": "license [949b4c5e-ffdb-418a-8774-e67b0b210a1a] mode [basic] - valid" }
{"type": "server", "timestamp": "2021-10-28T15:15:03,260Z", "level": "INFO", "component": "o.e.x.s.s.SecurityStatusChangeListener", "cluster.name": "security", "node.name": "security-master-0", "message": "Active license is now [BASIC]; Security is enabled" }
{"type": "server", "timestamp": "2021-10-28T15:15:03,271Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "security", "node.name": "security-master-0", "message": "publish_address {10.244.0.21:9200}, bound_addresses {[::]:9200}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:15:03,272Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-0", "message": "started", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:01,744Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "security", "node.name": "security-master-0", "message": "master node [{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}] failed, restarting discovery", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q" , 
"stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [security-master-1][10.244.1.12:9300][disconnected] disconnected"] }
{"type": "server", "timestamp": "2021-10-28T15:18:01,748Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-0", "message": "master node changed {previous [{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}], current []}, term: 8, version: 2507, reason: becoming candidate: onLeaderFailure", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:01,752Z", "level": "WARN", "component": "o.e.c.NodeConnectionsService", "cluster.name": "security", "node.name": "security-master-0", "message": "failed to connect to {security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true} (tried [1] times)", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q" , 
"stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [security-master-1][10.244.1.12:9300] connect_exception",
"at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:971) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:277) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:31) ~[elasticsearch-core-7.13.3.jar:7.13.3]",
"at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
"at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
"at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
"at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2158) ~[?:?]",
"at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:46) ~[elasticsearch-core-7.13.3.jar:7.13.3]",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:57) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10.244.1.12/10.244.1.12:9300",
"Caused by: java.net.ConnectException: Connection refused",
"at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
"at sun.nio.ch.Net.pollConnectNow(Net.java:669) ~[?:?]",
"at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:944) ~[?:?]",
"at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]",
"... 7 more"] }
{"type": "server", "timestamp": "2021-10-28T15:18:01,847Z", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "security", "node.name": "security-master-0", "message": "elected-as-master ([2] nodes joined)[{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw} elect leader, {security-master-2}{FfFqbAGDR0aYafc8UNVnMg}{yB7tniVIQsaMQ7N6iMBAIw}{10.244.2.22}{10.244.2.22:9300}{cdfhilmrstw} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 9, version: 2508, delta: master node changed {previous [], current [{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}]}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:01,977Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-0", "message": "master node changed {previous [], current [{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}]}, term: 9, version: 2508, reason: Publication{term=9, version=2508}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:01,997Z", "level": "WARN", "component": "o.e.c.InternalClusterInfoService", "cluster.name": "security", "node.name": "security-master-0", "message": "failed to retrieve stats for node [jBtim_WeTDuMfgxEtgCrzQ]: [security-master-1][10.244.1.12:9300] Node not connected", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,004Z", "level": "WARN", "component": "o.e.c.InternalClusterInfoService", "cluster.name": "security", "node.name": "security-master-0", "message": "failed to retrieve shard stats from node [jBtim_WeTDuMfgxEtgCrzQ]: [security-master-1][10.244.1.12:9300] Node not connected", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,092Z", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "security", "node.name": "security-master-0", "message": "node-left[{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw} reason: disconnected], term: 9, version: 2509, delta: removed {{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,876Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-0", "message": "removed {{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}}, term: 9, version: 2509, reason: Publication{term=9, version=2509}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,929Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [upload.error][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,964Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [.transform-notifications-000002][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,973Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.mexico][4] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,984Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.mexico][1] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,005Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.mexico][2] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,019Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.mexico][3] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,062Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.mexico][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,083Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [tika.error][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,102Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [.kibana-event-log-7.13.3-000001][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,106Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [filebeat-7.14.1-2021.09.01-000001][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,115Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.prd.user][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,168Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [metrics-endpoint.metadata_current_default][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,180Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.tst.test_storscan][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,199Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.suriname][4] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,217Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.suriname][1] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,230Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.suriname][2] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,268Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.suriname][3] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,275Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.suriname][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,305Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [filebeat-7.13.4-2021.08.12-000001][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,311Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.prd.crawl.data][4] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,310Z", "level": "ERROR", "component": "o.e.x.m.c.e.EnrichStatsCollector", "cluster.name": "security", "node.name": "security-master-0", "message": "collector [enrich_coordinator_stats] failed to collect data", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q" , 
"stacktrace": ["org.elasticsearch.action.FailedNodeException: Failed node [jBtim_WeTDuMfgxEtgCrzQ]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.onFailure(TransportNodesAction.java:254) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.access$100(TransportNodesAction.java:186) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1.handleException(TransportNodesAction.java:236) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:682) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction.start(TransportNodesAction.java:222) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction.doExecute(TransportNodesAction.java:100) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.nodes.TransportNodesAction.doExecute(TransportNodesAction.java:41) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.ActionFilter$Simple.apply(ActionFilter.java:42) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:171) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:160) ~[?:?]",
"at org.elasticsearch.action.ActionListener$DelegatingFailureActionListener.onResponse(ActionListener.java:217) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:281) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:688) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:663) ~[?:?]",
"at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:154) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:283) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:266) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:230) ~[?:?]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:127) ~[?:?]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:220) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:133) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:121) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:232) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.authorizeRequest(SecurityActionFilter.java:178) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$4(SecurityActionFilter.java:159) ~[?:?]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$0(AuthenticationService.java:336) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$8(AuthenticationService.java:414) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:425) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:333) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:274) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:167) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:154) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:106) ~[?:?]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:171) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.enrich.action.TransportEnrichStatsAction.masterOperation(TransportEnrichStatsAction.java:112) ~[?:?]",
"at org.elasticsearch.xpack.enrich.action.TransportEnrichStatsAction.masterOperation(TransportEnrichStatsAction.java:36) ~[?:?]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction.masterOperation(TransportMasterNodeAction.java:88) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.lambda$doStart$3(TransportMasterNodeAction.java:155) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.doStart(TransportMasterNodeAction.java:155) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction.doExecute(TransportMasterNodeAction.java:104) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.master.TransportMasterNodeAction.doExecute(TransportMasterNodeAction.java:46) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.ActionFilter$Simple.apply(ActionFilter.java:42) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:171) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:160) ~[?:?]",
"at org.elasticsearch.action.ActionListener$DelegatingFailureActionListener.onResponse(ActionListener.java:217) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:281) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:688) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:663) ~[?:?]",
"at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:154) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:283) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:266) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:230) ~[?:?]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:127) ~[?:?]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:220) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:133) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:121) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:232) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.authorizeRequest(SecurityActionFilter.java:178) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$4(SecurityActionFilter.java:159) ~[?:?]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$0(AuthenticationService.java:336) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$8(AuthenticationService.java:414) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:425) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:333) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:274) ~[?:?]",
"at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:167) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:154) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$apply$1(SecurityActionFilter.java:102) ~[?:?]",
"at org.elasticsearch.xpack.core.security.SecurityContext.executeAsUser(SecurityContext.java:139) ~[?:?]",
"at org.elasticsearch.xpack.security.authz.AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(AuthorizationUtils.java:133) ~[?:?]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:101) ~[?:?]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:171) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:390) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.monitoring.collector.enrich.EnrichStatsCollector.doCollect(EnrichStatsCollector.java:61) ~[x-pack-monitoring-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.monitoring.collector.Collector.collect(Collector.java:85) [x-pack-monitoring-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.monitoring.MonitoringService$MonitoringExecution$1.doRun(MonitoringService.java:252) [x-pack-monitoring-7.13.3.jar:7.13.3]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.3.jar:7.13.3]",
"at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]",
"at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:673) [elasticsearch-7.13.3.jar:7.13.3]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: org.elasticsearch.transport.NodeNotConnectedException: [security-master-1][10.244.1.12:9300] Node not connected",
"at org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:178) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:780) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:679) ~[elasticsearch-7.13.3.jar:7.13.3]",
"... 103 more"] }
{"type": "server", "timestamp": "2021-10-28T15:18:03,319Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.prd.crawl.data][1] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,326Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.prd.crawl.data][2] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,366Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.prd.crawl.data][3] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,374Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.prd.crawl.data][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,384Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [.kibana-event-log-7.13.3-000002][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,393Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.colombia][1] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,410Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.colombia][2] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,418Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.colombia][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,431Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [backup.ta.config.prd.user][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,472Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [filebeat-7.13.4-2021.09.11-000002][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,478Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.test_storscan_delta][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,485Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.trinidad][4] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,506Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.trinidad][1] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,517Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.trinidad][2] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,525Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.trinidad][3] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,533Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.data.prd.trinidad][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,575Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [.ds-ilm-history-5-2021.09.11-000002][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,585Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [.ds-ilm-history-5-2021.08.12-000001][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,597Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.test.prd.crawl.data][4] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,616Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.test.prd.crawl.data][1] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,627Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.test.prd.crawl.data][2] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,635Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.test.prd.crawl.data][3] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,649Z", "level": "INFO", "component": "o.e.c.r.DelayedAllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "scheduling reroute for delayed shards in [58.3s] (72 delayed shards)", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,674Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [ta.config.test.prd.crawl.data][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,687Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-0", "message": " [filebeat-7.13.4-2021.10.11-000003][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,700Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-es-7-2021.10.28][0] marking unavailable shards as stale: [HK5QprcVRRaXcVwtcYBxdA]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,937Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.transform-notifications-000002][0] marking unavailable shards as stale: [vrvjc-B0S_6qjY5jnMFNgQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:02,449Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.kibana_security_session_1][0] marking unavailable shards as stale: [HJ708n_hTNe1YVO-W7CDqA]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:02,509Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.security-7][0] marking unavailable shards as stale: [JJ-arL0JS8-sU88lexMQZA]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:02,914Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[backup.ta.config.prd.user][0] marking unavailable shards as stale: [q1rsoRulQe-vRYu-0NHCkw]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:02,914Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.kibana_task_manager_7.13.3_001][0] marking unavailable shards as stale: [8svSEOmQSw-LrM8fqg8EAQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:03,378Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.kibana_7.13.3_001][0] marking unavailable shards as stale: [O0_fF9YYQVGU0rqApZBL-w]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:03,479Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.transform-internal-007][0] marking unavailable shards as stale: [r6CX1yZBTtmTWbgW8REPHQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:03,918Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.apm-agent-configuration][0] marking unavailable shards as stale: [753A_pzWQWeSO1WBp0K0FA]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:04,133Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.async-search][0] marking unavailable shards as stale: [VwbdqVZqSPquAsG-2d_asA]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:04,524Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.apm-custom-link][0] marking unavailable shards as stale: [AOvd8HR3TSWGTsIeHRoGJQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:26,481Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-kibana-7-2021.10.27][0] marking unavailable shards as stale: [d9JIZOuCT3KSYCqoNdjgSg]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:27,337Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-kibana-7-2021.10.26][0] marking unavailable shards as stale: [yBKVdcHATAq9mhK8dELE-Q]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:30,697Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-es-7-2021.10.27][0] marking unavailable shards as stale: [5Xrm3FgiRgmMa6FqTIKmMQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:31,677Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-kibana-7-2021.10.25][0] marking unavailable shards as stale: [SKpHB58IRv-Nh2lnyIqWcA]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:55,724Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-es-7-2021.10.26][0] marking unavailable shards as stale: [BmHEqpd2RLqdP2kz68kooQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:56,470Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-es-7-2021.10.25][0] marking unavailable shards as stale: [239kYZVCT56qXIySihLpMQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:19:57,572Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-kibana-7-2021.10.24][0] marking unavailable shards as stale: [19YFUNKcSvyq0My-82uuow]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:20:28,086Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-es-7-2021.10.24][0] marking unavailable shards as stale: [CUiy4OyuTNWu1RORILP2FQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:20:28,086Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-es-7-2021.10.23][0] marking unavailable shards as stale: [lLSFzFNgRyaC4ms4HoWljw]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:20:28,986Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-kibana-7-2021.10.23][0] marking unavailable shards as stale: [r8irgiYmTjKKarkoXXLlig]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:20:29,068Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[.monitoring-kibana-7-2021.10.22][0] marking unavailable shards as stale: [nFGVq4WuS8SnPiYn7UGmgA]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:20:29,684Z", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "security", "node.name": "security-master-0", "message": "[ta.data.prd.test_storscan_delta_image][0] marking unavailable shards as stale: [NykWfY50TkmVX3PZ2-WUmQ]", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:20:30,091Z", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "security", "node.name": "security-master-0", "message": "node-join[{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{tQ1qcN1URp-NShjlXaW76Q}{10.244.1.15}{10.244.1.15:9300}{cdfhilmrstw} join existing leader], term: 9, version: 2573, delta: added {{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{tQ1qcN1URp-NShjlXaW76Q}{10.244.1.15}{10.244.1.15:9300}{cdfhilmrstw}}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }
{"type": "server", "timestamp": "2021-10-28T15:20:30,757Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-0", "message": "added {{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{tQ1qcN1URp-NShjlXaW76Q}{10.244.1.15}{10.244.1.15:9300}{cdfhilmrstw}}, term: 9, version: 2573, reason: Publication{term=9, version=2573}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "aCrYyxJTSAae4NIqlRaa5Q"  }

security-master-1:

{"type": "server", "timestamp": "2021-10-28T15:20:20,560Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-1", "message": "version[7.13.3], pid[6], build[default/docker/5d21bea28db1e89ecc1f66311ebdec9dc3aa7d64/2021-07-02T12:06:10.804015202Z], OS[Linux/5.4.0-1055-azure/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/16/16+36]" }
{"type": "server", "timestamp": "2021-10-28T15:20:20,562Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-1", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2021-10-28T15:20:20,563Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-1", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-5814191606372523925, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xmx7g, -Xms7g, -XX:MaxDirectMemorySize=3758096384, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,389Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-1", "message": "SLF4J: Failed to load class \"org.slf4j.impl.StaticLoggerBinder\"." }
{"type": "server", "timestamp": "2021-10-28T15:20:22,390Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-1", "message": "SLF4J: Defaulting to no-operation (NOP) logger implementation" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,390Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-1", "message": "SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details." }
{"type": "server", "timestamp": "2021-10-28T15:20:22,539Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,540Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,540Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,540Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,540Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,540Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,540Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,541Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,541Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,541Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,541Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,541Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,541Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,541Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,542Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,542Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,542Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,542Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,542Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,542Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,543Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,543Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,543Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,543Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,543Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,543Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,544Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,545Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,545Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,545Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,545Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,545Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,545Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,545Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,546Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,546Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,546Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,546Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,546Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,546Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,546Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,547Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,547Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,547Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,547Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,547Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,547Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,547Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,548Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,548Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,548Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,548Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-1", "message": "loaded plugin [repository-azure]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:20:22,552Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-1", "message": "[node.ml] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:20:22,567Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-1", "message": "[node.data] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-10-28T15:20:22,580Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "security", "node.name": "security-master-1", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sdc)]], net usable_space [268.5gb], net total_space [284.4gb], types [ext4]" }
{"type": "server", "timestamp": "2021-10-28T15:20:22,580Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "security", "node.name": "security-master-1", "message": "heap size [7gb], compressed ordinary object pointers [true]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:20:22,645Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-1", "message": "[node.master] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:20:22,649Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-1", "message": "[node.remote_cluster_client] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:20:22,650Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-1", "message": "[node.ingest] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-10-28T15:20:22,753Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-1", "message": "node name [security-master-1], node ID [jBtim_WeTDuMfgxEtgCrzQ], cluster name [security], roles [transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:20:22,754Z", "level": "DEPRECATION", "component": "o.e.d.n.Node", "cluster.name": "security", "node.name": "security-master-1", "message": "legacy role settings [node.data, node.remote_cluster_client, node.ingest, node.master, node.ml] are deprecated, use [node.roles=[transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]]" }
{"type": "server", "timestamp": "2021-10-28T15:20:26,010Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "security", "node.name": "security-master-1", "message": "[controller/359] [Main.cc@117] controller (64 bit): Version 7.13.3 (Build d74de47cfda9c3) Copyright (c) 2021 Elasticsearch BV" }
{"type": "server", "timestamp": "2021-10-28T15:20:26,311Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "security", "node.name": "security-master-1", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2021-10-28T15:20:26,774Z", "level": "INFO", "component": "o.e.i.g.LocalDatabases", "cluster.name": "security", "node.name": "security-master-1", "message": "initialized default databases [[GeoLite2-Country.mmdb, GeoLite2-City.mmdb, GeoLite2-ASN.mmdb]], config databases [[]] and watching [/usr/share/elasticsearch/config/ingest-geoip] for changes" }
{"type": "server", "timestamp": "2021-10-28T15:20:26,775Z", "level": "INFO", "component": "o.e.i.g.DatabaseRegistry", "cluster.name": "security", "node.name": "security-master-1", "message": "initialized database registry, using geoip-databases directory [/tmp/elasticsearch-5814191606372523925/geoip-databases/jBtim_WeTDuMfgxEtgCrzQ]" }
{"type": "server", "timestamp": "2021-10-28T15:20:27,301Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "security", "node.name": "security-master-1", "message": "creating NettyAllocator with the following configs: [name=elasticsearch_configured, chunk_size=1mb, suggested_max_allocation_size=1mb, factors={es.unsafe.use_netty_default_chunk_and_page_size=false, g1gc_enabled=true, g1gc_region_size=4mb}]" }
{"type": "server", "timestamp": "2021-10-28T15:20:27,353Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "security", "node.name": "security-master-1", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2021-10-28T15:20:27,697Z", "level": "INFO", "component": "o.e.g.DanglingIndicesState", "cluster.name": "security", "node.name": "security-master-1", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
{"type": "server", "timestamp": "2021-10-28T15:20:28,029Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-1", "message": "initialized" }
{"type": "server", "timestamp": "2021-10-28T15:20:28,030Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-1", "message": "starting ..." }
{"type": "server", "timestamp": "2021-10-28T15:20:28,065Z", "level": "INFO", "component": "o.e.x.s.c.f.PersistentCache", "cluster.name": "security", "node.name": "security-master-1", "message": "persistent cache index loaded" }
{"type": "server", "timestamp": "2021-10-28T15:20:28,140Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "security", "node.name": "security-master-1", "message": "publish_address {10.244.1.15:9300}, bound_addresses {[::]:9300}" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:20:28,607Z", "level": "DEPRECATION", "component": "o.e.d.c.j.Joda", "cluster.name": "security", "node.name": "security-master-1", "message": "'y' year should be replaced with 'u'. Use 'y' for year-of-era. Use new java.time date format specifiers." }
{"type": "server", "timestamp": "2021-10-28T15:20:29,256Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "security", "node.name": "security-master-1", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2021-10-28T15:20:29,258Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "security", "node.name": "security-master-1", "message": "cluster UUID [B5tIJoMcQ7ee3ybA-Hg5UA]" }
{"type": "server", "timestamp": "2021-10-28T15:20:30,297Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-1", "message": "master node changed {previous [], current [{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}]}, added {{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}, {security-master-2}{FfFqbAGDR0aYafc8UNVnMg}{yB7tniVIQsaMQ7N6iMBAIw}{10.244.2.22}{10.244.2.22:9300}{cdfhilmrstw}}, term: 9, version: 2573, reason: ApplyCommitRequest{term=9, version=2573, sourceNode={security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}}" }
{"type": "server", "timestamp": "2021-10-28T15:20:30,307Z", "level": "INFO", "component": "o.e.c.s.ClusterSettings", "cluster.name": "security", "node.name": "security-master-1", "message": "updating [xpack.monitoring.collection.enabled] from [false] to [true]" }
{"type": "server", "timestamp": "2021-10-28T15:20:30,534Z", "level": "INFO", "component": "o.e.x.s.a.TokenService", "cluster.name": "security", "node.name": "security-master-1", "message": "refresh keys" }
{"type": "server", "timestamp": "2021-10-28T15:20:30,710Z", "level": "INFO", "component": "o.e.x.s.a.TokenService", "cluster.name": "security", "node.name": "security-master-1", "message": "refreshed keys" }
{"type": "server", "timestamp": "2021-10-28T15:20:30,751Z", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "security", "node.name": "security-master-1", "message": "license [949b4c5e-ffdb-418a-8774-e67b0b210a1a] mode [basic] - valid" }
{"type": "server", "timestamp": "2021-10-28T15:20:30,752Z", "level": "INFO", "component": "o.e.x.s.s.SecurityStatusChangeListener", "cluster.name": "security", "node.name": "security-master-1", "message": "Active license is now [BASIC]; Security is enabled" }
{"type": "server", "timestamp": "2021-10-28T15:20:30,764Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "security", "node.name": "security-master-1", "message": "publish_address {10.244.1.15:9200}, bound_addresses {[::]:9200}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "jBtim_WeTDuMfgxEtgCrzQ"  }
{"type": "server", "timestamp": "2021-10-28T15:20:30,765Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-1", "message": "started", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "jBtim_WeTDuMfgxEtgCrzQ"  }

security-master-2:

{"type": "server", "timestamp": "2021-10-28T15:09:47,452Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-2", "message": "version[7.13.3], pid[7], build[default/docker/5d21bea28db1e89ecc1f66311ebdec9dc3aa7d64/2021-07-02T12:06:10.804015202Z], OS[Linux/5.4.0-1055-azure/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/16/16+36]" }
{"type": "server", "timestamp": "2021-10-28T15:09:47,454Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-2", "message": "JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]" }
{"type": "server", "timestamp": "2021-10-28T15:09:47,454Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-2", "message": "JVM arguments [-Xshare:auto, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -XX:+ShowCodeDetailsInExceptionMessages, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=ALL-UNNAMED, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-16765674889878093968, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xmx7g, -Xms7g, -XX:MaxDirectMemorySize=3758096384, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,340Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-2", "message": "SLF4J: Failed to load class \"org.slf4j.impl.StaticLoggerBinder\"." }
{"type": "server", "timestamp": "2021-10-28T15:09:49,342Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-2", "message": "SLF4J: Defaulting to no-operation (NOP) logger implementation" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,342Z", "level": "WARN", "component": "stderr", "cluster.name": "security", "node.name": "security-master-2", "message": "SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details." }
{"type": "server", "timestamp": "2021-10-28T15:09:49,494Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [aggs-matrix-stats]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,495Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [analysis-common]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,495Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [constant-keyword]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,495Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [frozen-indices]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,495Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [ingest-common]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,495Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [ingest-geoip]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,496Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [ingest-user-agent]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,496Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [kibana]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,496Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [lang-expression]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,496Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [lang-mustache]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,496Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [lang-painless]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,496Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [mapper-extras]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,497Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [mapper-version]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,497Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [parent-join]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,497Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,497Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,497Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,497Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [repositories-metering-api]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,497Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [repository-encrypted]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [repository-url]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [runtime-fields-common]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [search-business-rules]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [searchable-snapshots]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [snapshot-repo-test-kit]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,498Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,499Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [transport-netty4]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,499Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [unsigned-long]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,499Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,499Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [wildcard]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,499Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-aggregate-metric]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,499Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-analytics]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,499Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-async]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,499Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-async-search]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,500Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-autoscaling]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,500Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,500Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-core]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,500Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-data-streams]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,500Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-deprecation]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,500Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-enrich]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,500Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-eql]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,500Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-fleet]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-graph]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-identity-provider]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-logstash]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-monitoring]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,501Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-ql]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,502Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-rollup]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,502Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-security]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,502Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-shutdown]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,502Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,502Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-stack]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,502Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-text-structure]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,503Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,503Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded module [x-pack-watcher]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,503Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "security", "node.name": "security-master-2", "message": "loaded plugin [repository-azure]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:09:49,507Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-2", "message": "[node.ml] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:09:49,524Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-2", "message": "[node.data] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-10-28T15:09:49,538Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "security", "node.name": "security-master-2", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sdc)]], net usable_space [284.3gb], net total_space [284.4gb], types [ext4]" }
{"type": "server", "timestamp": "2021-10-28T15:09:49,538Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "security", "node.name": "security-master-2", "message": "heap size [7gb], compressed ordinary object pointers [true]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:09:49,541Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-2", "message": "[node.master] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:09:49,544Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-2", "message": "[node.remote_cluster_client] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:09:49,546Z", "level": "DEPRECATION", "component": "o.e.d.c.s.Settings", "cluster.name": "security", "node.name": "security-master-2", "message": "[node.ingest] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." }
{"type": "server", "timestamp": "2021-10-28T15:09:49,595Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-2", "message": "node name [security-master-2], node ID [FfFqbAGDR0aYafc8UNVnMg], cluster name [security], roles [transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:09:49,596Z", "level": "DEPRECATION", "component": "o.e.d.n.Node", "cluster.name": "security", "node.name": "security-master-2", "message": "legacy role settings [node.data, node.remote_cluster_client, node.ingest, node.master, node.ml] are deprecated, use [node.roles=[transform, data_frozen, master, remote_cluster_client, data, ml, data_content, data_hot, data_warm, data_cold, ingest]]" }
{"type": "server", "timestamp": "2021-10-28T15:09:53,113Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "security", "node.name": "security-master-2", "message": "[controller/360] [Main.cc@117] controller (64 bit): Version 7.13.3 (Build d74de47cfda9c3) Copyright (c) 2021 Elasticsearch BV" }
{"type": "server", "timestamp": "2021-10-28T15:09:53,409Z", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "security", "node.name": "security-master-2", "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2021-10-28T15:09:53,867Z", "level": "INFO", "component": "o.e.i.g.LocalDatabases", "cluster.name": "security", "node.name": "security-master-2", "message": "initialized default databases [[GeoLite2-Country.mmdb, GeoLite2-City.mmdb, GeoLite2-ASN.mmdb]], config databases [[]] and watching [/usr/share/elasticsearch/config/ingest-geoip] for changes" }
{"type": "server", "timestamp": "2021-10-28T15:09:53,868Z", "level": "INFO", "component": "o.e.i.g.DatabaseRegistry", "cluster.name": "security", "node.name": "security-master-2", "message": "initialized database registry, using geoip-databases directory [/tmp/elasticsearch-16765674889878093968/geoip-databases/FfFqbAGDR0aYafc8UNVnMg]" }
{"type": "server", "timestamp": "2021-10-28T15:09:54,421Z", "level": "INFO", "component": "o.e.t.NettyAllocator", "cluster.name": "security", "node.name": "security-master-2", "message": "creating NettyAllocator with the following configs: [name=elasticsearch_configured, chunk_size=1mb, suggested_max_allocation_size=1mb, factors={es.unsafe.use_netty_default_chunk_and_page_size=false, g1gc_enabled=true, g1gc_region_size=4mb}]" }
{"type": "server", "timestamp": "2021-10-28T15:09:54,476Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "security", "node.name": "security-master-2", "message": "using discovery type [zen] and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2021-10-28T15:09:54,868Z", "level": "INFO", "component": "o.e.g.DanglingIndicesState", "cluster.name": "security", "node.name": "security-master-2", "message": "gateway.auto_import_dangling_indices is disabled, dangling indices will not be automatically detected or imported and must be managed manually" }
{"type": "server", "timestamp": "2021-10-28T15:09:55,216Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-2", "message": "initialized" }
{"type": "server", "timestamp": "2021-10-28T15:09:55,216Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-2", "message": "starting ..." }
{"type": "server", "timestamp": "2021-10-28T15:09:55,236Z", "level": "INFO", "component": "o.e.x.s.c.f.PersistentCache", "cluster.name": "security", "node.name": "security-master-2", "message": "persistent cache index loaded" }
{"type": "server", "timestamp": "2021-10-28T15:09:55,308Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "security", "node.name": "security-master-2", "message": "publish_address {10.244.2.22:9300}, bound_addresses {[::]:9300}" }
{"type": "deprecation.elasticsearch", "timestamp": "2021-10-28T15:09:55,931Z", "level": "DEPRECATION", "component": "o.e.d.c.j.Joda", "cluster.name": "security", "node.name": "security-master-2", "message": "'y' year should be replaced with 'u'. Use 'y' for year-of-era. Use new java.time date format specifiers." }
{"type": "server", "timestamp": "2021-10-28T15:09:56,551Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "security", "node.name": "security-master-2", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2021-10-28T15:09:56,554Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "security", "node.name": "security-master-2", "message": "cluster UUID [B5tIJoMcQ7ee3ybA-Hg5UA]" }
{"type": "server", "timestamp": "2021-10-28T15:09:57,951Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-2", "message": "master node changed {previous [], current [{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}]}, added {{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{JDxIucJySYqjFLSCkLeYvw}{10.244.0.18}{10.244.0.18:9300}{cdfhilmrstw}, {security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}}, term: 8, version: 2337, reason: ApplyCommitRequest{term=8, version=2337, sourceNode={security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}}" }
{"type": "server", "timestamp": "2021-10-28T15:09:57,982Z", "level": "INFO", "component": "o.e.c.s.ClusterSettings", "cluster.name": "security", "node.name": "security-master-2", "message": "updating [xpack.monitoring.collection.enabled] from [false] to [true]" }
{"type": "server", "timestamp": "2021-10-28T15:09:58,164Z", "level": "INFO", "component": "o.e.x.s.a.TokenService", "cluster.name": "security", "node.name": "security-master-2", "message": "refresh keys" }
{"type": "server", "timestamp": "2021-10-28T15:09:58,331Z", "level": "INFO", "component": "o.e.x.s.a.TokenService", "cluster.name": "security", "node.name": "security-master-2", "message": "refreshed keys" }
{"type": "server", "timestamp": "2021-10-28T15:09:58,377Z", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "security", "node.name": "security-master-2", "message": "license [949b4c5e-ffdb-418a-8774-e67b0b210a1a] mode [basic] - valid" }
{"type": "server", "timestamp": "2021-10-28T15:09:58,378Z", "level": "INFO", "component": "o.e.x.s.s.SecurityStatusChangeListener", "cluster.name": "security", "node.name": "security-master-2", "message": "Active license is now [BASIC]; Security is enabled" }
{"type": "server", "timestamp": "2021-10-28T15:09:58,388Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "security", "node.name": "security-master-2", "message": "publish_address {10.244.2.22:9200}, bound_addresses {[::]:9200}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:09:58,388Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "security", "node.name": "security-master-2", "message": "started", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:12:49,093Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-2", "message": "removed {{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{JDxIucJySYqjFLSCkLeYvw}{10.244.0.18}{10.244.0.18:9300}{cdfhilmrstw}}, term: 8, version: 2340, reason: ApplyCommitRequest{term=8, version=2340, sourceNode={security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:15:02,619Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-2", "message": "added {{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}}, term: 8, version: 2421, reason: ApplyCommitRequest{term=8, version=2421, sourceNode={security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:01,745Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "security", "node.name": "security-master-2", "message": "master node [{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}] failed, restarting discovery", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg" , 
"stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [security-master-1][10.244.1.12:9300][disconnected] disconnected"] }
{"type": "server", "timestamp": "2021-10-28T15:18:01,749Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-2", "message": "master node changed {previous [{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}], current []}, term: 8, version: 2507, reason: becoming candidate: onLeaderFailure", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:01,752Z", "level": "WARN", "component": "o.e.c.NodeConnectionsService", "cluster.name": "security", "node.name": "security-master-2", "message": "failed to connect to {security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true} (tried [1] times)", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg" , 
"stacktrace": ["org.elasticsearch.transport.ConnectTransportException: [security-master-1][10.244.1.12:9300] connect_exception",
"at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:971) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$0(ActionListener.java:277) ~[elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:31) ~[elasticsearch-core-7.13.3.jar:7.13.3]",
"at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
"at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
"at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
"at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2158) ~[?:?]",
"at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:46) ~[elasticsearch-core-7.13.3.jar:7.13.3]",
"at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:57) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]",
"at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:702) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[?:?]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10.244.1.12/10.244.1.12:9300",
"Caused by: java.net.ConnectException: Connection refused",
"at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
"at sun.nio.ch.Net.pollConnectNow(Net.java:669) ~[?:?]",
"at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:944) ~[?:?]",
"at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]",
"at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]",
"... 7 more"] }
{"type": "server", "timestamp": "2021-10-28T15:18:01,973Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-2", "message": "master node changed {previous [], current [{security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}]}, term: 9, version: 2508, reason: ApplyCommitRequest{term=9, version=2508, sourceNode={security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,244Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-2", "message": "removed {{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{OjwGmq6VR2u1rQDqw3wQgw}{10.244.1.12}{10.244.1.12:9300}{cdfhilmrstw}}, term: 9, version: 2509, reason: ApplyCommitRequest{term=9, version=2509, sourceNode={security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,299Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.apm-agent-configuration][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,307Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [config.prd.user][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,320Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.kibana_task_manager_7.13.3_001][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,333Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [ta.data.prd.test_delta][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,371Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-es-7-2021.10.23][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,379Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.apm-custom-link][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,406Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.transform-internal-007][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,415Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.async-search][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,423Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [ta.data.prd.test_storscan_delta_image][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,467Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-es-7-2021.10.25][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,482Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-es-7-2021.10.27][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,513Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-es-7-2021.10.24][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,529Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-kibana-7-2021.10.26][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,573Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-kibana-7-2021.10.24][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,584Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-kibana-7-2021.10.27][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,615Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.kibana-event-log-7.13.3-000003][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,626Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-es-7-2021.10.28][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,627Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-es-7-2021.10.26][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,671Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [ta.data.prd.argentina][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,676Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-kibana-7-2021.10.23][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,687Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.kibana_security_session_1][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,719Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.ds-ilm-history-5-2021.10.11-000003][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,728Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-kibana-7-2021.10.22][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,835Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.kibana_7.13.3_001][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,842Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-kibana-7-2021.10.25][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,853Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.security-7][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,875Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [filebeat-7.14.1-2021.10.01-000002][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:02,884Z", "level": "INFO", "component": "o.e.i.s.IndexShard", "cluster.name": "security", "node.name": "security-master-2", "message": " [.monitoring-es-7-2021.10.22][0] primary-replica resync completed with 0 operations", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
{"type": "server", "timestamp": "2021-10-28T15:18:03,918Z", "level": "ERROR", "component": "o.e.x.t.t.TransformPersistentTasksExecutor", "cluster.name": "security", "node.name": "security-master-2", "message": "Failed to start task [endpoint.metadata_current-default-0.19.1] in node operation", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg" , 
"stacktrace": ["org.elasticsearch.ElasticsearchStatusException: Unable to start transform [endpoint.metadata_current-default-0.19.1] as it is in a failed state with failure: [Failed to gather field mappings for index [metrics-endpoint.metadata_current_default]]. Use force stop and then restart the transform once error is resolved.",
"at org.elasticsearch.xpack.transform.transforms.TransformTask.start(TransformTask.java:218) [transform-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.transform.transforms.TransformPersistentTasksExecutor.startTask(TransformPersistentTasksExecutor.java:376) [transform-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.transform.transforms.TransformPersistentTasksExecutor.lambda$nodeOperation$4(TransformPersistentTasksExecutor.java:215) [transform-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.transform.persistence.IndexBasedTransformConfigManager.lambda$getTransformCheckpoint$6(IndexBasedTransformConfigManager.java:320) [transform-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:83) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:77) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$2(SecurityActionFilter.java:163) [x-pack-security-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.ActionListener$DelegatingFailureActionListener.onResponse(ActionListener.java:217) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.ActionListener$RunAfterActionListener.onResponse(ActionListener.java:339) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.sendSearchResponse(AbstractSearchAsyncAction.java:655) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.ExpandSearchPhase.run(ExpandSearchPhase.java:109) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:424) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:418) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.FetchSearchPhase.moveToNextPhase(FetchSearchPhase.java:219) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.FetchSearchPhase.lambda$innerRun$1(FetchSearchPhase.java:101) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.FetchSearchPhase.innerRun(FetchSearchPhase.java:107) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.FetchSearchPhase.access$000(FetchSearchPhase.java:36) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.action.search.FetchSearchPhase$1.doRun(FetchSearchPhase.java:84) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.13.3.jar:7.13.3]",
"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.13.3.jar:7.13.3]",
"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",
"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",
"at java.lang.Thread.run(Thread.java:831) [?:?]"] }
{"type": "server", "timestamp": "2021-10-28T15:20:30,133Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "security", "node.name": "security-master-2", "message": "added {{security-master-1}{jBtim_WeTDuMfgxEtgCrzQ}{tQ1qcN1URp-NShjlXaW76Q}{10.244.1.15}{10.244.1.15:9300}{cdfhilmrstw}}, term: 9, version: 2573, reason: ApplyCommitRequest{term=9, version=2573, sourceNode={security-master-0}{aCrYyxJTSAae4NIqlRaa5Q}{HCdHKzisTeGfFj_kDqEihA}{10.244.0.21}{10.244.0.21:9300}{cdfhilmrstw}{ml.machine_memory=16106127360, ml.max_open_jobs=512, xpack.installed=true, ml.max_jvm_size=7516192768, transform.node=true}}", "cluster.uuid": "B5tIJoMcQ7ee3ybA-Hg5UA", "node.id": "FfFqbAGDR0aYafc8UNVnMg"  }
framsouza commented 2 years ago

hello, the fix for that is here #1519, please stay tuned for more updates on this PR, we are almost there to merge it.

botelastic[bot] commented 2 years ago

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

botelastic[bot] commented 2 years ago

This issue has been automatically closed because it has not had recent activity since being marked as stale.