search

infinispan / infinispan-operator

Infinispan Operator
https://infinispan.org/docs/infinispan-operator/main/operator.html
Apache License 2.0
51 stars 54 forks source link

Cross Site GossipRouter TLS: java.io.EOFException: SSL peer shut down incorrectly #1602

Closed vladimir22 closed 2 years ago

vladimir22 commented 2 years ago

Hi colleagues, I am using infinispan-operator v.2.2.x and trying to set up TLS for x-site GossipRouter following your documentation , and getting an error java.io.EOFException: SSL peer shut down incorrectly inside the mycluster-router POD

Could you explain what is wrong with my configuration (probably with my keystore inside the x-site secret) ?

Configuration details below:

mycluster(env-vpronkin) site

kubectl get secret -n infinispan mycluster-tls-secret -o jsonpath='{.data.keystore\.p12}' | base64 --decode > keystore.p12
keytool -list -v -storetype pkcs12 -keystore keystore.p12 -storepass password

Keystore type: PKCS12
Keystore provider: SUN

Your keystore contains 1 entry

Alias name: mycluster
Creation date: Jul 5, 2022
Entry type: PrivateKeyEntry
Certificate chain length: 2
Certificate[1]:
Owner:
Issuer: CN=Base - cert-manager CA, O=Digital, ST=Some-State, C=AU
Serial number: 5302b77eeaedc5506fc71cca43a00ece
Valid from: Tue Jul 05 16:49:59 EEST 2022 until: Mon Oct 03 16:49:59 EEST 2022
Certificate fingerprints:
         SHA1: 1B:67:15:D6:E8:FE:A4:B1:6F:17:1B:93:B8:23:D9:94:08:FF:AA:9B
         SHA256: D1:9D:40:0F:54:CC:64:33:55:54:CD:B5:70:6C:90:BC:DA:F0:4D:07:BD:4F:7B:BE:96:67:4C:54:A4:D3:78:07
Signature algorithm name: SHA256withRSA
Subject Public Key Algorithm: 2048-bit RSA key
Version: 3

Extensions:

#1: ObjectId: 2.5.29.19 Criticality=true
BasicConstraints:[
  CA:false
  PathLen: undefined
]

#2: ObjectId: 2.5.29.15 Criticality=true
KeyUsage [
  DigitalSignature
  Key_Encipherment
]

#3: ObjectId: 2.5.29.17 Criticality=true
SubjectAlternativeName [
  DNSName: mycluster.infinispan.svc
  DNSName: mycluster.infinispan.svc.cluster.local
]

Certificate[2]:
Owner: CN=Base - cert-manager CA, O=Digital, ST=Some-State, C=AU
Issuer: CN=Base - Root CA, O=Digital, ST=Some-State, C=AU
Serial number: 3eac62fdbe71c450323654458d6fce4dec1fc64f
Valid from: Wed Apr 21 09:54:00 EEST 2021 until: Sat Apr 19 09:54:00 EEST 2031
Certificate fingerprints:
         SHA1: C7:95:CF:58:72:4D:AB:3A:E4:12:F6:3C:21:5B:2F:DE:C5:CB:FF:23
         SHA256: 96:3F:EA:78:E8:C5:86:8E:31:7B:54:6A:7A:52:0C:F3:33:13:73:24:D1:18:99:F8:15:24:70:69:B4:ED:EF:2E
Signature algorithm name: SHA256withRSA
Subject Public Key Algorithm: 2048-bit RSA key
Version: 3

Extensions:

#1: ObjectId: 2.5.29.35 Criticality=false
AuthorityKeyIdentifier [
KeyIdentifier [
0000: 06 8D A0 3C 83 48 B1 45   6C DC 5A 04 9E CF 0C A1  ...<.H.El.Z.....
0010: 56 D2 B4 34                                        V..4
]
]

#2: ObjectId: 2.5.29.19 Criticality=false
BasicConstraints:[
  CA:true
  PathLen:2147483647
]

#3: ObjectId: 2.5.29.15 Criticality=false
KeyUsage [
  Key_CertSign
  Crl_Sign
]

*******************************************
*******************************************

kubectl get infinispan mycluster -n infinispan -o yaml

apiVersion: infinispan.org/v1
kind: Infinispan
metadata:
  annotations:
    infinispan.org/monitoring: "true"
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"infinispan.org/v1","kind":"Infinispan","metadata":{"annotations":{},"name":"mycluster","namespace":"infinispan"},"spec":{"configListener":{"enabled":false},"container":{"cpu":"1000m","extraJvmOpts":"-Dinfinispan.bind.address=0.0.0.0","memory":"1024Mi"},"logging":{"categories":{"org.infinispan":"debug","org.jgroups":"debug"}},"replicas":2,"security":{"endpointAuthentication":true,"endpointEncryption":{"certSecretName":"mycluster-tls-secret","clientCert":"None","type":"Secret"},"endpointSecretName":"infinispan-dg-identities"},"service":{"container":{"storage":"1024Mi","storageClassName":"local-path"},"replicationFactor":2,"sites":{"local":{"encryption":{"protocol":"TLSv1.3","routerKeyStore":{"alias":"mycluster","filename":"keystore.p12","secretName":"mycluster-tls-secret"},"transportKeyStore":{"alias":"mycluster","filename":"keystore.p12","secretName":"mycluster-tls-secret"},"trustStore":{"filename":"keystore.p12","secretName":"mycluster-tls-secret"}},"expose":{"annotations":{"traffic.sidecar.istio.io/excludeInboundPorts":"8888,11223,7900","traffic.sidecar.istio.io/excludeOutboundPorts":"8888,11223,7900"},"nodePort":31314,"type":"NodePort"},"name":"mycluster"},"locations":[{"name":"mycluster","secretName":"infinispan-operator-token-v28c6","url":"openshift://env-vpronkin-rntoweou-cp-01c000a0720319a3.elb.us-east-1.amazonaws.com:443"},{"name":"mycluster-site","secretName":"env-vpronkin-site-xsite-token","url":"openshift://env-vpronkin-site-rqkuvrrv-cp-00350df1736ab712.elb.us-east-1.amazonaws.com:443"}]},"type":"DataGrid"}}}
  creationTimestamp: "2022-07-05T14:01:12Z"
  generation: 2
  name: mycluster
  namespace: infinispan
  resourceVersion: "11949752"
  uid: 4152bb61-4967-45c8-b004-dd5107f5a7a4
spec:
  affinity:
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - podAffinityTerm:
          labelSelector:
            matchLabels:
              app: infinispan-pod
              clusterName: mycluster
              infinispan_cr: mycluster
          topologyKey: r.kubernetes.io/hostname
        weight: 100
  configListener:
    enabled: false
  container:
    cpu: 1000m
    extraJvmOpts: -Dinfinispan.bind.address=0.0.0.0
    memory: 1024Mi
  logging:
    categories:
      org.infinispan: debug
      org.jgroups: debug
  replicas: 2
  security:
    endpointAuthentication: true
    endpointEncryption:
      certSecretName: mycluster-tls-secret
      clientCert: None
      type: Secret
    endpointSecretName: infinispan-dg-identities
  service:
    container:
      storage: 1024Mi
      storageClassName: local-path
    replicationFactor: 2
    sites:
      local:
        encryption:
          protocol: TLSv1.3
          routerKeyStore:
            alias: mycluster
            filename: keystore.p12
            secretName: mycluster-tls-secret
          transportKeyStore:
            alias: mycluster
            filename: keystore.p12
            secretName: mycluster-tls-secret
          trustStore:
            filename: keystore.p12
            secretName: mycluster-tls-secret
        expose:
          annotations:
            traffic.sidecar.istio.io/excludeInboundPorts: 8888,11223,7900
            traffic.sidecar.istio.io/excludeOutboundPorts: 8888,11223,7900
          nodePort: 31314
          type: NodePort
        name: mycluster
      locations:
      - name: mycluster
        secretName: infinispan-operator-token-v28c6
        url: openshift://env-vpronkin-rntoweou-cp-01c000a0720319a3.elb.us-east-1.amazonaws.com:443
      - name: mycluster-site
        secretName: env-vpronkin-site-xsite-token
        url: openshift://env-vpronkin-site-rqkuvrrv-cp-00350df1736ab712.elb.us-east-1.amazonaws.com:443
    type: DataGrid
  upgrades:
    type: Shutdown
status:
  conditions:
  - status: "True"
    type: PreliminaryChecksPassed
  - status: "True"
    type: GossipRouterReady
  - message: 'View: mycluster-0-22504,mycluster-1-45233'
    status: "True"
    type: WellFormed
  - message: Site 'mycluster-site' not ready
    status: "False"
    type: CrossSiteViewFormed
  podStatus:
    ready:
    - mycluster-1
    - mycluster-2
  statefulSetName: mycluster

kubectl get pods -n infinispan

NAME                                                      READY   STATUS    RESTARTS   AGE
infinispan-operator-controller-manager-7764c699b6-rk2dg   1/1     Running   0          16m
mycluster-0                                               1/1     Running   0          33m
mycluster-1                                               1/1     Running   0          32m
mycluster-router-67b4bb9fc5-t2bzl                         1/1     Running   0          16m

kubectl logs -f -n infinispan mycluster-router-67b4bb9fc5-t2bzl

GossipRouter started in 2581 ms listening on 0.0.0.0:7900 (TLSv1.3)
Jul 05, 2022 3:54:50 PM org.jgroups.blocks.cs.TcpConnection$Receiver run
WARNING: failed handling message
javax.net.ssl.SSLHandshakeException: Remote host terminated the handshake
        at java.base/sun.security.ssl.SSLSocketImpl.handleEOF(SSLSocketImpl.java:1616)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1434)
        at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1336)
        at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:450)
        at java.base/sun.security.ssl.SSLSocketImpl.ensureNegotiated(SSLSocketImpl.java:841)
        at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:932)
        at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:252)
        at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:271)
        at java.base/java.io.DataInputStream.readInt(DataInputStream.java:392)
        at org.jgroups.blocks.cs.TcpConnection$Receiver.run(TcpConnection.java:305)
        at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.io.EOFException: SSL peer shut down incorrectly
        at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:483)
        at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
        at java.base/sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:160)
        at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:111)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1426)
        ... 9 more
...

added _mycluster-1-45233 (10.42.0.92:42228) to group xsite
Jul 05, 2022 3:54:58 PM org.jgroups.blocks.cs.TcpConnection$Receiver run
WARNING: failed handling message
javax.net.ssl.SSLHandshakeException: Remote host terminated the handshake
        at java.base/sun.security.ssl.SSLSocketImpl.handleEOF(SSLSocketImpl.java:1616)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1434)
        at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1336)
        at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:450)
        at java.base/sun.security.ssl.SSLSocketImpl.ensureNegotiated(SSLSocketImpl.java:841)
        at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:932)
        at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:252)
        at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:271)
        at java.base/java.io.DataInputStream.readInt(DataInputStream.java:392)
        at org.jgroups.blocks.cs.TcpConnection$Receiver.run(TcpConnection.java:305)
        at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.io.EOFException: SSL peer shut down incorrectly
        at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:483)
        at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
        at java.base/sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:160)
        at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:111)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1426)
        ... 9 more
...

mycluster-site(env-vpronkin-site) site

kubectl get secret -n infinispan mycluster-tls-secret -o jsonpath='{.data.keystore\.p12}' | base64 --decode > keystore.p12
keytool -list -v -storetype pkcs12 -keystore keystore.p12 -storepass password

Keystore type: PKCS12
Keystore provider: SUN

Your keystore contains 1 entry

Alias name: mycluster
Creation date: Jul 5, 2022
Entry type: PrivateKeyEntry
Certificate chain length: 2
Certificate[1]:
Owner:
Issuer: CN=Base - cert-manager CA, O=Digital, ST=Some-State, C=AU
Serial number: 90cecf4d358aec9ce77f876ada6aa5f7
Valid from: Tue Jul 05 16:49:53 EEST 2022 until: Mon Oct 03 16:49:53 EEST 2022
Certificate fingerprints:
         SHA1: A0:EF:70:C5:A9:F3:5C:61:2D:FD:E4:BD:01:6E:AB:8C:01:C1:1C:6E
         SHA256: 78:01:71:FA:07:9B:A7:89:9F:24:56:E1:8C:1B:9E:03:35:3C:37:6B:E2:50:AD:EF:8D:DB:F9:BA:41:37:49:92
Signature algorithm name: SHA256withRSA
Subject Public Key Algorithm: 2048-bit RSA key
Version: 3

Extensions:

#1: ObjectId: 2.5.29.19 Criticality=true
BasicConstraints:[
  CA:false
  PathLen: undefined
]

#2: ObjectId: 2.5.29.15 Criticality=true
KeyUsage [
  DigitalSignature
  Key_Encipherment
]

#3: ObjectId: 2.5.29.17 Criticality=true
SubjectAlternativeName [
  DNSName: mycluster.infinispan.svc
  DNSName: mycluster.infinispan.svc.cluster.local
]

Certificate[2]:
Owner: CN=Base - cert-manager CA, O=Digital, ST=Some-State, C=AU
Issuer: CN=Base - Root CA, O=Digital, ST=Some-State, C=AU
Serial number: 3eac62fdbe71c450323654458d6fce4dec1fc64f
Valid from: Wed Apr 21 09:54:00 EEST 2021 until: Sat Apr 19 09:54:00 EEST 2031
Certificate fingerprints:
         SHA1: C7:95:CF:58:72:4D:AB:3A:E4:12:F6:3C:21:5B:2F:DE:C5:CB:FF:23
         SHA256: 96:3F:EA:78:E8:C5:86:8E:31:7B:54:6A:7A:52:0C:F3:33:13:73:24:D1:18:99:F8:15:24:70:69:B4:ED:EF:2E
Signature algorithm name: SHA256withRSA
Subject Public Key Algorithm: 2048-bit RSA key
Version: 3

Extensions:

#1: ObjectId: 2.5.29.35 Criticality=false
AuthorityKeyIdentifier [
KeyIdentifier [
0000: 06 8D A0 3C 83 48 B1 45   6C DC 5A 04 9E CF 0C A1  ...<.H.El.Z.....
0010: 56 D2 B4 34                                        V..4
]
]

#2: ObjectId: 2.5.29.19 Criticality=false
BasicConstraints:[
  CA:true
  PathLen:2147483647
]

#3: ObjectId: 2.5.29.15 Criticality=false
KeyUsage [
  Key_CertSign
  Crl_Sign
]

*******************************************
*******************************************

kubectl get infinispan mycluster -n infinispan -o yaml

apiVersion: infinispan.org/v1
kind: Infinispan
metadata:
  annotations:
    infinispan.org/monitoring: "true"
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"infinispan.org/v1","kind":"Infinispan","metadata":{"annotations":{},"name":"mycluster","namespace":"infinispan"},"spec":{"configListener":{"enabled":false},"container":{"cpu":"1000m","extraJvmOpts":"-Dinfinispan.bind.address=0.0.0.0","memory":"1024Mi"},"logging":{"categories":{"org.infinispan":"debug","org.jgroups":"debug"}},"replicas":2,"security":{"endpointAuthentication":true,"endpointEncryption":{"certSecretName":"mycluster-tls-secret","clientCert":"None","type":"Secret"},"endpointSecretName":"infinispan-dg-identities"},"service":{"container":{"storage":"1024Mi","storageClassName":"local-path"},"replicationFactor":2,"sites":{"local":{"encryption":{"protocol":"TLSv1.3","routerKeyStore":{"alias":"mycluster","filename":"keystore.p12","secretName":"mycluster-tls-secret"},"transportKeyStore":{"alias":"mycluster","filename":"keystore.p12","secretName":"mycluster-tls-secret"},"trustStore":{"filename":"keystore.p12","secretName":"mycluster-tls-secret"}},"expose":{"annotations":{"traffic.sidecar.istio.io/excludeInboundPorts":"8888,11223,7900","traffic.sidecar.istio.io/excludeOutboundPorts":"8888,11223,7900"},"nodePort":31314,"type":"NodePort"},"name":"mycluster-site"},"locations":[{"name":"mycluster","secretName":"env-vpronkin-xsite-token","url":"openshift://env-vpronkin-rntoweou-cp-01c000a0720319a3.elb.us-east-1.amazonaws.com:443"},{"name":"mycluster-site","secretName":"infinispan-operator-token-56vc7","url":"openshift://env-vpronkin-site-rqkuvrrv-cp-00350df1736ab712.elb.us-east-1.amazonaws.com:443"}]},"type":"DataGrid"}}}
  creationTimestamp: "2022-07-05T14:01:26Z"
  generation: 2
  name: mycluster
  namespace: infinispan
  resourceVersion: "12001640"
  uid: c6ac7d5c-e464-4ab1-b319-4a4c4d6b64b3
spec:
  affinity:
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - podAffinityTerm:
          labelSelector:
            matchLabels:
              app: infinispan-pod
              clusterName: mycluster
              infinispan_cr: mycluster
          topologyKey: r.kubernetes.io/hostname
        weight: 100
  configListener:
    enabled: false
  container:
    cpu: 1000m
    extraJvmOpts: -Dinfinispan.bind.address=0.0.0.0
    memory: 1024Mi
  logging:
    categories:
      org.infinispan: debug
      org.jgroups: debug
  replicas: 2
  security:
    endpointAuthentication: true
    endpointEncryption:
      certSecretName: mycluster-tls-secret
      clientCert: None
      type: Secret
    endpointSecretName: infinispan-dg-identities
  service:
    container:
      storage: 1024Mi
      storageClassName: local-path
    replicationFactor: 2
    sites:
      local:
        encryption:
          protocol: TLSv1.3
          routerKeyStore:
            alias: mycluster
            filename: keystore.p12
            secretName: mycluster-tls-secret
          transportKeyStore:
            alias: mycluster
            filename: keystore.p12
            secretName: mycluster-tls-secret
          trustStore:
            filename: keystore.p12
            secretName: mycluster-tls-secret
        expose:
          annotations:
            traffic.sidecar.istio.io/excludeInboundPorts: 8888,11223,7900
            traffic.sidecar.istio.io/excludeOutboundPorts: 8888,11223,7900
          nodePort: 31314
          type: NodePort
        name: mycluster-site
      locations:
      - name: mycluster
        secretName: env-vpronkin-xsite-token
        url: openshift://env-vpronkin-rntoweou-cp-01c000a0720319a3.elb.us-east-1.amazonaws.com:443
      - name: mycluster-site
        secretName: infinispan-operator-token-56vc7
        url: openshift://env-vpronkin-site-rqkuvrrv-cp-00350df1736ab712.elb.us-east-1.amazonaws.com:443
    type: DataGrid
  upgrades:
    type: Shutdown
status:
  conditions:
  - status: "True"
    type: PreliminaryChecksPassed
  - status: "True"
    type: GossipRouterReady
  - message: 'View: mycluster-0-15542,mycluster-1-61750'
    status: "True"
    type: WellFormed
  - message: Site 'mycluster' not ready
    status: "False"
    type: CrossSiteViewFormed
  podStatus:
    ready:
    - mycluster-1
    - mycluster-2
  statefulSetName: mycluster

kubectl get pods -n infinispan

NAME                                                      READY   STATUS    RESTARTS   AGE
infinispan-operator-controller-manager-7764c699b6-rlbcf   1/1     Running   0          41s
mycluster-0                                               1/1     Running   0          17m
mycluster-1                                               1/1     Running   0          16m
mycluster-router-67b4bb9fc5-mv4mt                         1/1     Running   0          21s

kubectl logs -f -n infinispan mycluster-router-67b4bb9fc5-mv4mt

GossipRouter started in 780 ms listening on 0.0.0.0:7900 (TLSv1.3)
Jul 05, 2022 3:55:12 PM org.jgroups.blocks.cs.TcpConnection$Receiver run
WARNING: failed handling message
javax.net.ssl.SSLHandshakeException: Remote host terminated the handshake
        at java.base/sun.security.ssl.SSLSocketImpl.handleEOF(SSLSocketImpl.java:1616)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1434)
        at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1336)
        at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:450)
        at java.base/sun.security.ssl.SSLSocketImpl.ensureNegotiated(SSLSocketImpl.java:841)
        at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:932)
        at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:252)
        at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:271)
        at java.base/java.io.DataInputStream.readInt(DataInputStream.java:392)
        at org.jgroups.blocks.cs.TcpConnection$Receiver.run(TcpConnection.java:305)
        at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.io.EOFException: SSL peer shut down incorrectly
        at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:483)
        at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
        at java.base/sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:160)
        at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:111)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1426)
        ... 9 more

...

added _mycluster-1-61750 (10.42.4.97:53116) to group xsite
Jul 05, 2022 3:55:26 PM org.jgroups.blocks.cs.TcpConnection$Receiver run
WARNING: failed handling message
javax.net.ssl.SSLHandshakeException: Remote host terminated the handshake
        at java.base/sun.security.ssl.SSLSocketImpl.handleEOF(SSLSocketImpl.java:1616)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1434)
        at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1336)
        at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:450)
        at java.base/sun.security.ssl.SSLSocketImpl.ensureNegotiated(SSLSocketImpl.java:841)
        at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:932)
        at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:252)
        at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:271)
        at java.base/java.io.DataInputStream.readInt(DataInputStream.java:392)
        at org.jgroups.blocks.cs.TcpConnection$Receiver.run(TcpConnection.java:305)
        at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.io.EOFException: SSL peer shut down incorrectly
        at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:483)
        at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
        at java.base/sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:160)
        at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:111)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1426)
        ... 9 more
...

Thank you in advance, Volodymyr.

pruivo commented 2 years ago

Are you getting any exceptions in the Infinispan pods?

Those exceptions look like the liveness probe that establish the connection and then drops without doing the SSL handshake. I couldn't find a way to hide them.

vladimir22 commented 2 years ago

Hi @pruivo, There are no exceptions (except DEBUG exception) or ERROR logs inside the Infinispan PODs, but I'd like to notice that "xsite" is NOT working (without TLS settings xsite works well) , below the logs details: 

06:56:14,631 INFO  (main) [BOOT] JVM OpenJDK 64-Bit Server VM Red Hat, Inc. 11.0.13+8-LTS
06:56:14,700 INFO  (main) [BOOT] JVM arguments = [-server, -Dinfinispan.zero-capacity-node=false, -Dinfinispan.bind.address=0.0.0.0, -Xmx512m, -XX:+ExitOnOutOfMemoryError, -XX:MetaspaceSize=32m, -XX:MaxMetaspaceSize=96m, -Djava.net.preferIPv4Stack=true, -Djava.awt.headless=true, -Dvisualvm.display.name=infinispan-server, -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager, -Dinfinispan.server.home.path=/opt/infinispan, -classpath, :/opt/infinispan/boot/infinispan-server-runtime-13.0.10.Final-loader.jar, org.infinispan.server.loader.Loader, org.infinispan.server.Bootstrap, --bind-address=0.0.0.0, -l, /opt/infinispan/server/conf/operator/log4j.xml, -c, operator/infinispan.xml]
06:56:14,700 INFO  (main) [BOOT] PID = 141
06:56:14,804 INFO  (main) [org.infinispan.SERVER] ISPN080000: Infinispan Server starting
06:56:14,805 INFO  (main) [org.infinispan.SERVER] ISPN080017: Server configuration: operator/infinispan.xml
06:56:14,807 INFO  (main) [org.infinispan.SERVER] ISPN080032: Logging configuration: /opt/infinispan/server/conf/operator/log4j.xml
06:56:14,817 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: com.sun.jndi.dns.DnsContextFactory

...

06:55:50,686 WARN  (main) [org.infinispan.CONFIG] ISPN000653: The 'keystore-password' attribute on the 'keystore' element has been deprecated. Please use the 'password' attribute instead
06:55:51,588 INFO  (main) [org.wildfly.openssl.SSL] WFOPENSSL0002 OpenSSL Version OpenSSL 1.1.1k  FIPS 25 Mar 2021
06:55:51,590 INFO  (main) [org.infinispan.SECURITY] ISPN000946: Using OpenSSL Provider
06:55:52,604 DEBUG (main) [org.infinispan.SERVER] Enabled SCRAM, DIGEST and CRAM mechanisms for Hot Rod
06:55:52,604 DEBUG (main) [org.infinispan.SERVER] Enabled PLAIN mechanism for Hot Rod
06:55:52,607 DEBUG (main) [org.infinispan.SERVER] Enabled DIGEST for HTTP
06:55:52,607 DEBUG (main) [org.infinispan.SERVER] Enabled BASIC for HTTP
06:55:52,611 DEBUG (main) [org.infinispan.SERVER] Using endpoint realm "admin" for Hot Rod
06:55:52,611 DEBUG (main) [org.infinispan.SERVER] Enabled SCRAM, DIGEST and CRAM mechanisms for Hot Rod
06:55:52,690 DEBUG (main) [org.infinispan.SERVER] Actual configuration: <?xml version="1.0"?>
<infinispan xmlns="urn:infinispan:config:13.0">
    <jgroups transport="org.infinispan.remoting.transport.jgroups.JGroupsTransport">
        <stacks>
            <stack name="image-tcp" extends="tcp">
                <TCP port-range="0" bind-addr="SITE_LOCAL" bind-port="7800" enable-diagnostics="false"/>
                <dns.DNS-PING dns-record-type="A" stack.combine="REPLACE" stack.position="MPING" dns-query="mycluster-ping.infinispan.svc.cluster.local"/>
            </stack>
            <stack name="relay-tunnel" extends="udp">
                <TUNNEL stack.combine="REPLACE" port-range="0" bind-addr="SITE_LOCAL" bind-port="0" gossip-router-hosts="mycluster-site[7900],10.227.48.190[31314]" enable-diagnostics="false" stack.position="UDP"/>
                <FD-SOCK stack.combine="REMOVE"/>
            </stack>
            <stack name="xsite" extends="image-tcp">
                <relay.RELAY2 site="mycluster" max-site-masters="1"/>
                <remote-sites default-stack="relay-tunnel" cluster="xsite">
                    <remote-site name="mycluster" cluster="xsite" stack="relay-tunnel"/>
                    <remote-site name="mycluster-site" cluster="xsite" stack="relay-tunnel"/>
                </remote-sites>
            </stack>
        </stacks>
    </jgroups>
    <cache-container name="default" shutdown-hook="DONT_REGISTER" statistics="true">
        <transport cluster="" node-name="" stack="xsite"/>
        <global-state>
            <persistent-location path="/opt/infinispan/server/data"/>
            <shared-persistent-location path="/opt/infinispan/server/data"/>
            <overlay-configuration-storage/>
        </global-state>
        <caches>
            <replicated-cache-configuration name="org.infinispan.REPL_ASYNC" mode="ASYNC" statistics="true">
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
                <state-transfer timeout="60000"/>
            </replicated-cache-configuration>
            <scattered-cache-configuration name="org.infinispan.SCATTERED_SYNC" invalidation-batch-size="128" bias-acquisition="ON_WRITE" bias-lifespan="300000" mode="SYNC" remote-timeout="17500" statistics="true">
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
            </scattered-cache-configuration>
            <distributed-cache-configuration name="org.infinispan.DIST_SYNC" mode="SYNC" remote-timeout="17500" statistics="true">
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
                <state-transfer timeout="60000"/>
            </distributed-cache-configuration>
            <invalidation-cache-configuration name="org.infinispan.INVALIDATION_ASYNC" mode="ASYNC" statistics="true">
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
            </invalidation-cache-configuration>
            <local-cache-configuration name="org.infinispan.LOCAL" statistics="true">
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
            </local-cache-configuration>
            <invalidation-cache-configuration name="org.infinispan.INVALIDATION_SYNC" mode="SYNC" remote-timeout="17500" statistics="true">
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
            </invalidation-cache-configuration>
            <replicated-cache-configuration name="org.infinispan.REPL_SYNC" mode="SYNC" remote-timeout="17500" statistics="true">
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
                <state-transfer timeout="60000"/>
            </replicated-cache-configuration>
            <distributed-cache-configuration name="example.PROTOBUF_DIST" mode="SYNC" remote-timeout="17500" statistics="true">
                <encoding>
                    <key media-type="application/x-protostream"/>
                    <value media-type="application/x-protostream"/>
                </encoding>
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
                <state-transfer timeout="60000"/>
            </distributed-cache-configuration>
            <distributed-cache-configuration name="org.infinispan.DIST_ASYNC" mode="ASYNC" statistics="true">
                <locking concurrency-level="1000" acquire-timeout="15000" striping="false"/>
                <state-transfer timeout="60000"/>
            </distributed-cache-configuration>
        </caches>
    </cache-container>
    <server xmlns="urn:infinispan:server:13.0">
        <interfaces>
            <interface name="public">
                <inet-address value="0.0.0.0"/>
            </interface>
        </interfaces>
        <socket-bindings port-offset="0" default-interface="public">
            <socket-binding name="default" port="11222" interface="public"/>
            <socket-binding name="admin" port="11223" interface="public"/>
        </socket-bindings>
        <security>
            <credential-stores>
                <credential-store name="credentials" path="credentials.pfx">
                    <clear-text-credential clear-text="***"/>
                </credential-store>
            </credential-stores>
            <security-realms>
                <security-realm name="default">
                    <server-identities>
                        <ssl>
                            <keystore alias="mycluster" path="/etc/encrypt/keystore/keystore.p12">
                                <credential-reference store="credentials" alias="keystore"/>
                            </keystore>
                        </ssl>
                    </server-identities>
                    <properties-realm groups-attribute="Roles">
                        <user-properties digest-realm-name="default" path="cli-users.properties"/>
                        <group-properties path="cli-groups.properties"/>
                    </properties-realm>
                </security-realm>
                <security-realm name="admin">
                    <properties-realm groups-attribute="Roles">
                        <user-properties digest-realm-name="admin" path="cli-admin-users.properties"/>
                        <group-properties path="cli-admin-groups.properties"/>
                    </properties-realm>
                </security-realm>
                <security-realm name="transport">
                    <server-identities>
                        <ssl>
                            <keystore alias="mycluster" path="/etc/encrypt/transport-site-tls/keystore.p12" password="***"/>
                            <truststore path="/etc/encrypt/truststore-site-tls/keystore.p12" password="***"/>
                        </ssl>
                    </server-identities>
                </security-realm>
            </security-realms>
        </security>
        <endpoints>
            <endpoint socket-binding="default" security-realm="default">
                <hotrod-connector name="hotrod-default" socket-binding="default">
                    <authentication security-realm="default">
                        <sasl server-name="infinispan" mechanisms="SCRAM-SHA-512 SCRAM-SHA-384 SCRAM-SHA-256 SCRAM-SHA-1 DIGEST-SHA-512 DIGEST-SHA-384 DIGEST-SHA-256 DIGEST-SHA CRAM-MD5 DIGEST-MD5 PLAIN" qop="auth"/>
                    </authentication>
                </hotrod-connector>
                <rest-connector name="rest-default" socket-binding="default">
                    <authentication mechanisms="DIGEST BASIC" security-realm="default"/>
                </rest-connector>
            </endpoint>
            <endpoint socket-binding="admin" security-realm="admin">
                <rest-connector name="rest-admin" socket-binding="admin">
                    <authentication mechanisms="BASIC DIGEST" security-realm="admin"/>
                </rest-connector>
                <hotrod-connector name="hotrod-admin" socket-binding="admin">
                    <authentication security-realm="admin">
                        <sasl mechanisms="SCRAM-SHA-512 SCRAM-SHA-384 SCRAM-SHA-256 SCRAM-SHA-1 DIGEST-SHA-512 DIGEST-SHA-384 DIGEST-SHA-256 DIGEST-SHA CRAM-MD5 DIGEST-MD5"/>
                    </authentication>
                </hotrod-connector>
            </endpoint>
        </endpoints>
    </server>
</infinispan>

06:55:52,702 INFO  (main) [org.infinispan.SERVER] ISPN080060: SSL Transport using realm 'transport`

...

06:56:22,123 INFO  (main) [org.infinispan.CONTAINER] ISPN000389: Loaded global state, version=13.0.10.Final timestamp=2022-07-05T15:37:41.449163Z
06:56:22,607 INFO  (main) [org.infinispan.CLUSTER] ISPN000078: Starting JGroups channel `` with stack `xsite`
06:56:22,840 DEBUG (main) [org.jgroups.stack.Configurator] set property TCP.diagnostics_addr to default value /224.0.75.75
06:56:22,897 DEBUG (main) [org.jgroups.protocols.TCP] thread pool min/max/keep-alive: 0/200/60000 use_fork_join=false, internal pool: 0/4/30000 (1 cores available)
06:56:22,937 DEBUG (main) [org.infinispan.remoting.transport.jgroups.JGroupsTransport] JGroups protocol stack: TCP(time_service_interval=500;thread_pool_max_threads=200;ergonomics=true;port_range=0;conn_expire_time=0;recv_buf_size=0;loopback_copy=false;thread_naming_pattern=pl;use_common_fork_join_pool=false;send_buf_size=640000;tcp_nodelay=true;stats=true;buffered_input_stream_size=8192;suppress_time_different_version_warnings=60000;diagnostics_port_range=50;id=57;logical_addr_cache_max_size=2000;suppress_time_different_cluster_warnings=60000;max_length=0;bundler_wait_strategy=park;sock_conn_timeout=300;client_bind_port=0;use_fibers=false;bundler_num_spins=5;use_fork_join_pool=false;enable_diagnostics=false;external_port=0;log_discard_msgs=true;bind_addr=10.42.0.94;bundler_capacity=16384;log_discard_msgs_version=true;defer_client_bind_addr=false;use_ip_addrs=false;buffered_output_stream_size=8192;log_accept_error=true;thread_pool_keep_alive_time=60000;thread_dumps_threshold=10000;bind_port=7800;thread_pool_min_threads=0;diag_enable_udp=true;reaper_interval=0;diagnostics_ttl=8;loopback_separate_thread=true;logical_addr_cache_expiration=360000;peer_addr_read_timeout=1000;msg_processing_max_buffer_size=5000;linger=-1;diagnostics_addr=224.0.75.75;receive_on_all_interfaces=false;diag_enable_tcp=false;max_bundle_size=64000;thread_pool_enabled=true;bundler_type=transfer-queue;logical_addr_cache_reaper_interval=60000;diagnostics_port=7500;who_has_cache_timeout=2000;spawn_thread_on_full_pool=false)
:dns.DNS_PING(dns_record_type=A;async_discovery_use_separate_thread_per_request=false;dns_context_factory=com.sun.jndi.dns.DnsContextFactory;dns_query=mycluster-ping.infinispan.svc.cluster.local;ergonomics=true;stagger_timeout=0;async_discovery=false;max_rank_to_reply=0;probe_transport_ports=false;max_members_in_discovery_request=500;send_cache_on_join=true;num_discovery_runs=1;break_on_coord_rsp=true;stats=true;use_disk_cache=false;dns_address=;discovery_rsp_expiry_time=60000;id=61;return_entire_cache=false)
:MERGE3(check_interval=48000;stats=true;min_interval=10000;ergonomics=true;id=38;max_participants_in_merge=100;max_interval=30000;only_coords_run_consistency_checker=false)
:FD_SOCK(cache_max_age=10000;get_cache_timeout=1000;sock_conn_timeout=1000;client_bind_port=0;ergonomics=true;start_port=0;port_range=50;suspect_msg_interval=5000;num_tries=3;stats=true;external_port=0;bind_addr=10.42.0.94;keep_alive=true;id=2;cache_max_elements=200)
:FD_ALL(use_time_service=true;stats=true;timeout_check_interval=1000;ergonomics=true;interval=2000;id=20;timeout=10000)
:VERIFY_SUSPECT(num_msgs=1;use_mcast_rsps=false;stats=true;ergonomics=true;bind_addr=10.42.0.94;id=11;timeout=1000;use_icmp=false)
:pbcast.NAKACK2(resend_last_seqno_max_times=1;use_mcast_xmit=false;ergonomics=true;xmit_table_msgs_per_row=1024;xmit_table_max_compaction_time=30000;become_server_queue_size=50;xmit_interval=200;resend_last_seqno=true;max_xmit_req_size=511600;discard_delivered_msgs=true;suppress_time_non_member_warnings=60000;xmit_table_num_rows=50;stats=true;xmit_from_random_member=false;log_discard_msgs=true;log_not_found_msgs=true;xmit_table_resize_factor=1.2;id=41;max_rebroadcast_timeout=2000;use_mcast_xmit_req=false)
:UNICAST3(ergonomics=true;xmit_table_msgs_per_row=1024;xmit_table_max_compaction_time=30000;ack_threshold=100;sync_min_interval=2000;max_retransmit_time=60000;xmit_interval=200;max_xmit_req_size=511600;conn_close_timeout=5000;conn_expiry_timeout=120000;xmit_table_num_rows=50;stats=true;xmit_table_resize_factor=1.2;log_not_found_msgs=true;id=48)
:pbcast.STABLE(stability_delay=6000;stats=true;ergonomics=true;desired_avg_gossip=5000;max_bytes=1000000;id=13;send_stable_msgs_to_coord_only=true)
:pbcast.GMS(max_join_attempts=10;print_local_addr=false;view_bundling=true;leave_timeout=2000;all_clients_retry_timeout=100;log_view_warnings=true;install_view_locally_first=true;ergonomics=true;use_delta_views=true;use_flush_if_present=true;print_view_details=true;print_physical_addrs=true;join_timeout=2000;max_leave_attempts=10;view_ack_collection_timeout=2000;stats=true;num_prev_views=10;merge_timeout=5000;max_bundling_time=50;num_prev_mbrs=50;id=12;log_collect_msgs=false;membership_change_policy=org.jgroups.protocols.pbcast.GMS$DefaultMembershipPolicy@273dec10)
:UFC(min_threshold=0.4;stats=true;ergonomics=true;max_credits=4000000;min_credits=1600000;id=31;max_block_time=500)
:MFC(min_threshold=0.4;stats=true;ergonomics=true;max_credits=4000000;min_credits=1600000;id=30;max_block_time=500)
:FRAG3(frag_size=60000;stats=true;ergonomics=true;id=62)
--
06:56:23,097 DEBUG (main) [org.jgroups.protocols.dns.DNS_PING] mycluster-1-52723: no entries collected from DNS (in 71 ms)
06:56:25,107 INFO  (main) [org.jgroups.protocols.pbcast.GMS] mycluster-1-52723: no members discovered after 2082 ms: creating cluster as coordinator
06:56:25,110 DEBUG (main) [org.jgroups.protocols.pbcast.NAKACK2]
[mycluster-1-52723 setDigest()]
existing digest:  []
new digest:       mycluster-1-52723: [0 (0)]
resulting digest: mycluster-1-52723: [0 (0)]
06:56:25,110 DEBUG (main) [org.jgroups.protocols.pbcast.GMS] mycluster-1-52723: installing view [mycluster-1-52723|0] (1) [mycluster-1-52723] (mycluster-1-52723 joined)
06:56:25,112 DEBUG (main) [org.jgroups.protocols.pbcast.STABLE] resuming message garbage collection
06:56:25,126 INFO  (main) [org.infinispan.CLUSTER] ISPN000094: Received new cluster view for channel : [mycluster-1-52723|0] (1) [mycluster-1-52723]
06:56:25,128 DEBUG (main) [org.infinispan.remoting.transport.jgroups.JGroupsTransport] Joined: [mycluster-1-52723], Left: []
06:56:25,137 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.stack.Configurator] set property TUNNEL.diagnostics_addr to default value /224.0.75.75
06:56:25,144 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.TUNNEL] thread pool min/max/keep-alive: 0/100/30000 use_fork_join=false, internal pool: 0/4/30000 (1 cores available)
06:56:25,145 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.TUNNEL] gossip routers are [mycluster-site/10.43.41.45:7900, /10.227.48.190:31314]
06:56:25,145 DEBUG (main) [org.jgroups.protocols.pbcast.GMS] mycluster-1-52723: created cluster (first member). My view is [mycluster-1-52723|0], impl is CoordGmsImpl
06:56:25,197 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.pbcast.GMS] address=_mycluster-1-52723:mycluster, cluster=xsite, physical address=10.42.0.94:33786
06:56:25,506 INFO  (main) [org.infinispan.CLUSTER] ISPN000079: Channel `` local address is `mycluster-1-52723`, physical addresses are `[10.42.0.94:7800]`
06:56:25,529 DEBUG (main) [org.infinispan.registry.impl.InternalCacheRegistryImpl] Registering internal cache org.infinispan.CONFIG [GLOBAL]
06:56:25,531 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.anchored.configuration.AnchoredKeysConfigurationParser
06:56:25,532 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.persistence.jdbc.configuration.JdbcStoreConfigurationParser
06:56:25,532 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationParser
06:56:25,533 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.TUNNEL] _mycluster-1-52723:mycluster: connection to 10.227.48.190:31314 closed, trying to re-establish connection
06:56:25,533 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.persistence.rocksdb.configuration.RocksDBStoreConfigurationParser
06:56:25,533 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.persistence.sql.configuration.SqlStoreConfigurationParser
06:56:25,534 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.cloudevents.configuration.CloudEventsConfigurationParser
06:56:25,603 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.counter.configuration.CounterConfigurationParser
06:56:25,603 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.counter.configuration.CounterParser
06:56:25,603 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.lock.configuration.ClusteredLockConfigurationParser
06:56:25,604 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.configuration.parsing.CacheParser
06:56:25,604 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.configuration.parsing.Parser
06:56:25,604 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.server.configuration.ServerConfigurationParser
--
06:56:25,621 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Ignoring already loaded service: org.infinispan.configuration.parsing.CacheParser
06:56:25,622 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Ignoring already loaded service: org.infinispan.configuration.parsing.Parser
06:56:25,622 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Ignoring already loaded service: org.infinispan.server.configuration.ServerConfigurationParser
06:56:25,622 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Ignoring already loaded service: org.infinispan.server.configuration.hotrod.HotRodServerConfigurationParser
06:56:25,622 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Ignoring already loaded service: org.infinispan.server.configuration.memcached.MemcachedServerConfigurationParser
06:56:25,622 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Ignoring already loaded service: org.infinispan.server.configuration.rest.RestServerConfigurationParser
06:56:25,624 DEBUG (main) [org.infinispan.globalstate.impl.GlobalConfigurationManagerImpl] Starting statically defined caches: [___event_log_cache, ___protobuf_metadata, org.infinispan.CONFIG, org.infinispan.COUNTER, ___script_cache, org.infinispan.CLIENT_SERVER_TX_TABLE, org.infinispan.LOCKS]
06:56:25,625 DEBUG (main) [org.infinispan.manager.DefaultCacheManager] Creating cache ___event_log_cache on mycluster-1-52723
06:56:25,722 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.pbcast.GMS] _mycluster-1-52723:mycluster: sending JOIN(_mycluster-1-52723:mycluster) to _mycluster-0-3651:mycluster
06:56:25,829 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.pbcast.NAKACK2]
[_mycluster-1-52723:mycluster setDigest()]
existing digest:  []
new digest:       _mycluster-0-3651:mycluster: [0 (0)], _mycluster-1-52723:mycluster: [0 (0)]
resulting digest: _mycluster-0-3651:mycluster: [0 (0)], _mycluster-1-52723:mycluster: [0 (0)]
06:56:25,829 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.pbcast.GMS] _mycluster-1-52723:mycluster: installing view [_mycluster-0-3651:mycluster|1] (2) [_mycluster-0-3651:mycluster, _mycluster-1-52723:mycluster] ([_mycluster-0-3651:mycluster, _mycluster-1-52723:mycluster] joined)
06:56:25,911 INFO  (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.relay.RELAY2] _mycluster-1-52723:mycluster: joined bridge cluster 'xsite'
06:56:25,911 INFO  (jgroups-5,mycluster-1-52723) [org.infinispan.XSITE] ISPN000439: Received new x-site view: [mycluster]
06:56:26,208 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.query.core.impl.continuous.IckleContinuousQueryFilterIndexingServiceProvider
06:56:26,209 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.query.core.impl.eventfilter.IckleFilterIndexingServiceProvider
06:56:26,210 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.query.remote.impl.filter.IckleContinuousQueryProtobufFilterIndexingServiceProvider
06:56:26,211 DEBUG (main) [org.infinispan.commons.util.ServiceFinder] Loading service impl: org.infinispan.query.remote.impl.filter.IckleProtobufFilterIndexingServiceProvider
06:56:26,336 INFO  (main) [org.jboss.threads] JBoss Threads version 2.3.3.Final
06:56:26,413 DEBUG (main) [org.infinispan.interceptors.impl.AsyncInterceptorChainImpl] Interceptor chain size: 8
06:56:26,413 DEBUG (main) [org.infinispan.interceptors.impl.AsyncInterceptorChainImpl] Interceptor chain is:
        >> org.infinispan.interceptors.impl.InvocationContextInterceptor@f08d676
        >> org.infinispan.interceptors.impl.VersionInterceptor@3eedd0e3
        >> org.infinispan.interceptors.impl.CacheMgmtInterceptor@478089b6
        >> org.infinispan.interceptors.locking.NonTransactionalLockingInterceptor@72f35a31
        >> org.infinispan.interceptors.impl.EntryWrappingInterceptor@3b01897f
        >> org.infinispan.interceptors.impl.PassivationCacheLoaderInterceptor@6b3b2c34
        >> org.infinispan.interceptors.impl.PassivationWriterInterceptor@28b5d5dc
--
06:56:28,703 DEBUG (main) [org.infinispan.server.core.transport.NettyTransport] ISPN005025: Using Netty SocketChannel io.netty.channel.epoll.EpollServerSocketChannel for SinglePortRouterConfiguration = [cache=null, name=, host=0.0.0.0, port=11223, idle-timeout=-1, ignored-caches=[], receive-buffer-size=0, send-buffer-size=0, start-transport=true, tcp-nodelay=true, tcp-keepalive=false, io-threads=2, admin-operation-handler=null, zero-capacity-node=false, socket-binding=admin, implicit-connector=false, worker-threads=1]
06:56:28,723 DEBUG (main) [org.infinispan.SERVER] REST EndpointRouter listening on 0.0.0.0:11223
06:56:28,723 INFO  (main) [org.infinispan.SERVER] ISPN080004: Connector SinglePort (admin) listening on 0.0.0.0:11223
06:56:28,723 INFO  (main) [org.infinispan.SERVER] ISPN080034: Server 'mycluster-1-52723' listening on http://0.0.0.0:11223
06:56:28,823 INFO  (main) [org.infinispan.SERVER] ISPN080001: Infinispan Server 13.0.10.Final started in 14012ms
06:56:43,447 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.dns.DNS_PING] mycluster-1-52723: entries collected from DNS (in 4 ms): [10.42.0.94:0, 10.42.2.165:0]
06:56:43,447 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.dns.DNS_PING] mycluster-1-52723: sending discovery requests to hosts [10.42.0.94:0, 10.42.2.165:0] on ports [7800 .. 7800]
06:56:47,811 DEBUG (jgroups-5,mycluster-1-52723) [org.jgroups.protocols.pbcast.STABLE] suspending message garbage collection
06:56:47,812 DEBUG (jgroups-5,mycluster-1-52723) [org.jgroups.protocols.pbcast.STABLE] mycluster-1-52723: resume task started, max_suspend_time=220000
06:56:47,824 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.pbcast.NAKACK2]
[mycluster-1-52723 mergeDigest()]
existing digest:  mycluster-1-52723: [9 (9)]
new digest:       mycluster-0-3651: [9 (9)], mycluster-1-52723: [9 (9)]
resulting digest: mycluster-0-3651: [9 (9)], mycluster-1-52723: [9 (9)]
06:56:47,824 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.pbcast.GMS] mycluster-1-52723: installing view MergeView::[mycluster-0-3651|1] (2) [mycluster-0-3651, mycluster-1-52723], 2 subgroups: [mycluster-1-52723|0] (1) [mycluster-1-52723], [mycluster-0-3651|0] (1) [mycluster-0-3651] (mycluster-0-3651 joined)
06:56:47,826 INFO  (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.relay.RELAY2] _mycluster-1-52723:mycluster: leaving bridge cluster 'xsite'
06:56:47,834 DEBUG (FD_SOCK pinger-11,mycluster-1-52723) [org.jgroups.protocols.FD_SOCK] mycluster-1-52723: pingable_mbrs=[mycluster-0-3651, mycluster-1-52723], ping_dest=mycluster-0-3651
06:56:47,874 INFO  (jgroups-6,mycluster-1-52723) [org.infinispan.CLUSTER] ISPN000093: Received new, MERGED cluster view for channel : MergeView::[mycluster-0-3651|1] (2) [mycluster-0-3651, mycluster-1-52723], 2 subgroups: [mycluster-1-52723|0] (1) [mycluster-1-52723], [mycluster-0-3651|0] (1) [mycluster-0-3651]
06:56:47,875 DEBUG (jgroups-6,mycluster-1-52723) [org.infinispan.remoting.transport.jgroups.JGroupsTransport] Joined: [mycluster-0-3651], Left: []
06:56:47,879 INFO  (jgroups-6,mycluster-1-52723) [org.infinispan.CLUSTER] ISPN100000: Node mycluster-0-3651 joined the cluster
06:56:47,882 DEBUG (jgroups-6,mycluster-1-52723) [org.jgroups.protocols.pbcast.STABLE] resuming message garbage collection
06:56:47,886 DEBUG (non-blocking-thread--p2-t1) [org.infinispan.topology.LocalTopologyManagerImpl] Sending cluster status response for view 1
06:56:47,919 DEBUG (non-blocking-thread--p2-t1) [org.infinispan.topology.LocalTopologyManagerImpl] Updating local topology for cache ___hotRodTopologyCache_hotrod-admin: CacheTopology{id=2, phase=NO_REBALANCE, rebalanceId=2, currentCH=ReplicatedConsistentHash{ns = 256, owners = (1)[mycluster-1-52723: 256+0]}, pendingCH=null, unionCH=null, actualMembers=[mycluster-1-52723], persistentUUIDs=[e3d8e7db-e6e8-408a-81b5-6a4887bc9937]}
06:56:47,921 DEBUG (non-blocking-thread--p2-t1) [org.infinispan.statetransfer.StateConsumerImpl] Removing no longer owned entries for cache ___hotRodTopologyCache_hotrod-admin
06:56:47,937 DEBUG (non-blocking-thread--p2-t2) [org.infinispan.topology.LocalTopologyManagerImpl] Updating local topology for cache ___hotRodTopologyCache_hotrod-admin: CacheTopology{id=3, phase=NO_REBALANCE, rebalanceId=2, currentCH=ReplicatedConsistentHash{ns = 256, owners = (1)[mycluster-1-52723: 256+0]}, pendingCH=null, unionCH=null, actualMembers=[mycluster-1-52723], persistentUUIDs=[e3d8e7db-e6e8-408a-81b5-6a4887bc9937]}
06:56:47,941 DEBUG (non-blocking-thread--p2-t2) [org.infinispan.statetransfer.StateConsumerImpl] Removing no longer owned entries for cache ___hotRodTopologyCache_hotrod-admin
06:56:47,950 DEBUG (non-blocking-thread--p2-t1) [org.infinispan.topology.LocalTopologyManagerImpl] Updating local topology for cache ___hotRodTopologyCache_hotrod-admin: CacheTopology{id=4, phase=READ_OLD_WRITE_ALL, rebalanceId=3, currentCH=ReplicatedConsistentHash{ns = 256, owners = (1)[mycluster-1-52723: 256+0]}, pendingCH=ReplicatedConsistentHash{ns = 256, owners = (2)[mycluster-1-52723: 131+125, mycluster-0-3651: 125+131]}, unionCH=null, actualMembers=[mycluster-1-52723, mycluster-0-3651], persistentUUIDs=[e3d8e7db-e6e8-408a-81b5-6a4887bc9937, eed9e843-b38d-4d3c-abb1-f9aa382e6e87]}
06:56:47,951 DEBUG (non-blocking-thread--p2-t1) [org.infinispan.topology.LocalTopologyManagerImpl] Starting local rebalance for cache ___hotRodTopologyCache_hotrod-admin, topology = CacheTopology{id=4, phase=READ_OLD_WRITE_ALL, rebalanceId=3, currentCH=ReplicatedConsistentHash{ns = 256, owners = (1)[mycluster-1-52723: 256+0]}, pendingCH=ReplicatedConsistentHash{ns = 256, owners = (2)[mycluster-1-52723: 131+125, mycluster-0-3651: 125+131]}, unionCH=null, actualMembers=[mycluster-1-52723, mycluster-0-3651], persistentUUIDs=[e3d8e7db-e6e8-408a-81b5-6a4887bc9937, eed9e843-b38d-4d3c-abb1-f9aa382e6e87]}
06:56:47,956 DEBUG (non-blocking-thread--p2-t1) [org.infinispan.statetransfer.StateConsumerImpl] Finished receiving of segments for cache ___hotRodTopologyCache_hotrod-admin for topology 4.
06:56:47,983 DEBUG (non-blocking-thread--p2-t2) [org.infinispan.topology.LocalTopologyManagerImpl] Updating local topology for cache ___hotRodTopologyCache_hotrod-default: CacheTopology{id=3, phase=NO_REBALANCE, rebalanceId=2, currentCH=ReplicatedConsistentHash{ns = 256, owners = (1)[mycluster-1-52723: 256+0]}, pendingCH=null, unionCH=null, actualMembers=[mycluster-1-52723], persistentUUIDs=[e3d8e7db-e6e8-408a-81b5-6a4887bc9937]}
06:56:47,984 DEBUG (non-blocking-thread--p2-t2) [org.infinispan.statetransfer.StateConsumerImpl] Removing no longer owned entries for cache ___hotRodTopologyCache_hotrod-default
tristantarrant commented 2 years ago

I see FIPS there. Are you using PKCS#11 keystores ?

vladimir22 commented 2 years ago

I am using keys and certs content which are generated by cert-manager using encoding: PKCS1 and algorithm: RSA values (the details below), It is working well for Hotrod client TLS Encryption. Did you mean that it is not not supported by GossipRouter ? 

CLUSTER_NAME=mycluster
## Create Certificate resource to generate certificate using cert-manager
kubectl -n infinispan apply -f - <<EOF
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: $CLUSTER_NAME-cert
spec:
  dnsNames:
  - $CLUSTER_NAME.infinispan.svc
  - $CLUSTER_NAME.infinispan.svc.cluster.local
  duration: 2160h0m0s
  issuerRef:
    kind: ClusterIssuer
    name: cert-manager-ca-issuer
  privateKey:
    algorithm: RSA
    encoding: PKCS1
    size: 2048
  renewBefore: 360h0m0s
  secretName: $CLUSTER_NAME-cert
EOF

## View secret created by cert-manager
kubectl get secret -n infinispan $CLUSTER_NAME-cert -o yaml

## Create keystore.pkcs12
kubectl get secret -n infinispan $CLUSTER_NAME-cert -o jsonpath='{.data.tls\.crt}' | base64 --decode > tls.crt
kubectl get secret -n infinispan $CLUSTER_NAME-cert -o jsonpath='{.data.tls\.key}' | base64 --decode > tls.key

cat tls.key tls.crt > tls.pem

openssl pkcs12 -export -in tls.pem -out keystore.pkcs12 -name $CLUSTER_NAME -noiter -nomaciter
Enter Export Password: password

## Create custom secret
kubectl create secret -n infinispan generic $CLUSTER_NAME-tls-secret \
  --from-file=tls.crt=./tls.crt \
  --from-file=tls.key=./tls.key \
  --from-file=keystore.p12=./keystore.pkcs12 \
  --from-literal=alias=$CLUSTER_NAME \
  --from-literal=password=password 

kubectl get secret -n infinispan $CLUSTER_NAME-tls-secret -o yaml

apiVersion: v1
data:
  alias: bXljbHVzdGVy
  keystore.p12: MIINrgIBAzCCDXgGCSqGSIb3DQEHAaCCDWkEgg1lMIINYTCCB/YGCSqGSIb3DQEHBqCCB+cwggfjAgEAMIIH3AYJKoZIhvcNAQcBMBsGCiqGSIb3DQEMAQYwDQQIvCBKjoM0OzYCAQGAggew5NWrslXauCY2C+TDkvayQjHVIdKVfSotOMLzcJM2FhRvsOvLSqpWROM+uzZcO0fL14WrHfC8epoo8ZMbEfdFiW6l8QPlcfX8h/Xm+6AGq6bR8a+M2do0sCByFurqybblNvbF0qGdfUNAkyCoZbL/cAdBNuLW7YqQ3fyH7p7g1gfFDGCmdHXzj2BLRKAb9RNEsiEKCyQSWSdy4GtJBpvw+/ZkhG0mtC4nmieET4bPyP58sN84EdNolMGcprC3yf7yJ3LtMUPXPRBk/oCjk9NWCd15MIy1ylf/UuOgTPNbpYUcv15I5yxj6wsvQxNmlB+q3FTyUSVpYerKWDfIBY92TE77etokZDbXbkpR7cdKl1ndNj/2m3TWZFGCfHZ5fmsVYNmM+XJYaFeTIl6y3k0CJHefSZ6x4Vd2bAhIg5aN0DmVHTPTI7aGdjivT+5a0s2jnOAZ/HcnI7MdbkGEi3qawVN9W3D3BAb8Ef/TFMITobC5m0EBFImLhQ2CsAmuW+S6+YV6mnPDYvNircqsp/B9GZFdmz39PM4xVofnjZHdcLZadx9xlYYMlTvnMVV6Ir/gOw4cGmh1Xk2a2fe4YlUnCsEFWDHvgi9nbu6E2BtTkG0SSCx/gamCXG++n21XLuFUFevx9di7c6HhWxFB16v/9AlWQ6btoN33rCRCDsSri6RU2uh7tge070nTtBovM+gYzyF/QqHc23sgEd8FqiavCDSpwgOSLDmM7YYU9AxwFoGK6IvOhGOPWEYRWaqUgLOYrapuDdsXEErnaZBXwFmmY2HkZeX+5yvUtl7+9brjjLoAF1bmRXuLx+1IxjTBbh6f1ZCR82qYEFziX6AL0AvDbHZAcFxqYL/74l8bOlWy0lXF6Ijqx/eJRwL5tKACx6v7hL3JvXx+HFkHPQmC6I6lhXp13Yw3P0gzDblR1eKsLfPUFgTnwXmmxerkETtFaEHkmlZywUhQDQph44Su1cwDPJAQAv6Xx0rkQVrjdmUtnlPNMSmVtr59654kuGeYdan047l3IDNVGr9SWXYfVc4jiYg+M5GIHFgFVtLoPQ3wO4r8V9MvwktKI4ZtUOu7X38lfoh5nODJgCv6bkwtzA0qxgLRetbsWXtXSwsRM2Z/ibAChJlLj9L05EpTd4P8Ot9oBmVgxrPRPvRTx6xEDCDpm3F1YJmtkJBA6vVkUTWgpStPZROV7rYugmcq5lf+fuTjwQ/laUvje94/p8OdHT0UsMeJm1gahj06YtgWT84jX1mzBxEEqnXqwFcKQOifFZLInooRHLkrpWTyEDgkCTLnzvDOnrGIq2a4tfHDl9pfeAWONXXQzR1l5rFJ+118ucr/FRsCFBD1BhWyrqtY1Eaj3nbPY8fzdxO0PKxUr0/2CHqj+a/Z4ALQ4y9Ny6lrJnUA8S/59b6k30fZSyoGOsXQLRppC2FrCB2aRKpgb/I5gdF9zbQq9Yg2xJ6Mj5DXQg+OxVo++9ar3dQhuOBgMK1OJ0gv8a9UWOF6V/1YEgFUTzFGpr/A/LxRRx/ACZa7SFKfQ45Gz4ObusWGzS3Li5DGtkbOu258lQWYR07UHBhi/CPBMrYVT5/CEp77S+i9YCsFMIPORR+p2WM/S+AUDYQ1AWT5jRyhEeZ3Lmc6oBV79WFI4x8rngbiKUE9x7KigizOF0tTr3jCiADBiitkj2v8BMbVKMf8GQQtBcwD75T0PJJoaMXJeXCJxxJp+ham4/vjnI5ealTWT82KrxmxY6la7muJEw/pgjCWSJuCC25Hs20oA0waJ4Ojv7foP/1QN50rwhx3haR9wynsiPl44vVlD53kLhqk0tHAPZmkNGM3Y02FGiDIQ5xDU+0cHUJR+GDXRjb2nKVjPGnl4idLfebVkqvDOu80EnjITp6Bgjq2KgsY6za2x32YPN/3k28Twz1XCL2+kP7O9iPzm3ULE6Vil83/XTrWM8qOpbDm9dSEcgZoR30adSTqoUhiOSNJHdKiadUJw5hEWGCvD7J23er4YTAEGDTPCzb9Mf97CByST3DV/fkfo8GhdcaIHQ8muX8lyNrgJgBsYtnPGqWuq7fWFTWOQtzZLBW+i0lxWEGjloSxN1hLVBMLNBfFbD7S6puMzxwtnTMw13EcRSriZWVsNravMSXfVj7/9T+1LL2Jrht1kcqrpbTmpHJhl9UPSsRh4fcVQp6mm7QFuphRw/EF99iGgxZd9xn1W80m+W7cBhCzxPnYOyfmtgV0uQcvSR4YPyc96/xt1Tt0ppfr8dQCLgf3wGTrn451XFAyl9WVwugXezde628NLocHHCjjWDM4qmYLBQ6s/YPDV8n4oXVv1XhUyFE5a/ZOeMexsr57jaZcNIYcoDneN5ZbeZeG+oOMylHyapXvAkzGGrcA8ffZOTAKECJTaa0N+KZ9ZoIlcAqCEXALFP5gYZz7MDZTe8wmdWs30BMrywJF+hUHH41cTA/KGNuMuAFwtspdNj23SQ12MtvVRkhsaehZxbq1bip7nf17Lb/8V82CVoDIE8FX4Ji/JFohXbEVbM38IUXtAT/jfOlwm46OZBUsHkkEcZ0FxRDz5ePPeylQD0HTs+zaqK7R3biPmzdvhdJq5Hi4FYhye0sjLlUaLmKaJC71dBN8MIIFYwYJKoZIhvcNAQcBoIIFVASCBVAwggVMMIIFSAYLKoZIhvcNAQwKAQKgggTtMIIE6TAbBgoqhkiG9w0BDAEDMA0ECDx8+J53kGrTAgEBBIIEyHfIwsRL1QPdJSdcsCMjV2nNESRSh4hBFnlcuwNXJR2tK0x59VK8JPLy7Zr/ZVJwt5070FZtvcC58EOU77TajQ3BUrBg39ObuR2NZ0nKkroibkIlifWO4tNyajac1IWujxOVMz0lAmDlnQi9n45Y7AZtxmaFciJR8A486ZSfsyujdF/ftsG+SD8QKVFdXPxtL58bEriULjEiBPUNlPXAOLOpugiSPgnhHSf0aMgVhvmDS+ccnDYGlpiD517K75V/eIoxUUu1cw53xi5ut9zQbulYpdaspp94ivpgUgy1wzPS5pevFcC5AtS2byoTKKX4EvI8yFKFwW5zH8+eWBzeZ8F4a3Lubw/PDRxL7PoOM5Ct+HF222YXG7wHybPlrJb9k3NzrZAoZQMl+T+D0OFgEtVU/qTPfUlk3aAogazlM0iHXyTYTHVvz+MYIx/CDlcUtk1yWjgZ9HaQXVDxmreDVxPlQf3sh39+KyJ4JQAU+NjuJbLEhNurmiJhjt/k0Om3teWD/v5yCq35Afg0W9c2bfEAvFSmLtSzz3jSR9I/SMROgR4s2qmb1lw39hNhnMlFtyqTT9u8gFBHdt7h/qxJgugaZhYvHrG3/UCwzLFSmFX1UejrbW+LqpayEbyhFcUrQmNnkNZLiGNq36gghK2tp+H1yIFNTJ3dg9LUP/8dULLsqOqwoDvblvgER3VwR9RWXCQ4QRcHRqhDYfe14HgWT7SAupcj8Lzi3YhBliFgenEtCRR7uSrgWuLQrs4dnhLf6aNy839DyDvGTdOrpxxBafvL3flapUWp1WenaisUQMUPvns9nZ7216lzrDIS7DT3PFq4wokhYDdyHCFpFGFIfP8J+xWTPooMyPZT1TyUte6sXqJjSYkEGircro/xZnbP/RHjb87uivTGC7WZlopn+oG7vn2uKf9ielvNeWH8QFNq8jb3l0rOB3YSBm+iBuBX3iC+ghVI7bR6W27NxrHCws+08dg5SgGRikGWqLCBKielSWn55apMNc4D0Ia0o20jvP3n2I9wPUVVVLplYY5FsvD6Ng6LEDoDeBONChqx1o7hunAw5+eBbOPH0YsXfyCXZt0l3dY+5wK1kheen3oNzmP+hWUcIO+urzbhl/J1j3C3iz2llrwy7rsT7h+09ON/APddT3uOnW0EdQLid+v6KcmQGHZ8rgv448SUCTfKeKLxISf9El/9RzF9SYsKQEL1QAGu7XyqyDxUq6zwtoi1OBnhnj7tWt9SwlfmEH3A8yIj500026H7NyH/w9BrDiI72wn/9A75S1IdWatuKdtjLRFKPNuHSBtZopvBb8vHhVzV/mgfoEabOH20xTdof32WZs+5UqkcH+fdEDQeXUOf0kVlHO7dF17Y4nIpjlNmptCSkHCER3eglfFBjq8BUv32JqsO6QeyCwtP4hARLMj/SESNVbKNUEIO/+BzXOSfcjo5tdQ4qbOzaXB+3hozlOsIyocXwyhQK31I/yxOEmrLnRHYIElcrvRPE7d+SevOloehK/OCY9GPN0JY/0yKU/G8bHi6jMYNpkbwWgFUTluwy/BnDViaQJYcYiA+1FUigAg1OJ3iKKkl/4LpRUpJ6nU4eYR43NzV3axZ529NRkqpxRzTL8kDxKhCHTFIMCEGCSqGSIb3DQEJFDEUHhIAbQB5AGMAbAB1AHMAdABlAHIwIwYJKoZIhvcNAQkVMRYEFBtnFdbo/qSxbxcbk7gj2ZQI/6qbMC0wITAJBgUrDgMCGgUABBQgznYVr1uuBnDJMrWZL5Y83Vb5hgQIoRPGwo70mI0=
  password: cGFzc3dvcmQ=
  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURZekNDQWt1Z0F3SUJBZ0lRVXdLM2Z1cnR4VkJ2eHh6S1E2QU96akFOQmdrcWhraUc5dzBCQVFzRkFEQnAKTVFzd0NRWURWUVFHRXdKQlZURVRNQkVHQTFVRUNBd0tVMjl0WlMxVGRHRjBaVEVUTUJFR0ExVUVDZ3dLUjBVZwpSR2xuYVhSaGJERXdNQzRHQTFVRUF3d25SRVV5TGpBZ1JtOTFibVJoZEdsdmJpQkNZWE5sSUMwZ1kyVnlkQzF0CllXNWhaMlZ5SUVOQk1CNFhEVEl5TURjd05URXpORGsxT1ZvWERUSXlNVEF3TXpFek5EazFPVm93QURDQ0FTSXcKRFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUtpdm5aUWxkbldrbUl4N051NXp4QzdHUWY2UwpsaU9hT1p4OWlXbW0rTU96R2VJMFZGeTd6MGljaHVuWTA1RnR3NTd2dDZIcTJpZ2FtdlVQdjhzTzZPbm13QmtLCnpTbjg1by9xVlhJb2Z2OENTNFZlMkNDL0hKdk9QQWVNOWFTbDhvbFV1Rjg1L3hpNmJTSXJqd1FrYjlHZkRRREUKa3dXeG1LRUNMNGVBaElaRnJlTktBVVJpSGFHQjJOL0dYUmZyNEozSXlmdkhkQ0FEaXdaT0M4Q3Z3d3ptZThCKwpyU1ZZdmEvNktoT3RBWU9vT2hVa0xISXJhaTUwNTR3eVBSY3VuSFYxV2tJRFp1bVA4MmtGVFhhbHd5S3BqZnJ3CmZwQ3pzd2RveGtaWWhMczhFZDdSUUF1cmc0QU42UVh5clBuWWRBNXBqSWE1UnZtZnRCK1huSVlNczVjQ0F3RUEKQWFOd01HNHdEZ1lEVlIwUEFRSC9CQVFEQWdXZ01Bd0dBMVVkRXdFQi93UUNNQUF3VGdZRFZSMFJBUUgvQkVRdwpRb0lZYlhsamJIVnpkR1Z5TG1sdVptbHVhWE53WVc0dWMzWmpnaVp0ZVdOc2RYTjBaWEl1YVc1bWFXNXBjM0JoCmJpNXpkbU11WTJ4MWMzUmxjaTVzYjJOaGJEQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUF3dUJsczN6YTZvUzAKOFlZSUlzbGZFL0p1YUpZRDR5YWZJQjVxeGlVaWtacDlmcDRCS0pNRncxSmQzK2pFVlhzeXB1N3NyY0d0TEZPcwpYanNQcU50S1h4QmwvcTNmWmhOMm44YjlOeTF6RkxEVUlURjF6TW9iVDRnYjcxOWFHY3BDTHdvek1remMyOVN2ClhrWHZyaFNjZEJVclZWVWpSSUlSbHFNK3pFVTJRWmxiTWYzWElwdVBieFY4RlNoejVjRzcvQnVVZmNtdFVzaEIKdmR1ZXBkTFJkMll1ZG5qd3p1dE1KS0pLK3RudWhHcDdTRENBTGgwa3JSR1RFS29aM0lsbXM0M3k4QWcvRytjbApyT1V3NnFKanVyRDVvTzdFNGtyZ2w1ZW80emRRTWhIQm12Y0pWNUllMWNNa2NFQysrbzY5N25IWjNuOEpGQnB5CjB1NHM0ekpNdUE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlEbGpDQ0FuNmdBd0lCQWdJVVBxeGkvYjV4eEZBeU5sUkZqVy9PVGV3ZnhrOHdEUVlKS29aSWh2Y05BUUVMCkJRQXdZVEVMTUFrR0ExVUVCaE1DUVZVeEV6QVJCZ05WQkFnTUNsTnZiV1V0VTNSaGRHVXhFekFSQmdOVkJBb00KQ2tkRklFUnBaMmwwWVd3eEtEQW1CZ05WQkFNTUgwUkZNaTR3SUVadmRXNWtZWFJwYjI0Z1FtRnpaU0F0SUZKdgpiM1FnUTBFd0hoY05NakV3TkRJeE1EWTFOREF3V2hjTk16RXdOREU1TURZMU5EQXdXakJwTVFzd0NRWURWUVFHCkV3SkJWVEVUTUJFR0ExVUVDQXdLVTI5dFpTMVRkR0YwWlRFVE1CRUdBMVVFQ2d3S1IwVWdSR2xuYVhSaGJERXcKTUM0R0ExVUVBd3duUkVVeUxqQWdSbTkxYm1SaGRHbHZiaUJDWVhObElDMGdZMlZ5ZEMxdFlXNWhaMlZ5SUVOQgpNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXowV3psQmhYQk12clRva2dFY2x4CnB1dytHZEgwSmY2Tk1DOXJnM1lmeXZOaVhPZVZFTDZpUWRnR1RUUnhpbWNZcklSRWcrb3cwRi94RXZiL2w5RUcKUDhTbW9wQlB3aFF1VWNKWjlZRlhsbis0dWJIRG1BSW1XbktrN0FDTUFqVjJ6N045UjNoRldQbVJ0Zm9JaDlqTgpsZVRYeWhydFJyRW4ydUcrUjBZd1dNRjJ4TC9CU3VIbTd6NUpuZ0o5MVkzcnd1alExS1ZPR3gvVUszRHJOdklOCnE0N0ZYclhpTUZsYjV1aFJKbnk2em8xc2o3SVM5SzBoeXJMUWlGL0xGSTM3WUxHS0Q0Z0JxbG9VZXNtamxBcWQKN0l4dm4yRWpKTU9vMmlOc3p1Z09Gb0E3bStRVlNPbEJLS20waTJVNzVMQ0hJc0JqOGs4U1BXdmNZb3VWd3ZFYwowd0lEQVFBQm96NHdQREFmQmdOVkhTTUVHREFXZ0JRR2phQThnMGl4Uld6Y1dnU2V6d3loVnRLME5EQU1CZ05WCkhSTUVCVEFEQVFIL01Bc0dBMVVkRHdRRUF3SUJCakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBQmNRd2loVFgKTlBpeDVUaWFTcnF6KzF3ZXNrR3B1cnlvU2ZFR3c4ZzBBL0VZOGYrUnd5UloyNUdxdjlFenJHS0VnWmdyT2VpVAovTi9od2tHdXJwM2dBTkk0a1Jsc3ZHRk1hQS9WaEdYcmZ0TmFJTVo3UW41WE1Pczk2Y1pQYVEyT3hHSnk3Z3c5ClQwUDB0cThtY0FNVnNPbEhjZmxBdU01TThrT3NPaVArcDZPL3V5NTV4TkNSZUlGMHZ0N0diSkN2OG9YNE9DNHIKNlJ5ZnNPSmxNVXNEMGExMnBmM0pkWjZHK3QvNkpDMURLVENKZER6UC92OEVHcGtpWEpYV3J0YzVxWDh6ZEdZUwp5Qm1rTFYxQld3QjV5bU9ldlBTTndhTktjN2NQOVo1QjdOMDd6REYxWDlEYkxrWGFCVXV6Uk41Y2ZZekZqOWtjClYvaDhzWEhqZVB2eHlBPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
  tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBcUsrZGxDVjJkYVNZakhzMjduUEVMc1pCL3BLV0k1bzVuSDJKYWFiNHc3TVo0alJVClhMdlBTSnlHNmRqVGtXM0RudSszb2VyYUtCcWE5USsveXc3bzZlYkFHUXJOS2Z6bWorcFZjaWgrL3dKTGhWN1kKSUw4Y204NDhCNHoxcEtYeWlWUzRYem4vR0xwdElpdVBCQ1J2MFo4TkFNU1RCYkdZb1FJdmg0Q0Voa1d0NDBvQgpSR0lkb1lIWTM4WmRGK3ZnbmNqSis4ZDBJQU9MQms0THdLL0RET1o3d0g2dEpWaTlyL29xRTYwQmc2ZzZGU1FzCmNpdHFMblRuakRJOUZ5NmNkWFZhUWdObTZZL3phUVZOZHFYRElxbU4rdkIra0xPekIyakdSbGlFdXp3UjN0RkEKQzZ1RGdBM3BCZktzK2RoMERtbU1ocmxHK1orMEg1ZWNoZ3l6bHdJREFRQUJBb0lCQUJjNWJvbWFDdEF4cENzYgpQN25leUhEa0xPdmV4THNNSGNoMDVEbGVtcnRlNmo1aGxBZmNkN0o0a242WnNxOUFISUhDWVZieW5heU4xUzNPClNFcXR0OVdVUm5oc0swa1BuN0xLd1k3aEVuTUxudGd5OGJDNGZ6allVUjE5QThqeWJBMEtWQVFRV1JYUGZQMEsKWUR0aXhONW9kb0xYamFKb0RDcUd3WUozYUVlUUVSOGY5bDU3THZiQ3VNclJJbVoxb2pSb1NUYytKMVZlZE91bAplbVFabVhIVWh4bEdTVnJXNVF0MURuV25lU1krbmM1Sm1pb3lOTzllbkpsTU53dmxRZWFZamh2M1lYbzdzYW1pCnZzMHlzOU9CdEVZT2RzVXZ6MnBjVnVRTThRUGJOUFBGTFRzM0dyQXhjSm5ublRoNXIzOEs2M2VwMlhPeHMvdHAKSWhleDhJRUNnWUVBeXd5c0d1MmhwaGJLbkVtck8wSkoxNlVpUU5taFVTV3FsWFV0V1I2bG1xa25yNWhhcDRhOQpGdWVMaC9JZ3pmSzRRUFZnVXpQalRCQmZqM3RjTXlWSDRrL0tLM055Um1RL3VzQUd1L3drdWlIVFVaUnpjMU9uClQyTWhpS1F4RWxRejluM1gvNHJzM3BjRVJ3cjdGVnpBNnJDY0xpMDBkQlh6cVZQSGJTV1ZyTmNDZ1lFQTFLemgKR1UrSG1OTGY0TUpzenc2RWlhcE5PMFFLL3loZmEzcE9MVXY4NUlNREQvaXAzT2I2VGJhajNJZFpHL0M2U3NDOApYN3RrTTcrN0xjdE5QM3JKaWNielFZU0FrdGVka1BHNmZkakZDc0dqTTNoUVArRForOHVxWFh2cDJYdkQ1LzkwCnBEZ2FYNWJYS2padHVyeE53WlExTWdRamdUZ3dDYUV4LzBxUGwwRUNnWUVBdDNHZWQ4R2kzbFFPL3I0THhrRHcKUjdRNFN4Q1RqcVdTQk85VlpnT3JLQnlVN3k2ZEVHZEpwRXpTb2FLZUFwSzc1bXdKZDRJVVkxOFRocVN6dXIxKwp3VGJHOXhTWTlJR1RsaHZ3dVp0bWJ3Y3FUTFRxelJHUndjWlV3SWRla2ltaXZKb2JNQk1LUjF0MmtOeVoxVVgwClFOUUlkaXQvRFRYYjZtbjNiV09jWXdjQ2dZQkJwR3FGNldvczNVbUVrZjRMbXF1RjRpNmNVRkFrbVUxNld2R3UKUVhuNC9nYi84dnhhMW1BRGhJL25EVStQdENtTDBSbFRZdjkyaVlNNGl6dDRrR21NUVAvc0JQMjZWUnFwRmpNOQpqSjZreDBlY0oxSjlwT1VqTnFXSzNTZzhuRnJIVEsydjBqRXh0OWtKaW1zcjY5c0VHTDZpaFd5QUFkdXgzVE1WClhPb2RBUUtCZ1FDS0xtOVZWUlJRbHNBeVJiaVN5VHVxS0hGNmJ1aFFGMjl1OW5CdjVPN3hjdE84S05lWjNEbU8KMFpzYVBlbUlsVi9heTlWWmxLai9VYUY4TSt3Q0ljTnNXcENuM0RodGFINWI5UDd2TVVoTG9pa3JqcHFTR1dUZQphOEJDRTNIV0ZCSUlaTVhlKzltalBsTkhScVpOckFEUlBEaHhrZnZNSk90dVp0R3o2R2N2V1E9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
kind: Secret
metadata:
  creationTimestamp: "2022-07-05T13:56:53Z"
  name: mycluster-tls-secret
  namespace: infinispan
  resourceVersion: "11882356"
  uid: d1b41260-60e0-4fce-9729-2e1998bc8bb2
type: Opaque
pruivo commented 2 years ago

double-check that 10.227.48.190:31314 (if the IP is not the same, check the site service public ip) returns the correct certificate.

vladimir22 commented 2 years ago

Hi @pruivo, Yes, you are right, the log message below is related to liveness probe requests ONLY:

Jul 06, 2022 2:11:48 PM org.jgroups.blocks.cs.TcpConnection$Receiver run
WARNING: failed handling message
javax.net.ssl.SSLHandshakeException: Remote host terminated the handshake
        at java.base/sun.security.ssl.SSLSocketImpl.handleEOF(SSLSocketImpl.java:1616)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1434)
        at java.base/sun.security.ssl.SSLSocketImpl.readHandshakeRecord(SSLSocketImpl.java:1336)
        at java.base/sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:450)
        at java.base/sun.security.ssl.SSLSocketImpl.ensureNegotiated(SSLSocketImpl.java:841)
        at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:932)
        at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:252)
        at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:271)
        at java.base/java.io.DataInputStream.readInt(DataInputStream.java:392)
        at org.jgroups.blocks.cs.TcpConnection$Receiver.run(TcpConnection.java:305)
        at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.io.EOFException: SSL peer shut down incorrectly
        at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:483)
        at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
        at java.base/sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:160)
        at java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:111)
        at java.base/sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1426)
        ... 9 more

It would be nice to find a solution to suppress that message because now it is really confusing and litters container logs (probably by updating /opt/gossiprouter/bin/launch.sh script)

Finally, I was able to setup "xsite TLS encryption" by using keystore and trustore generated by cert-manager (details below), and probably, it would be nice to consider a case when infinispan-operator can re-use secret generated by cert-manager without creating intermediate duplicated secret.

Thanks for your help, You can close this issue and create new one to suppress warning message Caused by: java.io.EOFException: SSL peer shut down incorrectly inside the GossipRouter POD because of liveness probe requests.

The details of my steps:

## Create Certificate resource that generates  'infinispan-dg-keystore' secret with keystore and trustsore inside
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: infinispan-dg-keystore
  namespace: infinispan
spec:
  dnsNames:
  - infinispan-dg
  - infinispan-dg.infinispan.svc
  - infinispan-dg.infinispan.svc.cluster.local
  duration: 2160h0m0s
  issuerRef:
    kind: ClusterIssuer
    name: cert-manager-ca-issuer
  keystores:
    pkcs12:
      create: true
      passwordSecretRef:
        key: password-key
        name: infinispan-dg-keystore-creds
  secretName: infinispan-dg-keystore

## View generated 'infinispan-dg-keystore' secret content
kubectl get secret -n infinispan infinispan-dg-keystore -o yaml

apiVersion: v1
data:
  ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsakNDQW42Z0F3SUJBZ0lVUHF4aS9iNXh4RkF5TmxSRmpXL09UZXdmeGs4d0RRWUpLb1pJaHZjTkFRRUwKQlFBd1lURUxNQWtHQTFVRUJoTUNRVlV4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeEV6QVJCZ05WQkFvTQpDa2RGSUVScFoybDBZV3d4S0RBbUJnTlZCQU1NSDBSRk1pNHdJRVp2ZFc1a1lYUnBiMjRnUW1GelpTQXRJRkp2CmIzUWdRMEV3SGhjTk1qRXdOREl4TURZMU5EQXdXaGNOTXpFd05ERTVNRFkxTkRBd1dqQnBNUXN3Q1FZRFZRUUcKRXdKQlZURVRNQkVHQTFVRUNBd0tVMjl0WlMxVGRHRjBaVEVUTUJFR0ExVUVDZ3dLUjBVZ1JHbG5hWFJoYkRFdwpNQzRHQTFVRUF3d25SRVV5TGpBZ1JtOTFibVJoZEdsdmJpQkNZWE5sSUMwZ1kyVnlkQzF0WVc1aFoyVnlJRU5CCk1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBejBXemxCaFhCTXZyVG9rZ0VjbHgKcHV3K0dkSDBKZjZOTUM5cmczWWZ5dk5pWE9lVkVMNmlRZGdHVFRSeGltY1lySVJFZytvdzBGL3hFdmIvbDlFRwpQOFNtb3BCUHdoUXVVY0paOVlGWGxuKzR1YkhEbUFJbVduS2s3QUNNQWpWMno3TjlSM2hGV1BtUnRmb0loOWpOCmxlVFh5aHJ0UnJFbjJ1RytSMFl3V01GMnhML0JTdUhtN3o1Sm5nSjkxWTNyd3VqUTFLVk9HeC9VSzNEck52SU4KcTQ3RlhyWGlNRmxiNXVoUkpueTZ6bzFzajdJUzlLMGh5ckxRaUYvTEZJMzdZTEdLRDRnQnFsb1Vlc21qbEFxZAo3SXh2bjJFakpNT28yaU5zenVnT0ZvQTdtK1FWU09sQktLbTBpMlU3NUxDSElzQmo4azhTUFd2Y1lvdVZ3dkVjCjB3SURBUUFCb3o0d1BEQWZCZ05WSFNNRUdEQVdnQlFHamFBOGcwaXhSV3pjV2dTZXp3eWhWdEswTkRBTUJnTlYKSFJNRUJUQURBUUgvTUFzR0ExVWREd1FFQXdJQkJqQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFCY1F3aWhUWApOUGl4NVRpYVNycXorMXdlc2tHcHVyeW9TZkVHdzhnMEEvRVk4ZitSd3lSWjI1R3F2OUV6ckdLRWdaZ3JPZWlUCi9OL2h3a0d1cnAzZ0FOSTRrUmxzdkdGTWFBL1ZoR1hyZnROYUlNWjdRbjVYTU9zOTZjWlBhUTJPeEdKeTdndzkKVDBQMHRxOG1jQU1Wc09sSGNmbEF1TTVNOGtPc09pUCtwNk8vdXk1NXhOQ1JlSUYwdnQ3R2JKQ3Y4b1g0T0M0cgo2Unlmc09KbE1Vc0QwYTEycGYzSmRaNkcrdC82SkMxREtUQ0pkRHpQL3Y4RUdwa2lYSlhXcnRjNXFYOHpkR1lTCnlCbWtMVjFCV3dCNXltT2V2UFNOd2FOS2M3Y1A5WjVCN04wN3pERjFYOURiTGtYYUJVdXpSTjVjZll6Rmo5a2MKVi9oOHNYSGplUHZ4eUE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
  keystore.p12: MIIRSwIBAzCCERcGCSqGSIb3DQEHAaCCEQgEghEEMIIRADCCC7cGCSqGSIb3DQEHBqCCC6gwggukAgEAMIILnQYJKoZIhvcNAQcBMBwGCiqGSIb3DQEMAQYwDgQI1TVfTSUqgoUCAggAgIILcBPlat0+C/NbGfAczNwjLQBtv3xsA7v6bLsPYRYQGjPmgEPQ5TUsx8zS4aiEgEUOc+Mfc3gO4z4A6KDuFeaPsgg/8P461rlb/13uPetLDKuWiAAPVBvjNTJm4nMjz1c0FKn6THxgPy8PuPF1XDsvyyO94PVHpv5BPFkTVdr6ww4tHXfEHcimQ/USQs5VqXTr1tQEkFmvwxnypG7C/0XJxluURdZhX3JgituYbRykYbBcVIvn4LQ1egxvXQlIEPxR+7JuH18fttVbET5S6CBRCBgsrl7e9b2noBeTZbiMSQSGh3fqK0Od1ZMqrvT3mVBGI1qv1anawHChzRL+CGwuvoJzkqSak8/eCfWy1dbE3PKyKtb3UdYiYDbeEKRmGCQ8QbcPLS+iIkO1gUPu0QiSRZAUuC4dP5+/VU71beWvtKvJq4kps5dcr6+shOp29bIYrmboLY9NMUC6lHElUOA9s5jhytSkR88J+N7nErHfxziRx+VQTTDV6YgXYyHZ0pbnyojiiIf/sBEmKYOp4hpTkU3dIor0tLHvUCJXYZ8rx3Boc263A/6FNHFhUEUg+xm1tAze2dKWf5hP58SLIciO1nv3NDy4Zo3YXBg9oFskaCk9on0fMLN3AebYogLjRu/tXVm9SD9ngPwuwpIfN/0umDrKDDxqIm6grqRJb0euGzStRn518bRrhPeXchBn23BL2fmUO4Av5oV1OzAuCWHP+SnqEHPtL6zpI8EIK4gVxsoVhMk5Bkp1kq18XunCYq97eFBsBnF4+r5CqoExn+ebgsvvaAz5bL0Z6b5yqRAMjH2jTlrw5r5TNVfAYUTEL4UBYYK5R//R9hzlzvKEXKMh201Zo9cj5ZVQlA1RRpVq4oXBQcoTn5OGbFwT/EttI+Wm0yjJcjnnX9oWyOnIUsMWnvoNt08rOJV+tXpgGn9pKRU8YCHSkxdf8ujJ7eoZXfuxYIlpujKX/Mmn9QZUShK9TuJGZZDmCJ+gPxpVK5D9U7zN88PLzuDRW8N6z/8BiKFWrPCdqkFFmxHX5x7hvgYKDj9M24bH54TX8C4J4nU3E5K1mNhgCKLv91IARsALsa8bCviHqJl67AlcOB5JVV/s/PMecM3zUIpx6r5Dx360dIzs1A+3O3chNMMBGvuokWW1T3xYLy+X2ceRswrO1U70BluCbD8ulRIah6M8TizfJwupOLrByKjZ31l0INNEu7ixfstiw/IiCBUfEohbw5I1xbUzRUI6IXgCKcpPO5cHmCxhRy5tpf7/ogrU4JMymLEhej0h9KvCji1tFTrSxehCQ+Fg2g6/zApRGJoDN2UyVxmPrCxkLB9Vb4T15R7zTebOFx0V7dDXBoO5a/jtA3xL5HKxFLZ3dhJns6+QCdgO8mAdej3nCk4fnQD+gtXxceg9v98bwqDtj9Aj3khYa6xuQgvo21qDZ3kF9pSIhs+zvHIijjY0ZVngr0xxGtXf0hJlVcOFbPDSZztuI1FFEKftjP7ka2/159tP1oYh3a7IPLDIVi10RV8omtQR/LoP184M4HN5WGXR95Fn5XqoEZphtB3ah0tC4D7PVXfgc83h+TVzhyjSsvVfm1sbxH0cVjqBFZDxhG9/k6NgHYXJ2Z2zLnxI4KUC8SJjgShIBRqUujT7lKJuylkJQajXH1Svch2UpH7/5YS13sVXm1T6FbWyLeL+z1BrXJ/3GOKbHaEoYK/WCOCD731yShKIwSZ2UjAzu2XA4dV/Vdnx6rE88u7SVcFXHQC6icvvpew0/BjUTT9/yOWpi7JaMLQE2+HGxMNDMXG666KgyQxtmDxUQB0XsgutWXlilQq3iKFHPA78CKRj5BChb03LCYWoDEUlQvy3rlJ53t+DCVjpMm4jX+r66SOHb68+ZN0sjpkGh8H9cbPOETowRkig4ddn41DCvjwvY2ZMf+S5cqy+l6RVYihchhkgOoaCTiCLX61zmu9Xb+UO/gcEk0FfXpY8OC2Fj3FoWL3CrR78MfSjFyWV8nYC4cjMFBY42JeQfFrd70t91Okl+/O2JCt1YxnUZWG35GdU+mY856RRSKTgycIWfcb2CHzCd1QLGkn9OQ4M/ZgkMJgFfsEfw87JzWBEEQLs0bfUCVgTVfHoNGQunip2lLlyxdEE1tQUdUy5VATg9VaEQRSRVZ2t1RwlrCHmxrUqMgFVn/oyKhRLNnInbmp3TzjhnN3bdJ5Cxm7rhlDNGtULZOPTSA4ZPjk/cUpNTuJmBoQnhn+L49G0Rb/J1evhbjQbKqGqITTLYI4UmeU/xqNu7VNiJqB6oY+ABs8J8CteO0CuW+Nze7yPTo024vr+v28aknmY0DdNj6k+ZYXKs78Zm8Qy+weatoaBjhbLpF6zO7SenTN7XMRKUewUH1Lqtv+h+Pr9HLOaGw91sw1ko06iQnpalA2Ds+7CZWVFgDT7+wHSRQKgNxJlFY6ojw8oer/LhXnp9HklE2wYkA4BvBpJl/ogVTz5lswQV9tq9mNRs6y62TM9VDur5Zfypon3IbNQBQnfBqy/4lxpoq1stXWp7toAcHmTR/DstQD2teT//ebcH3Bn2Y8KtfW7Z8rT3dXmS+zlHok6vWinsn/b9lvbDVUuejTSXR9ncrVvtmh1xTUXQBm0v6uLkBmV9NRSRYaOcqGrENCtRfJerDwgr/jruz8CvKfVjjWJbO+YHk3SFPnzdph+0Z/1RedocsehmmbpEKe9uZbbyHb+AB0NtAtcl0chBZPpRtygEvfImQ37aPjgfKRSrORRyt8Hq+Ojnb7RlLfvGfwkRdHSIRgEXVk+OJ1XE7sjuAgPjiaypOrSCPQak480ToP+HLjkHpD4UnqTtwg4s5sl1sY49XTZ1FAqoSzlEmQjRHdfRIFiADI+JowYJkeQVeFmTP0YM9RX2IkYaJgF+jgTQPto1SPUoDO0j3HRVpuxt5BH6gf8qworutBvkEBjZS1vhuKBFt6ibHmf7HfNOdC4bC13ulHs5sFHpY5EzRORUe2WBr7qGs50zE3d5VvypIpmA5ghSS/HYpHMxquL3YeQpT8gbkBRg543oCV4MQcdXO42OTfrq+0KoklMujBe2MlF74NknOpyzTo8yI4SzYJPB4csTISfrUuV1QLEDYoJlFi/d/qA5fEppXoJr+d8dB8efVQQPSsliIXv+e2Gap661CKuVpFBj0a7nWDRMqNcoEohpmDCuXog++eUb/yXuIdycLLBmTmTjdjGBCWyDxPO2AQh7cNbjzwo4w2PIVAiesk95RTenXC0J03bpTg1kLC/Z0XAu1OhhI3fTNBKV721gbN3MtbEH+SE9IuljvT+aTT08kReIpJ1uox4l8CefKAptiW6x3fnNLbh4FJ49iHTcRN+fYaeXCDPTWy1XYYLHC6yg5t5NXebUmjIjk4g/83RnNxXZ483lBHkxLb0N6ny7nGj3ATkaN1POPAEOEvowjzV+gpnF65V3I5107TFH6pgyC0qe5pNQcy4+DS+AXz5iB8W8wqNOzxncMmr+aUrUTjV/6tB/emGDMwTJLibWugOa+d7krEbXdOOM+5T6IRIyKRFFcUeijFlXAYXGPUDjoI7fsKiUQS6mAXkUG+ygFnLIBqgVcxJDhHHU2qezmcI5/PnSE7J5jUzdV80msselpn7nffSXXQucqz8RYvBuZiFtHgeo1+Tbkmo6ZMtp/Jyd868S18L7WBZbuSuGZCTGQrv7dEYgi8snwDeItp4iX0WWorylfD/8ztodZ+WjcQfetfwYY09vWSG32AFeLvDdBiusMCIdO5QE2+1Zn6lfrTSP/JrRre8TerVtUF3G4EaqhmKKEVXYgdSJhoVH+3fhBYIxgl1t75UM/mm5XPNbqLkJvHMb4+j8sE7wAdO7bENhTiinxETwJPK1lFdtPTU4sRIB95VzvtJODfAzTCCBUEGCSqGSIb3DQEHAaCCBTIEggUuMIIFKjCCBSYGCyqGSIb3DQEMCgECoIIE7jCCBOowHAYKKoZIhvcNAQwBAzAOBAiqh1qYeqgKTwICCAAEggTIZ8BFtSl0KJvXYUUddHfPxmryzjCTjynbMMd0q/fGuHakBnYV8lq93dAwoPuvx0Tr9CAE7OulknJq7Kk868R3kc9C3MweRqepIBlguZTb4v20o/zmYgSb0Btc6IMSqrng4nU+IZFj4R7liPyf3NWs7eZbkgU/ylonCvEb9ML6PtzQ4ScjpwftrhS/Xy0DPWyVYcquGktRRHajQ5uil0LSwyvAMDmqnE9rpJ3CsHVfv6A9kORdq8pRbWeDmVCrcjEnSnBgRm7Ofh6HiDtWGU5KibgMI53VWBxtsMufbr8EVdrIgZju2CuCoIckvyDvlnITDmPw2WPTkE5ixc8JHrCPq3lJkJA5ORoeXrJxYAv1Jlm1TakuwAZ24vFBoeFmMywKY6iMLfVlZbEDyOOxOQu68GEch+sM6lxERdA0rwegKcP4m5Jr2T+qr9tR1jaH5dGOrHaGDGzdYMt6AMw4KfdiM0P2PndVDZAmaezYwMMdc7w1LYpnkWMM6tVq/NTVJLx2HtkEdfBZ9js9emAPXZA5f7EStoetK9Z9WnV1ZOVoIH3kRreuYbHfXvOYN5VSpM2JmD96dbI2wDaxac/OwWNMePATWgXigXmzCiU1ksNCZwXAsrc/BuWgTSFXWmXWzmBhWcl58sBqd/pHFgZK+cmNNFnvHLM1ZH5F/ZGa/X5lN7bZzlviJt8Ps1iZixzVKOkZg3GIzsiulJ+aEKeRgu9MGpITjaq3ScqEDuzY66NFRw4hzYvlLPz0UNSuAO5HSuRf9hHoiRglKXU1lP18JtaglT4NECb+Z3Btau+W2D09gmyxfB/n8K69cSXAb3p1pgJNgH007LAQn1VJf6Y5d/X3FXerkCn2oWAq24TV9SUD6RzgYysocyFm6eN92zX2HjBsqb9qn1gnF3KBgVy/C3LWG6qbE49AoLepvTRFgD/XlBM80civO8p8E8YX3CbBZsVpnjulXqt3rN7tTnHDWiehgnVkxpmtl+kKLAJStS3DOtffpBRHHDWuVJvjEsbSSfeX1lq7VnAhj8cD1/0patqmo2iWVqko5LJzdqlsdw0fy+CfqgTeoSX1MPG/avHBEwPcIH/ykA7pMXBO8nxCkuncvPrbLs5Mjy1Q2apudE7HR1xu6bkXDYkhceWos2fTw3dBXqeXYfulSfoJ/2/bh70Z6R+OE+pLg9inFDnP0GP3grifilnAqi/D2MOkmoxoZqqugr+O/XzkW6LPD5903aFOHelu4kBvsffxBTcEqDFYHTLjiEpK19tkj/pOknLZECrK9BdzPPG6CeYW4maZEyrGJOhmd2ACwz8Ke0pZHyHB0JyFj9/JhSR+BEwziioSjEgh0aKz5q+VgLM3v1QRn+InSpajtOWdX5stJKdkDVKbOsa7qmSw+3EeT4vX59/RFTqm4wSmnauQF3xunB+oX09DOSVdwOuVqKq6HoN/1+R7PqMaZFhmWYUD/r1fZrDrHponXHdj8BFFdxNJXddKU2QXZ5PPG7PdD20loReWHkEHG1RKXzYKtfFe95EN/tmnsJw+P8IsSLT/QERmg8KDMcyNG3nvd6T1t9FlHSv3Z0Su9R9mWX8TKDPHlaW3TVO0ofM7h/3VtE2qLZGmZbX69fsQigXZhIrRyhV1MSUwIwYJKoZIhvcNAQkVMRYEFFvO6+rFxexfXrERjC0T9jpgOXSlMCswHzAHBgUrDgMCGgQUc6YyUBdE0No+IN0KH8+UB/n/o4YECMZH5OAkXBjx
  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURmVENDQW1XZ0F3SUJBZ0lSQVBiUCtUbXROWStvOGVjaGdmcTlLOG93RFFZSktvWklodmNOQVFFTEJRQXcKYVRFTE1Ba0dBMVVFQmhNQ1FWVXhFekFSQmdOVkJBZ01DbE52YldVdFUzUmhkR1V4RXpBUkJnTlZCQW9NQ2tkRgpJRVJwWjJsMFlXd3hNREF1QmdOVkJBTU1KMFJGTWk0d0lFWnZkVzVrWVhScGIyNGdRbUZ6WlNBdElHTmxjblF0CmJXRnVZV2RsY2lCRFFUQWVGdzB5TWpBM01EWXhNVE15TWpWYUZ3MHlNakV3TURReE1UTXlNalZhTUFBd2dnRWkKTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFDa3lxU09mWkRaT2dhS2lreGQ2UFZSMTRjZQpwU2xFOUpiU3FCRSt6SVMwanliYWg5dkw5MjIyM3dPVUdVTkpnUTAxV05HUlNrWkoxRjU5MUR3U1RhWmpSem54CnlxSFlkanlVMmFYaU5zbGJzblJMMkhONlNEcXdodSsyVk5MdWZQRk12dWovYUo1NlBpUkI0LzhoM3V0V0J5Y0kKTFVGN01ZQ25aU0pxR2xtWjAwUms2elM4clVhbTFSUWExRlZwRmFMTVMvUC84R25ZTXZCa3NFZHFjWFBOMGhlWApxamVjQldiN2ZpeFZRYkczU2VSNmZRVzB0aDlWd0E2Mk9OZGpRRHVwWWhObUJMZlVsN2FYUUkzWStrNWYvaTVPClVGRVA5UzhWdHB4UEprVDEzVktqWmlmNER3WlVhY0NVQlk3TEt5R1FISDZ4b0NSaDZOWGdXUXJ5WXEzL0FnTUIKQUFHamdZZ3dnWVV3RGdZRFZSMFBBUUgvQkFRREFnV2dNQXdHQTFVZEV3RUIvd1FDTUFBd1pRWURWUjBSQVFILwpCRnN3V1lJTmFXNW1hVzVwYzNCaGJpMWtaNEljYVc1bWFXNXBjM0JoYmkxa1p5NXBibVpwYm1semNHRnVMbk4yClk0SXFhVzVtYVc1cGMzQmhiaTFrWnk1cGJtWnBibWx6Y0dGdUxuTjJZeTVqYkhWemRHVnlMbXh2WTJGc01BMEcKQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUI4SVYxSkIvbTAzU2kzNW9pZGRLMGp6TzFDaHFoK3R6cWJ3WFBBY2tRKwpqZFBsb1VmNFIraVcyQ2t6REF2OVBJR1Zqa2tha2N1Ky93WWMxUkpqSVVXVnR6YVVzcFNCRzBuM0JrNXh1enh1CjdtVGpMK05vOU9ZNzQ4c2F2Y2pKUy9XajNDMW16NlRFOTZaNmRqWHR1TXUrTmt3YXZIYlBhaElaVmNHM3NpR3gKTzhVVHlqUjI4NHo0S0xQL3dqZFphazRxZzBkR0dCdHNVVHZOUWhadmZsY0RkZjlSQVFrV0lXM05QaDF4cDJHTgo0QVkyVDVDY1c3bm9oUWRENWJuUy9HUTNiWHRxM04wZFNQZGRST21vNE56bGNxME85NEYxZC9hRDlFdFVUNnNGClhIS3BZZ0lRZXM3aUU1ZmtvbENkQmhJOFg5MDh0bFpoaTlrZXNjUVRBcWttCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURsakNDQW42Z0F3SUJBZ0lVUHF4aS9iNXh4RkF5TmxSRmpXL09UZXdmeGs4d0RRWUpLb1pJaHZjTkFRRUwKQlFBd1lURUxNQWtHQTFVRUJoTUNRVlV4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeEV6QVJCZ05WQkFvTQpDa2RGSUVScFoybDBZV3d4S0RBbUJnTlZCQU1NSDBSRk1pNHdJRVp2ZFc1a1lYUnBiMjRnUW1GelpTQXRJRkp2CmIzUWdRMEV3SGhjTk1qRXdOREl4TURZMU5EQXdXaGNOTXpFd05ERTVNRFkxTkRBd1dqQnBNUXN3Q1FZRFZRUUcKRXdKQlZURVRNQkVHQTFVRUNBd0tVMjl0WlMxVGRHRjBaVEVUTUJFR0ExVUVDZ3dLUjBVZ1JHbG5hWFJoYkRFdwpNQzRHQTFVRUF3d25SRVV5TGpBZ1JtOTFibVJoZEdsdmJpQkNZWE5sSUMwZ1kyVnlkQzF0WVc1aFoyVnlJRU5CCk1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBejBXemxCaFhCTXZyVG9rZ0VjbHgKcHV3K0dkSDBKZjZOTUM5cmczWWZ5dk5pWE9lVkVMNmlRZGdHVFRSeGltY1lySVJFZytvdzBGL3hFdmIvbDlFRwpQOFNtb3BCUHdoUXVVY0paOVlGWGxuKzR1YkhEbUFJbVduS2s3QUNNQWpWMno3TjlSM2hGV1BtUnRmb0loOWpOCmxlVFh5aHJ0UnJFbjJ1RytSMFl3V01GMnhML0JTdUhtN3o1Sm5nSjkxWTNyd3VqUTFLVk9HeC9VSzNEck52SU4KcTQ3RlhyWGlNRmxiNXVoUkpueTZ6bzFzajdJUzlLMGh5ckxRaUYvTEZJMzdZTEdLRDRnQnFsb1Vlc21qbEFxZAo3SXh2bjJFakpNT28yaU5zenVnT0ZvQTdtK1FWU09sQktLbTBpMlU3NUxDSElzQmo4azhTUFd2Y1lvdVZ3dkVjCjB3SURBUUFCb3o0d1BEQWZCZ05WSFNNRUdEQVdnQlFHamFBOGcwaXhSV3pjV2dTZXp3eWhWdEswTkRBTUJnTlYKSFJNRUJUQURBUUgvTUFzR0ExVWREd1FFQXdJQkJqQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFCY1F3aWhUWApOUGl4NVRpYVNycXorMXdlc2tHcHVyeW9TZkVHdzhnMEEvRVk4ZitSd3lSWjI1R3F2OUV6ckdLRWdaZ3JPZWlUCi9OL2h3a0d1cnAzZ0FOSTRrUmxzdkdGTWFBL1ZoR1hyZnROYUlNWjdRbjVYTU9zOTZjWlBhUTJPeEdKeTdndzkKVDBQMHRxOG1jQU1Wc09sSGNmbEF1TTVNOGtPc09pUCtwNk8vdXk1NXhOQ1JlSUYwdnQ3R2JKQ3Y4b1g0T0M0cgo2Unlmc09KbE1Vc0QwYTEycGYzSmRaNkcrdC82SkMxREtUQ0pkRHpQL3Y4RUdwa2lYSlhXcnRjNXFYOHpkR1lTCnlCbWtMVjFCV3dCNXltT2V2UFNOd2FOS2M3Y1A5WjVCN04wN3pERjFYOURiTGtYYUJVdXpSTjVjZll6Rmo5a2MKVi9oOHNYSGplUHZ4eUE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
  tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBcE1xa2puMlEyVG9HaW9wTVhlajFVZGVISHFVcFJQU1cwcWdSUHN5RXRJOG0yb2ZiCnkvZHR0dDhEbEJsRFNZRU5OVmpSa1VwR1NkUmVmZFE4RWsybVkwYzU4Y3FoMkhZOGxObWw0amJKVzdKMFM5aHoKZWtnNnNJYnZ0bFRTN256eFRMN28vMmllZWo0a1FlUC9JZDdyVmdjbkNDMUJlekdBcDJVaWFocFptZE5FWk9zMAp2SzFHcHRVVUd0UlZhUldpekV2ei8vQnAyREx3WkxCSGFuRnp6ZElYbDZvM25BVm0rMzRzVlVHeHQwbmtlbjBGCnRMWWZWY0FPdGpqWFkwQTdxV0lUWmdTMzFKZTJsMENOMlBwT1gvNHVUbEJSRC9VdkZiYWNUeVpFOWQxU28yWW4KK0E4R1ZHbkFsQVdPeXlzaGtCeCtzYUFrWWVqVjRGa0s4bUt0L3dJREFRQUJBb0lCQUFZRzkxL1N6ZUowRWRzdApWYisxam90d3c1dG1WUTBWbVJLWlhSQWI4TnFkeGNFR2FRNmJ2U0tNSTVoZUdDenZFR21jRndZbVhrM0MrN1hFCmRuSUE3dzRBZytrSkRVUmFocnEyL3k4MzY4TnowbmNYc3BtTlhNQXFIMGhlNzl3VEs4d3Z5MXZVR3IvRDJCV2sKM2ljQi9UK0hRWEUySG9mLzlvR28zRXBmTGxuRzVsbVdoNWF2ajFJOUM3T3dETkJsQjBzeHRGT21tZ09NaUJuZgorZ2tzVC9YTkZ2V0FhNUlJOWZramNuYmFIemx3THBqd2JFVWtmMUpwb3B0MUZZY2hWVnpvWXhtclZyc3JTT2pZCmJ3NnBiYXJ4V2dnWDIwcnRoTmFwK2JFVTJIZVhpclhIZjNQVnVBblFENkZzRnhmbFIwN0lUZ2tGSTlMc2ZDbmEKSWZJVEZla0NnWUVBMFFJaDZRVVJldmhIdnc1RHNwOW5ZUTJSektTc1RUS1JTdHN5N1FNVGZqWWxZMHFXbWpkVwpqM1VBR2orMXI0c3A0MGN6VTNlM0pZdW4yM2MxZDNxdjU2b3pnSlpFMlpOc2hRT0d5NnBqNE9jSjNWWDhGRVZXCi81R1hsZzVLcUVSdFZCQjRqSTBEMm1GMTFZNXNQYTNYNWpUdkdmU1BKY2dPOHIxOG1GNmZ5Wk1DZ1lFQXlkZUkKSVBsaHNoSkRwWCtDS2c1ZXVWWmlNNW5BZG82QkhDWU01dS9TQ1hINEJpZFY3aFhxSFhnSGtKeEVIN0k0L2Z1Lwo1dFNSL1c0azZFTTJHemFLNzNqYW9MQ0VWblVTYUZvRVVMLy9ZSUtRZGZ0d0xZckZnMVd1MktURDgxNDlPMnB0CnZoZTRvUEg3SWhSQ3BaaFJ0d1kwck9hWHl3SjdkM0lUUWZZRG5XVUNnWUFFdnRlbVFzY3kwNUJOZGpWelNNdEEKR043R2xyYVN5aU1pWC9lNENlblhZOE5WamN2OWxHUVN2Y1ZTMHo5OHpOd1duUGM5WGVwNm44RWY2b1Q4UVh6Ywo5YkJWcXhQRkhPUEo0VnFlSlRBL1d2K09WdlZ5UUdESlRMVDBZejZ1czE4SUdaZWpHVkNORk1sb0grTzFRczAwCkhOK1E1MHlZdndJWDROV0RZQ0tnb1FLQmdIVEpZTnF6Y1BPMzNheEV1dXZhcExSWVp6bTBITWg4YS9mOWpEbVQKU2RSdFBUVlNUVVlWS0FFbEVqdDZMM0NuanBKbXVBSzN2RUVIR2hQSnorU3ZDblRGV0xyaFo3RWV6Q2ZVN2VwTApMVmlLTTB3UTBEZUpzLzdwOU9FbGRMdThLM0hWZ3VvYWo1bVFhZDZ1R21IMllGR21KWlJsV0M3K29MSGNMNGhhCjhIeUJBb0dBSEZoNERNOVpsYjNFcC9HUzZSSGNVOFE4OGRyMFBIYU1KeGlJWERZWXFDVTVacFlJWEJZTnVwWmEKRTJ1cTlBSVlUZXU1eTJ4RGZYRUNZUjJPRW9nWDlLRnRRbVlhL2xaSkhHYjViUW1JMnkzbVpVU3VGQzhNbVJjOAp3S2RQVFFWdjhZQzRBUituV0M1QlBKbVFqUzVFTDNDckVpOVpTTDZNa1V1L1hQdlA5MEk9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
  truststore.p12: MIIFJgIBAzCCBPIGCSqGSIb3DQEHAaCCBOMEggTfMIIE2zCCBNcGCSqGSIb3DQEHBqCCBMgwggTEAgEAMIIEvQYJKoZIhvcNAQcBMBwGCiqGSIb3DQEMAQYwDgQIE3xWtU9R2doCAggAgIIEkMLUbzoQnfbwqILhK7torJqZyZUBd5Ct322vrefQ8ncrEFD7grQ/ojkb3T+Cq7vCJshBvk3QvrAY70XVsWm/M6m3ZAsyeQaTQMwrTiHoqVuN3rFUMHlkmVjlvjiKt17LH9nkQrUq4hC+H+Lj61hNVLh8BnEl0GKH4AV0Y4F5BQdKjXnPlu5KpqTRz7/wEsLJuVDp2cE8kPMEz1ZpMJfbWqnCD0/tRlqXx/7m+1/SdI6swFPjBjO60EBkHQQKbFdT1l0tliIxU3jul3LzqjZQQlPeCLWfVHDMqs2U3c0sCbqC39wK+3U14tN3Ik3GB+0fk+a4L/gNnurDj1BdiyzI1i2c8wRqO1KP21fqKNO3IqNJc6+NH9wAw8/l0dVU/unIALOJbVeRoQYhU22J7IKJgGNybeI+w0Igp5kjE1YurMOXGZFojvMkuAALLdBMjCZRTJb746dzh00CsSXUiR85aFwXM7TZzYSmdU3yU1natkprdXvsUhOp6ZDu0E+nLBlUZ0RBkFziHN75OTy9tBZWwTsWrzMel4HQ5q5hkgVKShPYQ6HY8YnN40ruztWOsBXdecjf5CAyS5Ata5HnQXl1ZodnMPQa7IFnt+mzXvy1/JyQQXVZsMqkZ2TGL7KMX+gpGoCK7RrTEqqIfoFUSXuXLSB0tDizUU60+95iwH5jvtjdrLzp/ya+UKWl3IN3V3b0/brkuTjyTkCOfzdXWNAjCCQYEjW1+iWVWu9WejQFO0cBuoKROmxe4HS1q7CKJZuT3Vp4xI1y+/+a2OkO38lRuJAb0boeqK/sYB8l2GWiJgzjB6GCWkIleT7quQZu/sZA3qt4Vb0p8lQ84kLMfPBHUKQGOOqeJDD2MkUZG7gaEx93mdKwB1UNabitFrbk2gvQTVL7x//kqJc7e0mQXyfZfzD9AVvTRhrsZ36qMSKGIwIF1NMksy7KmJz56hUD/0s+nNveNmxKFmHIfij5/P5nCBNhcdbwACcGNE05DL2ZvD4FFR6NrTtT6OeUyEyjHojFCCjuPtTU5L05WsO6bRJFmoBy9xjBqe32irBJSgLXhboBmBvpTj/IgPjnUuOOP+O9aD5nuahG8jKaU8mynxe3ULl+9W2snXXEA6/Wqr8juDFFT0YiF7iEO7vu3+Ns08BhkkSAlX4fAHqT9gmaVE16zr0lqnZUJTLrdDenZUZUTf+jbcAseKNj2GZ2PDW5nNWkZY7NY/Ihq2dmCwPL91EObmfLVOxWOKzXKzfWBfWiM+45Qw2TwEcRVlFtVQqAwFa924ZLKFhJ72MFbLAZlMwZ6uuLeExOTmtZ9PZaNAs4rwQkIJAG2Lb9ABHYEODJ4eItR78s3HZ0E622Z+1BLNw3zqcX1+/rclDV8lDGipHsmThHrDzZSt1/FFgZiQELnyCb5O8kszxDI7Jx89QtloNq5Jqc6Eas4rpFcUTB1mdBhKSvfaVEoCOyk7Vgfd92Dbg9/Bl+QaQiKZgxDJvv1kWsIYEqkj9N3Y7lho1q5ztNrtUi8dli5xzLqYHeBgVyOHYh2jlAFyirtCPM4b/Q+F9/XTgwKzAfMAcGBSsOAwIaBBQXOX5MywJ6m8FXUyOUiskDGynLYAQIb64QqbJyYXU=
kind: Secret
metadata:
  name: infinispan-dg-keystore
  namespace: infinispan
type: kubernetes.io/tls

## Create duplicated  'infinispan-dg-keystore-template' secret with format required by infinispan-operator
kubectl get secret -n infinispan infinispan-dg-keystore-template -o yaml
apiVersion: v1
data:
  keystore.p12: MIIRSwIBAzCCERcGCSqGSIb3DQEHAaCCEQgEghEEMIIRADCCC7cGCSqGSIb3DQEHBqCCC6gwggukAgEAMIILnQYJKoZIhvcNAQcBMBwGCiqGSIb3DQEMAQYwDgQI1TVfTSUqgoUCAggAgIILcBPlat0+C/NbGfAczNwjLQBtv3xsA7v6bLsPYRYQGjPmgEPQ5TUsx8zS4aiEgEUOc+Mfc3gO4z4A6KDuFeaPsgg/8P461rlb/13uPetLDKuWiAAPVBvjNTJm4nMjz1c0FKn6THxgPy8PuPF1XDsvyyO94PVHpv5BPFkTVdr6ww4tHXfEHcimQ/USQs5VqXTr1tQEkFmvwxnypG7C/0XJxluURdZhX3JgituYbRykYbBcVIvn4LQ1egxvXQlIEPxR+7JuH18fttVbET5S6CBRCBgsrl7e9b2noBeTZbiMSQSGh3fqK0Od1ZMqrvT3mVBGI1qv1anawHChzRL+CGwuvoJzkqSak8/eCfWy1dbE3PKyKtb3UdYiYDbeEKRmGCQ8QbcPLS+iIkO1gUPu0QiSRZAUuC4dP5+/VU71beWvtKvJq4kps5dcr6+shOp29bIYrmboLY9NMUC6lHElUOA9s5jhytSkR88J+N7nErHfxziRx+VQTTDV6YgXYyHZ0pbnyojiiIf/sBEmKYOp4hpTkU3dIor0tLHvUCJXYZ8rx3Boc263A/6FNHFhUEUg+xm1tAze2dKWf5hP58SLIciO1nv3NDy4Zo3YXBg9oFskaCk9on0fMLN3AebYogLjRu/tXVm9SD9ngPwuwpIfN/0umDrKDDxqIm6grqRJb0euGzStRn518bRrhPeXchBn23BL2fmUO4Av5oV1OzAuCWHP+SnqEHPtL6zpI8EIK4gVxsoVhMk5Bkp1kq18XunCYq97eFBsBnF4+r5CqoExn+ebgsvvaAz5bL0Z6b5yqRAMjH2jTlrw5r5TNVfAYUTEL4UBYYK5R//R9hzlzvKEXKMh201Zo9cj5ZVQlA1RRpVq4oXBQcoTn5OGbFwT/EttI+Wm0yjJcjnnX9oWyOnIUsMWnvoNt08rOJV+tXpgGn9pKRU8YCHSkxdf8ujJ7eoZXfuxYIlpujKX/Mmn9QZUShK9TuJGZZDmCJ+gPxpVK5D9U7zN88PLzuDRW8N6z/8BiKFWrPCdqkFFmxHX5x7hvgYKDj9M24bH54TX8C4J4nU3E5K1mNhgCKLv91IARsALsa8bCviHqJl67AlcOB5JVV/s/PMecM3zUIpx6r5Dx360dIzs1A+3O3chNMMBGvuokWW1T3xYLy+X2ceRswrO1U70BluCbD8ulRIah6M8TizfJwupOLrByKjZ31l0INNEu7ixfstiw/IiCBUfEohbw5I1xbUzRUI6IXgCKcpPO5cHmCxhRy5tpf7/ogrU4JMymLEhej0h9KvCji1tFTrSxehCQ+Fg2g6/zApRGJoDN2UyVxmPrCxkLB9Vb4T15R7zTebOFx0V7dDXBoO5a/jtA3xL5HKxFLZ3dhJns6+QCdgO8mAdej3nCk4fnQD+gtXxceg9v98bwqDtj9Aj3khYa6xuQgvo21qDZ3kF9pSIhs+zvHIijjY0ZVngr0xxGtXf0hJlVcOFbPDSZztuI1FFEKftjP7ka2/159tP1oYh3a7IPLDIVi10RV8omtQR/LoP184M4HN5WGXR95Fn5XqoEZphtB3ah0tC4D7PVXfgc83h+TVzhyjSsvVfm1sbxH0cVjqBFZDxhG9/k6NgHYXJ2Z2zLnxI4KUC8SJjgShIBRqUujT7lKJuylkJQajXH1Svch2UpH7/5YS13sVXm1T6FbWyLeL+z1BrXJ/3GOKbHaEoYK/WCOCD731yShKIwSZ2UjAzu2XA4dV/Vdnx6rE88u7SVcFXHQC6icvvpew0/BjUTT9/yOWpi7JaMLQE2+HGxMNDMXG666KgyQxtmDxUQB0XsgutWXlilQq3iKFHPA78CKRj5BChb03LCYWoDEUlQvy3rlJ53t+DCVjpMm4jX+r66SOHb68+ZN0sjpkGh8H9cbPOETowRkig4ddn41DCvjwvY2ZMf+S5cqy+l6RVYihchhkgOoaCTiCLX61zmu9Xb+UO/gcEk0FfXpY8OC2Fj3FoWL3CrR78MfSjFyWV8nYC4cjMFBY42JeQfFrd70t91Okl+/O2JCt1YxnUZWG35GdU+mY856RRSKTgycIWfcb2CHzCd1QLGkn9OQ4M/ZgkMJgFfsEfw87JzWBEEQLs0bfUCVgTVfHoNGQunip2lLlyxdEE1tQUdUy5VATg9VaEQRSRVZ2t1RwlrCHmxrUqMgFVn/oyKhRLNnInbmp3TzjhnN3bdJ5Cxm7rhlDNGtULZOPTSA4ZPjk/cUpNTuJmBoQnhn+L49G0Rb/J1evhbjQbKqGqITTLYI4UmeU/xqNu7VNiJqB6oY+ABs8J8CteO0CuW+Nze7yPTo024vr+v28aknmY0DdNj6k+ZYXKs78Zm8Qy+weatoaBjhbLpF6zO7SenTN7XMRKUewUH1Lqtv+h+Pr9HLOaGw91sw1ko06iQnpalA2Ds+7CZWVFgDT7+wHSRQKgNxJlFY6ojw8oer/LhXnp9HklE2wYkA4BvBpJl/ogVTz5lswQV9tq9mNRs6y62TM9VDur5Zfypon3IbNQBQnfBqy/4lxpoq1stXWp7toAcHmTR/DstQD2teT//ebcH3Bn2Y8KtfW7Z8rT3dXmS+zlHok6vWinsn/b9lvbDVUuejTSXR9ncrVvtmh1xTUXQBm0v6uLkBmV9NRSRYaOcqGrENCtRfJerDwgr/jruz8CvKfVjjWJbO+YHk3SFPnzdph+0Z/1RedocsehmmbpEKe9uZbbyHb+AB0NtAtcl0chBZPpRtygEvfImQ37aPjgfKRSrORRyt8Hq+Ojnb7RlLfvGfwkRdHSIRgEXVk+OJ1XE7sjuAgPjiaypOrSCPQak480ToP+HLjkHpD4UnqTtwg4s5sl1sY49XTZ1FAqoSzlEmQjRHdfRIFiADI+JowYJkeQVeFmTP0YM9RX2IkYaJgF+jgTQPto1SPUoDO0j3HRVpuxt5BH6gf8qworutBvkEBjZS1vhuKBFt6ibHmf7HfNOdC4bC13ulHs5sFHpY5EzRORUe2WBr7qGs50zE3d5VvypIpmA5ghSS/HYpHMxquL3YeQpT8gbkBRg543oCV4MQcdXO42OTfrq+0KoklMujBe2MlF74NknOpyzTo8yI4SzYJPB4csTISfrUuV1QLEDYoJlFi/d/qA5fEppXoJr+d8dB8efVQQPSsliIXv+e2Gap661CKuVpFBj0a7nWDRMqNcoEohpmDCuXog++eUb/yXuIdycLLBmTmTjdjGBCWyDxPO2AQh7cNbjzwo4w2PIVAiesk95RTenXC0J03bpTg1kLC/Z0XAu1OhhI3fTNBKV721gbN3MtbEH+SE9IuljvT+aTT08kReIpJ1uox4l8CefKAptiW6x3fnNLbh4FJ49iHTcRN+fYaeXCDPTWy1XYYLHC6yg5t5NXebUmjIjk4g/83RnNxXZ483lBHkxLb0N6ny7nGj3ATkaN1POPAEOEvowjzV+gpnF65V3I5107TFH6pgyC0qe5pNQcy4+DS+AXz5iB8W8wqNOzxncMmr+aUrUTjV/6tB/emGDMwTJLibWugOa+d7krEbXdOOM+5T6IRIyKRFFcUeijFlXAYXGPUDjoI7fsKiUQS6mAXkUG+ygFnLIBqgVcxJDhHHU2qezmcI5/PnSE7J5jUzdV80msselpn7nffSXXQucqz8RYvBuZiFtHgeo1+Tbkmo6ZMtp/Jyd868S18L7WBZbuSuGZCTGQrv7dEYgi8snwDeItp4iX0WWorylfD/8ztodZ+WjcQfetfwYY09vWSG32AFeLvDdBiusMCIdO5QE2+1Zn6lfrTSP/JrRre8TerVtUF3G4EaqhmKKEVXYgdSJhoVH+3fhBYIxgl1t75UM/mm5XPNbqLkJvHMb4+j8sE7wAdO7bENhTiinxETwJPK1lFdtPTU4sRIB95VzvtJODfAzTCCBUEGCSqGSIb3DQEHAaCCBTIEggUuMIIFKjCCBSYGCyqGSIb3DQEMCgECoIIE7jCCBOowHAYKKoZIhvcNAQwBAzAOBAiqh1qYeqgKTwICCAAEggTIZ8BFtSl0KJvXYUUddHfPxmryzjCTjynbMMd0q/fGuHakBnYV8lq93dAwoPuvx0Tr9CAE7OulknJq7Kk868R3kc9C3MweRqepIBlguZTb4v20o/zmYgSb0Btc6IMSqrng4nU+IZFj4R7liPyf3NWs7eZbkgU/ylonCvEb9ML6PtzQ4ScjpwftrhS/Xy0DPWyVYcquGktRRHajQ5uil0LSwyvAMDmqnE9rpJ3CsHVfv6A9kORdq8pRbWeDmVCrcjEnSnBgRm7Ofh6HiDtWGU5KibgMI53VWBxtsMufbr8EVdrIgZju2CuCoIckvyDvlnITDmPw2WPTkE5ixc8JHrCPq3lJkJA5ORoeXrJxYAv1Jlm1TakuwAZ24vFBoeFmMywKY6iMLfVlZbEDyOOxOQu68GEch+sM6lxERdA0rwegKcP4m5Jr2T+qr9tR1jaH5dGOrHaGDGzdYMt6AMw4KfdiM0P2PndVDZAmaezYwMMdc7w1LYpnkWMM6tVq/NTVJLx2HtkEdfBZ9js9emAPXZA5f7EStoetK9Z9WnV1ZOVoIH3kRreuYbHfXvOYN5VSpM2JmD96dbI2wDaxac/OwWNMePATWgXigXmzCiU1ksNCZwXAsrc/BuWgTSFXWmXWzmBhWcl58sBqd/pHFgZK+cmNNFnvHLM1ZH5F/ZGa/X5lN7bZzlviJt8Ps1iZixzVKOkZg3GIzsiulJ+aEKeRgu9MGpITjaq3ScqEDuzY66NFRw4hzYvlLPz0UNSuAO5HSuRf9hHoiRglKXU1lP18JtaglT4NECb+Z3Btau+W2D09gmyxfB/n8K69cSXAb3p1pgJNgH007LAQn1VJf6Y5d/X3FXerkCn2oWAq24TV9SUD6RzgYysocyFm6eN92zX2HjBsqb9qn1gnF3KBgVy/C3LWG6qbE49AoLepvTRFgD/XlBM80civO8p8E8YX3CbBZsVpnjulXqt3rN7tTnHDWiehgnVkxpmtl+kKLAJStS3DOtffpBRHHDWuVJvjEsbSSfeX1lq7VnAhj8cD1/0patqmo2iWVqko5LJzdqlsdw0fy+CfqgTeoSX1MPG/avHBEwPcIH/ykA7pMXBO8nxCkuncvPrbLs5Mjy1Q2apudE7HR1xu6bkXDYkhceWos2fTw3dBXqeXYfulSfoJ/2/bh70Z6R+OE+pLg9inFDnP0GP3grifilnAqi/D2MOkmoxoZqqugr+O/XzkW6LPD5903aFOHelu4kBvsffxBTcEqDFYHTLjiEpK19tkj/pOknLZECrK9BdzPPG6CeYW4maZEyrGJOhmd2ACwz8Ke0pZHyHB0JyFj9/JhSR+BEwziioSjEgh0aKz5q+VgLM3v1QRn+InSpajtOWdX5stJKdkDVKbOsa7qmSw+3EeT4vX59/RFTqm4wSmnauQF3xunB+oX09DOSVdwOuVqKq6HoN/1+R7PqMaZFhmWYUD/r1fZrDrHponXHdj8BFFdxNJXddKU2QXZ5PPG7PdD20loReWHkEHG1RKXzYKtfFe95EN/tmnsJw+P8IsSLT/QERmg8KDMcyNG3nvd6T1t9FlHSv3Z0Su9R9mWX8TKDPHlaW3TVO0ofM7h/3VtE2qLZGmZbX69fsQigXZhIrRyhV1MSUwIwYJKoZIhvcNAQkVMRYEFFvO6+rFxexfXrERjC0T9jpgOXSlMCswHzAHBgUrDgMCGgQUc6YyUBdE0No+IN0KH8+UB/n/o4YECMZH5OAkXBjx
  password: bXkta2V5c3RvcmUtcGFzc3dvcmQ=
  truststore.p12: MIIFJgIBAzCCBPIGCSqGSIb3DQEHAaCCBOMEggTfMIIE2zCCBNcGCSqGSIb3DQEHBqCCBMgwggTEAgEAMIIEvQYJKoZIhvcNAQcBMBwGCiqGSIb3DQEMAQYwDgQIE3xWtU9R2doCAggAgIIEkMLUbzoQnfbwqILhK7torJqZyZUBd5Ct322vrefQ8ncrEFD7grQ/ojkb3T+Cq7vCJshBvk3QvrAY70XVsWm/M6m3ZAsyeQaTQMwrTiHoqVuN3rFUMHlkmVjlvjiKt17LH9nkQrUq4hC+H+Lj61hNVLh8BnEl0GKH4AV0Y4F5BQdKjXnPlu5KpqTRz7/wEsLJuVDp2cE8kPMEz1ZpMJfbWqnCD0/tRlqXx/7m+1/SdI6swFPjBjO60EBkHQQKbFdT1l0tliIxU3jul3LzqjZQQlPeCLWfVHDMqs2U3c0sCbqC39wK+3U14tN3Ik3GB+0fk+a4L/gNnurDj1BdiyzI1i2c8wRqO1KP21fqKNO3IqNJc6+NH9wAw8/l0dVU/unIALOJbVeRoQYhU22J7IKJgGNybeI+w0Igp5kjE1YurMOXGZFojvMkuAALLdBMjCZRTJb746dzh00CsSXUiR85aFwXM7TZzYSmdU3yU1natkprdXvsUhOp6ZDu0E+nLBlUZ0RBkFziHN75OTy9tBZWwTsWrzMel4HQ5q5hkgVKShPYQ6HY8YnN40ruztWOsBXdecjf5CAyS5Ata5HnQXl1ZodnMPQa7IFnt+mzXvy1/JyQQXVZsMqkZ2TGL7KMX+gpGoCK7RrTEqqIfoFUSXuXLSB0tDizUU60+95iwH5jvtjdrLzp/ya+UKWl3IN3V3b0/brkuTjyTkCOfzdXWNAjCCQYEjW1+iWVWu9WejQFO0cBuoKROmxe4HS1q7CKJZuT3Vp4xI1y+/+a2OkO38lRuJAb0boeqK/sYB8l2GWiJgzjB6GCWkIleT7quQZu/sZA3qt4Vb0p8lQ84kLMfPBHUKQGOOqeJDD2MkUZG7gaEx93mdKwB1UNabitFrbk2gvQTVL7x//kqJc7e0mQXyfZfzD9AVvTRhrsZ36qMSKGIwIF1NMksy7KmJz56hUD/0s+nNveNmxKFmHIfij5/P5nCBNhcdbwACcGNE05DL2ZvD4FFR6NrTtT6OeUyEyjHojFCCjuPtTU5L05WsO6bRJFmoBy9xjBqe32irBJSgLXhboBmBvpTj/IgPjnUuOOP+O9aD5nuahG8jKaU8mynxe3ULl+9W2snXXEA6/Wqr8juDFFT0YiF7iEO7vu3+Ns08BhkkSAlX4fAHqT9gmaVE16zr0lqnZUJTLrdDenZUZUTf+jbcAseKNj2GZ2PDW5nNWkZY7NY/Ihq2dmCwPL91EObmfLVOxWOKzXKzfWBfWiM+45Qw2TwEcRVlFtVQqAwFa924ZLKFhJ72MFbLAZlMwZ6uuLeExOTmtZ9PZaNAs4rwQkIJAG2Lb9ABHYEODJ4eItR78s3HZ0E622Z+1BLNw3zqcX1+/rclDV8lDGipHsmThHrDzZSt1/FFgZiQELnyCb5O8kszxDI7Jx89QtloNq5Jqc6Eas4rpFcUTB1mdBhKSvfaVEoCOyk7Vgfd92Dbg9/Bl+QaQiKZgxDJvv1kWsIYEqkj9N3Y7lho1q5ztNrtUi8dli5xzLqYHeBgVyOHYh2jlAFyirtCPM4b/Q+F9/XTgwKzAfMAcGBSsOAwIaBBQXOX5MywJ6m8FXUyOUiskDGynLYAQIb64QqbJyYXU=
kind: Secret
metadata:
  creationTimestamp: "2022-07-06T11:37:07Z"
  name: infinispan-dg-keystore-template
  namespace: infinispan
  resourceVersion: "12803544"
  uid: 0bba7851-6750-4cdc-9295-b1034721ba45
type: Opaque

## Add "xsite TLS encryption" section into Infinispan resource
...
        local:
          encryption:
            protocol: TLSv1.3
            routerKeyStore:
              alias: "1"
              filename: keystore.p12
              secretName: infinispan-dg-keystore-template
            transportKeyStore:
              alias: "1"
              filename: keystore.p12
              secretName: infinispan-dg-keystore-template
            trustStore:
              filename: truststore.p12
              secretName: infinispan-dg-keystore-template
...