search

toluaina / pgsync

Postgres to Elasticsearch/OpenSearch sync
https://pgsync.com
MIT License
1.16k stars 180 forks source link

Authentication exception (401) when connecting to elasticsearch: 'unable to authenticate user [elastic] for REST request [/]' #522

Open d3vr10 opened 8 months ago

d3vr10 commented 8 months ago

PGSync version: 3.1.0

Postgres version:

Elasticsearch/OpenSearch version: elastic search 8.11.1

Redis version: 7.2.3

Python version: 3.11.7

Problem Description: I'm having trouble with elastic search authentication though I'm setting the same password and SSL certificates, the three of them: client, CA and key certificates. If I turn off ssl and https as schema everything works fine, but I need to use secure connection. Here below is a reproduceable example. I can login to elasticsearch through kibana and curl just fine (I haven't tested yet with curl supplying the ca and client certificates).

Any feedback on this would be appreciated.

Docker YAML File:

version: '3.9'

services:
  setup:
    networks:
      - elastic
    image: elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: '0'
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ['CMD-SHELL', '[ -f config/certs/es01/es01.crt ]']
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    depends_on:
      setup:
        condition: service_healthy

    networks:
      - elastic
      - frontend
      - data-center

    image: elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          'CMD-SHELL',
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    depends_on:
      es01:
        condition: service_healthy

    networks:
      - elastic

    image: kibana:${STACK_VERSION}

    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data

    ports:
      - ${KIBANA_PORT}:5601

    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt

    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        [
          'CMD-SHELL',
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  postgres:
    ports:
      - 5432:5432
    image: postgres:16.1
    volumes:
      - "./init_db:/docker-entrypoint-initdb.d"
      # - "../postgres/data:/var/lib/postgresql/data"
    environment:
      - "POSTGRES_USER=revista"
      - "POSTGRES_PASSWORD=${ELASTIC_PASSWORD}"
    restart: always

    networks:
      - data-center

    command: 
      - "postgres"
      - "-c"
      - "wal_level=logical"
      - "-c"
      - "max_replication_slots=3"

  redis:
    image: redis:7.2.3
    ports:
      - 6379:6379
    command: redis-server --requirepass ${ELASTIC_PASSWORD}
    networks:
      - data-center

  pgsync:
    build:
      context: ../pgsync
      dockerfile: ./Dockerfile-pgsync

    restart: on-failure

    labels:
      org.label-schema.name: "pgsync"
      org.label-schema.description: "Postgres to Elasticsearch sync"
      com.label-schema.service-type: "daemon"

    sysctls:
      - net.ipv4.tcp_keepalive_time=200
      - net.ipv4.tcp_keepalive_intvl=200
      - net.ipv4.tcp_keepalive_probes=5

    depends_on:
      - postgres
      - redis

    environment:
      - CHECKPOINT_PATH=/pgsync
      - PG_USER=revista
      - PG_HOST=postgres
      - PG_PORT=5432
      - PG_PASSWORD=${ELASTIC_PASSWORD}
      - ELASTICSEARCH_SCHEME=https
      - ELASTICSEARCH_HOST=es01
      - ELASTICSEARCH_PORT=9200
      - ELASTICSEARCH_USER=elastic
      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
      - ELASTICSEARCH_VERIFY_CERTS=True
      - ELASTICSEARCH_USE_SSL=True
      - REDIS_HOST=redis
      - REDIS_PORT=6379
      - REDIS_AUTH=${ELASTIC_PASSWORD}
      - ELASTICSEARCH=True
      - OPENSEARCH=False
      - SCHEMA=/pgsync/schema.json
      - LOG_LEVEL=INFO
      - ELASTICSEARCH_CA_CERTS=/usr/share/elasticsearch/config/certs/ca/ca.crt
      - ELASTICSEARCH_CLIENT_CERT=/usr/share/elasticsearch/config/certs/es01/es01.crt
      - ELASTICSEARCH_CLIENT_KEY=/usr/share/elasticsearch/config/certs/es01/es01.key

    command: ./runserver.sh && sleep inf

    volumes:
      # - ../pgsync/example-schema.json:/pgsync/schema.json
      - ../pgsync/schema.json:/pgsync/schema.json
      - certs:/usr/share/elasticsearch/config/certs

    networks:
      - data-center

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  kibanadata:
    driver: local

networks:
  networks:
  data-center:
  frontend:
  elastic:

Error Message (if any): 

pgsync-1  | Traceback (most recent call last):
pgsync-1  |   File "/usr/local/bin/pgsync", line 7, in <module>
pgsync-1  |  0:00:00.202205 (0.20 sec)
pgsync-1  |     sync.main()
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1157, in __call__
pgsync-1  |     return self.main(*args, **kwargs)
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1078, in main
pgsync-1  |     rv = self.invoke(ctx)
pgsync-1  |          ^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/click/core.py", line 1434, in invoke
pgsync-1  |     return ctx.invoke(self.callback, **ctx.params)
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/click/core.py", line 783, in invoke
pgsync-1  |     return __callback(*args, **kwargs)
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/pgsync/sync.py", line 1480, in main
pgsync-1  |     sync: Sync = Sync(
pgsync-1  |                  ^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/pgsync/singleton.py", line 36, in __call__
pgsync-1  |     cls._instances[key] = super(Singleton, cls).__call__(
pgsync-1  |                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/pgsync/sync.py", line 88, in __init__
pgsync-1  |     self.search_client: SearchClient = SearchClient()
pgsync-1  |                                        ^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/pgsync/search_client.py", line 48, in __init__
pgsync-1  |     self.__client.info()["version"]["number"].split(".")[0]
pgsync-1  |     ^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/utils.py", line 402, in wrapped
pgsync-1  |     return api(*args, **kwargs)
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/__init__.py", line 2278, in info
pgsync-1  |     return self.perform_request(  # type: ignore[return-value]
pgsync-1  |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pgsync-1  |   File "/usr/local/lib/python3.11/site-packages/elasticsearch/_sync/client/_base.py", line 320, in perform_request
pgsync-1  |     raise HTTP_EXCEPTIONS.get(meta.status, ApiError)(
pgsync-1  | elasticsearch.AuthenticationException: AuthenticationException(401, 'security_exception', 'unable to authenticate user [elastic] for REST request [/]')
HalfSea commented 3 months ago

Try setting the ElasticSearch password to not include special characters.