search

elastic / fleet-server

The Fleet server allows managing a fleet of Elastic Agents.
Other
82 stars 81 forks source link

Failed Fleet Server Install #2279

Open bigverm23 opened 1 year ago

bigverm23 commented 1 year ago 
{"log.level":"info","@timestamp":"2023-01-16T14:21:07.770-0500","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":407},"message":"Generating self-signed certificate for Fleet Server","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2023-01-16T14:21:10.415-0500","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":770},"message":"Waiting for Elastic Agent to start Fleet Server","ecs.version":"1.6.0"}
{"log.level":"info","@timestamp":"2023-01-16T14:22:10.430-0500","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":803},"message":"Fleet Server - Error - EOF","ecs.version":"1.6.0"}
Error: fleet-server failed: context canceled
For help, please see our troubleshooting guide at https://www.elastic.co/guide/en/fleet/8.6/fleet-troubleshooting.html
Error: enroll command failed with exit code: 1
For help, please see our troubleshooting guide at https://www.elastic.co/guide/en/fleet/8.6/fleet-troubleshooting.html
curl -L -O https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-8.6.0-linux-x86_64.tar.gz
tar xzvf elastic-agent-8.6.0-linux-x86_64.tar.gz
cd elastic-agent-8.6.0-linux-x86_64
sudo ./elastic-agent install \
  --fleet-server-es=http://localhost:9200 \
  --fleet-server-service-token=xxxxxxxxxx \
  --fleet-server-policy=fleet-server-policy

Used docker-compose: 

  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.0
    volumes:
      - $USERDIR/docker/certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es02\n"\
          "    dns:\n"\
          "      - es02\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es03\n"\
          "    dns:\n"\
          "      - es03\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120
  es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.0
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 3
      start_period: 30s
    ports:
      - 9200:9200
    environment:
      - node.name=es01
      - cluster.name=ken
      - cluster.initial_master_nodes=es01,es02
      - discovery.seed_hosts=es02
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=basic
    mem_limit: 1073741824
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120
    volumes:
      - $USERDIR/docker/elasticsearch/es01:/usr/share/elasticsearch/data
      - $USERDIR/docker/certs:/usr/share/elasticsearch/config/certs
  es02:
    depends_on:
      - es01
    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.0
    volumes:
      - $USERDIR/docker/elasticsearch/es02:/usr/share/elasticsearch/data
      - $USERDIR/docker/certs:/usr/share/elasticsearch/config/certs
    environment:
      - node.name=es02
      - cluster.name=ken
      - cluster.initial_master_nodes=es01,es02
      - discovery.seed_hosts=es01
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es02/es02.key
      - xpack.security.http.ssl.certificate=certs/es02/es02.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es02/es02.key
      - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=basic
    mem_limit: 1073741824
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120
  es03:
    depends_on:
      - es02
    image: docker.elastic.co/elasticsearch/elasticsearch:8.6.0
    volumes:
      - $USERDIR/docker/elasticsearch/es03:/usr/share/elasticsearch/data
      - $USERDIR/docker/certs:/usr/share/elasticsearch/config/certs
    environment:
      - node.name=es03
      - cluster.name=ken
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es02
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es03/es03.key
      - xpack.security.http.ssl.certificate=certs/es03/es03.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es03/es03.key
      - xpack.security.transport.ssl.certificate=certs/es03/es03.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=basic
    mem_limit: 1073741824
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120
  kibana:
    depends_on:
      es01:
        condition: service_healthy
      es02:
        condition: service_healthy
      es03:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:8.6.0
    volumes:
      - $USERDIR/docker/certs:/usr/share/kibana/config/certs
      - $USERDIR/docker/kibana:/usr/share/kibana/data
    ports:
      - 5601:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
    mem_limit: 1073741824
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

For confirmed bugs, please report:

bigverm23 commented 1 year ago

curl -k -u user:password https://10.0.0.7:9200 produces:

{
  "name" : "es01",
  "cluster_name" : "ken",
  "cluster_uuid" : "*******************",
  "version" : {
    "number" : "8.6.0",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "f67ef2df40237445caa70e2fef79471cc608d70d",
    "build_date" : "2023-01-04T09:35:21.782467981Z",
    "build_snapshot" : false,
    "lucene_version" : "9.4.2",
    "minimum_wire_compatibility_version" : "7.17.0",
    "minimum_index_compatibility_version" : "7.0.0"
  },
  "tagline" : "You Know, for Search"
}
nirajchandak commented 1 year ago

I am also facing the same issue with version 8.6.2 when trying to install fleet-server on MAC

nchaulet commented 1 year ago

It seems to me that the Elasticsearch output is not configured correctly the EOF error is probably happening because Fleet Server cannot reach elasticsearch, can you try to configure the output with an address that Fleet Server can reach? in the Fleet UI (Fleet plugin => the settings tab) or via an environment variables in Kibana XPACK_FLEET_AGENTS_ELASTICSEARCH_HOSTS: '["https://elasticsearch:9200"]'

fritzfs commented 1 year ago

Hey! I had the same problem. The issue is with generated instructions from Kibana. It should be HTTPS (not HTTP) when specifying fleet-server-es.

Elastic, can this be fixed in the next version? Thanks!

This is correct.

sudo ./elastic-agent install \ --fleet-server-es=https://localhost:9200 \ --fleet-server-service-token=xxxxxxxxxx \ --fleet-server-policy=fleet-server-policy

lxm065 commented 4 days ago

use this doc to create ssl:https://www.elastic.co/guide/en/fleet/8.1/secure-connections.html

then run as:

./bin/elasticsearch-certutil ca --pem ./bin/elasticsearch-certutil cert \ --name fleet-server \ --ca-cert /usr/share/elasticsearch/fleek/ca/ca.crt \ --ca-key /usr/share/elasticsearch/fleek/ca/ca.key \ --dns fleet.xxx.com \ --ip 192.168.20.7 \ --pem

sudo ./elastic-agent install -f \ --url=https://192.168.20.7:8220 \ --fleet-server-es=https://192.168.20.7:9200 \ --fleet-server-service-token=AAEAAWVsYXN0aWMvZmxlZXQtc2VydmVyL3Rva2VuLTE3MjYxOTIzOTQ3NjY6c0NsSDJKVFFSSXU4ODFpTHBCRFVwUQ \ --fleet-server-es-ca=/var/lib/docker/volumes/es8_certs/_data/ca/ca.crt \ --certificate-authorities=/data/es-kibana/es8/feek-key/ca/ca.crt \ --fleet-server-cert=/data/es-kibana/es8/feek-key/fleet-server/fleet-server.crt \ --fleet-server-cert-key=/data/es-kibana/es8/feek-key/fleet-server/fleet-server.key

then it's ok.

{"log.level":"info","@timestamp":"2024-09-13T14:19:24.187+0800","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":777},"message":"Fleet Server - Starting","ecs.version":"1.6.0"} {"log.level":"info","@timestamp":"2024-09-13T14:19:28.192+0800","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":758},"message":"Fleet Server - Running on default policy with Fleet Server integration; missing config fleet.agent.id (expected during bootstrap process)","ecs.version":"1.6.0"} {"log.level":"info","@timestamp":"2024-09-13T14:19:29.157+0800","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":455},"message":"Starting enrollment to URL: https://192.168.20.7:8220/","ecs.version":"1.6.0"} {"log.level":"info","@timestamp":"2024-09-13T14:19:33.012+0800","log.origin":{"file.name":"cmd/enroll_cmd.go","file.line":255},"message":"Successfully triggered restart on running Elastic Agent.","ecs.version":"1.6.0"} Successfully enrolled the Elastic Agent. Elastic Agent has been successfully installed.