search

mesos / kibana

Kibana on Mesos
Apache License 2.0
27 stars 9 forks source link

Kibana Executor fails on real Mesos Cluster #8

Closed sadovnikov closed 8 years ago

sadovnikov commented 8 years ago

Kibana executor fails after a few minutes of running

image

viktor_sadovnikov@elk-mesos-slave-0:~$ sudo docker inspect 404349f442a3
[
{
    "Id": "404349f442a393476560ae32b022eb7bbadac16f9c4b50532815464b14a35d3b",
    "Created": "2016-01-21T13:27:31.780649657Z",
    "Path": "/docker-entrypoint.sh",
    "Args": [
        "kibana"
    ],
    "State": {
        "Status": "running",
        "Running": true,
        "Paused": false,
        "Restarting": false,
        "OOMKilled": false,
        "Dead": false,
        "Pid": 6847,
        "ExitCode": 0,
        "Error": "",
        "StartedAt": "2016-01-21T13:27:31.969726662Z",
        "FinishedAt": "0001-01-01T00:00:00Z"
    },
    "Image": "544887fbfa30e56bd67de8e877fc5e350ba555dfa9a9930035c054dfc0792e20",
    "ResolvConfPath": "/var/lib/docker/containers/404349f442a393476560ae32b022eb7bbadac16f9c4b50532815464b14a35d3b/resolv.conf",
    "HostnamePath": "/var/lib/docker/containers/404349f442a393476560ae32b022eb7bbadac16f9c4b50532815464b14a35d3b/hostname",
    "HostsPath": "/var/lib/docker/containers/404349f442a393476560ae32b022eb7bbadac16f9c4b50532815464b14a35d3b/hosts",
    "LogPath": "/var/lib/docker/containers/404349f442a393476560ae32b022eb7bbadac16f9c4b50532815464b14a35d3b/404349f442a393476560ae32b022eb7bbadac16f9c
4b50532815464b14a35d3b-json.log",
    "Name": "/mesos-ed010bcd-29d3-4cf0-aadb-a5cc92207117-S1.c1b8014e-6515-473e-bb25-ccc0c1ca1fa6",
    "RestartCount": 0,
    "Driver": "aufs",
    "ExecDriver": "native-0.2",
    "MountLabel": "",
    "ProcessLabel": "",
    "AppArmorProfile": "",
    "ExecIDs": null,
    "HostConfig": {
        "Binds": [
            "/tmp/mesos/slaves/ed010bcd-29d3-4cf0-aadb-a5cc92207117-S1/frameworks/edcdb2b1-bfa1-46b7-94f1-3c04ff0e9c0e-0001/executors/kibana-2/runs/c1
b8014e-6515-473e-bb25-ccc0c1ca1fa6:/mnt/mesos/sandbox"
        ],
        "ContainerIDFile": "",
        "LxcConf": [],
        "Memory": 134217728,
        "MemoryReservation": 0,
        "MemorySwap": -1,
        "KernelMemory": 0,
        "CpuShares": 102,
        "CpuPeriod": 0,
        "CpusetCpus": "",
        "CpusetMems": "",
        "CpuQuota": 0,
        "BlkioWeight": 0,
        "OomKillDisable": false,
        "MemorySwappiness": -1,
        "Privileged": false,
        "PortBindings": {
            "5601/tcp": [
                {
                    "HostIp": "",
                    "HostPort": "31002"
                }
            ]
        },
        "Links": null,
        "PublishAllPorts": false,
        "Dns": [],
        "DnsOptions": [],
        "DnsSearch": [],
        "ExtraHosts": null,
        "VolumesFrom": null,
        "Devices": [],
        "NetworkMode": "bridge",
        "IpcMode": "",
        "PidMode": "",
        "UTSMode": "",
        "CapAdd": null,
        "CapDrop": null,
        "GroupAdd": null,
        "RestartPolicy": {
            "Name": "no",
            "MaximumRetryCount": 0
        },
        "SecurityOpt": null,
        "ReadonlyRootfs": false,
        "Ulimits": null,
        "LogConfig": {
            "Type": "json-file",
            "Config": {}
        },
        "CgroupParent": "",
        "ConsoleSize": [
            0,
            0
        ],
        "VolumeDriver": ""
    },
    "GraphDriver": {
        "Name": "aufs",
        "Data": null
    },
    "Mounts": [
        {
            "Source": "/tmp/mesos/slaves/ed010bcd-29d3-4cf0-aadb-a5cc92207117-S1/frameworks/edcdb2b1-bfa1-46b7-94f1-3c04ff0e9c0e-0001/executors/kibana-2/runs/c1b8014e-6515-473e-bb25-ccc0c1ca1fa6",
            "Destination": "/mnt/mesos/sandbox",
            "Mode": "",
            "RW": true
        }
    ],
    "Config": {
        "Hostname": "404349f442a3",
        "Domainname": "",
        "User": "",
        "AttachStdin": false,
        "AttachStdout": true,
        "AttachStderr": true,
        "ExposedPorts": {
            "5601/tcp": {}
        },
        "Tty": false,
        "OpenStdin": false,
        "StdinOnce": false,
        "Env": [
            "ELASTICSEARCH_URL=http://10.20.30.2:31000/",
            "MESOS_SANDBOX=/mnt/mesos/sandbox",
            "MESOS_CONTAINER_NAME=mesos-ed010bcd-29d3-4cf0-aadb-a5cc92207117-S1.c1b8014e-6515-473e-bb25-ccc0c1ca1fa6",
            "PATH=/opt/kibana/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
            "KIBANA_VERSION=4.3.1",
            "KIBANA_SHA1=115ba22882df75eb5f07330b7ad8781a57569b00"
        ],
        "Cmd": [
            "kibana"
        ],
        "Image": "kibana:4.3.1",
        "Volumes": null,
        "WorkingDir": "",
        "Entrypoint": [
            "/docker-entrypoint.sh"
        ],
        "OnBuild": null,
        "Labels": {},
        "StopSignal": "SIGTERM"
    },
    "NetworkSettings": {
        "Bridge": "",
        "SandboxID": "c6945344412941807bd9630272bfc3ab7834ae2f07571b0fe555427b7be0ba7a",
        "HairpinMode": false,
        "LinkLocalIPv6Address": "",
        "LinkLocalIPv6PrefixLen": 0,
        "Ports": {
            "5601/tcp": [
                {
                    "HostIp": "0.0.0.0",
                    "HostPort": "31002"
                }
            ]
        },
        "SandboxKey": "/var/run/docker/netns/c69453444129",
        "SecondaryIPAddresses": null,
        "SecondaryIPv6Addresses": null,
        "EndpointID": "a587a7ee71b5e9d50fb3ef0367f9d3209db4064112d5c25f4673e56b434789a7",
        "Gateway": "172.17.0.1",
        "GlobalIPv6Address": "",
        "GlobalIPv6PrefixLen": 0,
        "IPAddress": "172.17.0.2",
        "IPPrefixLen": 16,
        "IPv6Gateway": "",
        "MacAddress": "02:42:ac:11:00:02",
        "Networks": {
            "bridge": {
                "EndpointID": "a587a7ee71b5e9d50fb3ef0367f9d3209db4064112d5c25f4673e56b434789a7",
                "Gateway": "172.17.0.1",
                "IPAddress": "172.17.0.2",
                "IPPrefixLen": 16,
                "IPv6Gateway": "",
                "GlobalIPv6Address": "",
                "GlobalIPv6PrefixLen": 0,
                "MacAddress": "02:42:ac:11:00:02"
            }
        }
    }
}
]
sadovnikov commented 8 years ago

Possibly the executor hits the limit of claimed resources

image

sadovnikov commented 8 years ago

It hits the memory limit

[516637.308233] Task in /system.slice/docker-404349f442a393476560ae32b022eb7bbadac16f9c4b50532815464b14a35d3b.scope killed as a result of limit of /system.slice/docker-404349f442a393476560ae32b022eb7bbadac16f9c4b50532815464b14a35d3b.scope
[516637.308236] memory: usage 131072kB, limit 131072kB, failcnt 48
[516637.308237] memory+swap: usage 0kB, limit 9007199254740988kB, failcnt 0
[516637.308238] kmem: usage 0kB, limit 9007199254740988kB, failcnt 0
[516637.308239] Memory cgroup stats for /system.slice/docker-404349f442a393476560ae32b022eb7bbadac16f9c4b50532815464b14a35d3b.scope: cache:128KB rss:130944KB rss_huge:32768KB mapped_file:0KB writeback:0KB inactive_anon:8KB active_anon:131028KB inactive_file:0KB active_file:0KB unevictable:0KB
[516637.308246] [ pid ]   uid  tgid total_vm      rss nr_ptes swapents oom_score_adj name
[516637.308287] [ 6847]   999  6847   228345    36319     139        0             0 node
[516637.308289] Memory cgroup out of memory: Kill process 6847 (node) score 1112 or sacrifice child
[516637.317342] Killed process 6847 (node) total-vm:913380kB, anon-rss:130888kB, file-rss:14388kB
[516637.369496] docker0: port 1(vethbb45986) entered disabled state
sadovnikov commented 8 years ago

Memory requirements are configurable through parameter -mem, otherwise 128 MB

  "args": ["java",
    "-jar",
    "kibana-0.3.0.jar",
    "-zookeeper",
    "zk://10.20.30.5:2181/mesos",
    "-version",
    "4.3.1",
    "-elasticsearch",
    "http://_ES_IP_:_ES_PORT_/",
    "-mem",
    "512"
  ],