search

elastic / elasticsearch-docker

Official Elasticsearch Docker image
Apache License 2.0
789 stars 240 forks source link

unable to write file [/usr/share/elasticsearch/config/users] - mount point not found #205

Closed fenchu closed 5 years ago

fenchu commented 5 years ago

WARNING: No swap limit support

* `docker inspect NAME|ID` output

$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE docker.elastic.co/elasticsearch/elasticsearch 6.4.2 e47ebd7ec3ee 13 days ago 828MB $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 03238b45703f docker.elastic.co/elasticsearch/elasticsearch:6.4.2 "/usr/local/bin/dock…" 2 hours ago Up 2 hours 0.0.0.0:9200->9200/tcp, 9300/tcp elasticsearch

$ docker inspect 03238b45703f [ { "Id": "03238b45703fc709675688a722631c0afbdeb888456302b584ec63b399e128d8", "Created": "2018-10-10T09:19:10.590250446Z", "Path": "/usr/local/bin/docker-entrypoint.sh", "Args": [ "eswrapper" ], "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 173852, "ExitCode": 0, "Error": "", "StartedAt": "2018-10-10T09:19:11.384495433Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:e47ebd7ec3eef5e48746af2ae376f180bbfbd23c7514b5b18a116f8349eb4c34", "ResolvConfPath": "/var/lib/docker/containers/03238b45703fc709675688a722631c0afbdeb888456302b584ec63b399e128d8/resolv.conf", "HostnamePath": "/var/lib/docker/containers/03238b45703fc709675688a722631c0afbdeb888456302b584ec63b399e128d8/hostname", "HostsPath": "/var/lib/docker/containers/03238b45703fc709675688a722631c0afbdeb888456302b584ec63b399e128d8/hosts", "LogPath": "/var/lib/docker/containers/03238b45703fc709675688a722631c0afbdeb888456302b584ec63b399e128d8/03238b45703fc709675688a722631c0afbdeb888456302b584ec63b399e128d8-json.log", "Name": "/elasticsearch", "RestartCount": 0, "Driver": "overlay2", "Platform": "linux", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [ "eslogs:/usr/share/elasticsearch/logs:rw", "esdata:/usr/share/elasticsearch/data:rw" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": {} }, "NetworkMode": "docker-elasticsearch_esnet", "PortBindings": { "9200/tcp": [ { "HostIp": "", "HostPort": "9200" } ] }, "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": [], "CapAdd": null, "CapDrop": null, "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": null, "GroupAdd": null, "IpcMode": "shareable", "Cgroup": "", "Links": null, "OomScoreAdj": 0, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": null, "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 0, "Memory": 0, "NanoCpus": 0, "CgroupParent": "", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": null, "DeviceCgroupRules": null, "DiskQuota": 0, "KernelMemory": 0, "MemoryReservation": 0, "MemorySwap": 0, "MemorySwappiness": null, "OomKillDisable": false, "PidsLimit": 0, "Ulimits": [ { "Name": "memlock", "Hard": -1, "Soft": -1 } ], "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0, "MaskedPaths": [ "/proc/acpi", "/proc/kcore", "/proc/keys", "/proc/latency_stats", "/proc/timer_list", "/proc/timer_stats", "/proc/sched_debug", "/proc/scsi", "/sys/firmware" ], "ReadonlyPaths": [ "/proc/asound", "/proc/bus", "/proc/fs", "/proc/irq", "/proc/sys", "/proc/sysrq-trigger" ] }, "GraphDriver": { "Data": { "LowerDir": "/var/lib/docker/overlay2/b5b62f59f860e3718825e63f2a0ea0e841d308f5b59c3ed05dfdc385d679c7b1-init/diff:/var/lib/docker/overlay2/d8841a5ec3be1c8de6403c7889b231cf40569bb94e833f9a69e69e73c13bb7af/diff:/var/lib/docker/overlay2/3af938470848152c486a28b8ec49a0fab6180a5f7f015736a6cadf34a992206f/diff:/var/lib/docker/overlay2/60957076dfc697ab60789419770e0f99fe9088a28d91d519a98dd2fc3690eaaa/diff:/var/lib/docker/overlay2/e93078c3ec45b5541391a4dcc243938803534bda19d475c78b79c19acb34689e/diff:/var/lib/docker/overlay2/52a07cb59fa672506f0aeb35632744b8be493745ea7725b8b3fe0b8bb29fe458/diff:/var/lib/docker/overlay2/1f5f9c1bd4a83ddc91dc222d43d7e25c496bb921be7d0ce953f43d1ec2e34781/diff:/var/lib/docker/overlay2/4ca3106e2ff5d2d2547262cb0f9bddef3a3cc98dfd91e15f5077c5fbecd254aa/diff:/var/lib/docker/overlay2/a21c7deccd155bfa0103cde62211b7db984ef0a5f922cf87a7fdae77f7bd2a9f/diff", "MergedDir": "/var/lib/docker/overlay2/b5b62f59f860e3718825e63f2a0ea0e841d308f5b59c3ed05dfdc385d679c7b1/merged", "UpperDir": "/var/lib/docker/overlay2/b5b62f59f860e3718825e63f2a0ea0e841d308f5b59c3ed05dfdc385d679c7b1/diff", "WorkDir": "/var/lib/docker/overlay2/b5b62f59f860e3718825e63f2a0ea0e841d308f5b59c3ed05dfdc385d679c7b1/work" }, "Name": "overlay2" }, "Mounts": [ { "Type": "volume", "Name": "esdata", "Source": "/var/lib/docker/volumes/esdata/_data", "Destination": "/usr/share/elasticsearch/data", "Driver": "local", "Mode": "rw", "RW": true, "Propagation": "" }, { "Type": "volume", "Name": "eslogs", "Source": "/var/lib/docker/volumes/eslogs/_data", "Destination": "/usr/share/elasticsearch/logs", "Driver": "local", "Mode": "rw", "RW": true, "Propagation": "" } ], "Config": { "Hostname": "03238b45703f", "Domainname": "", "User": "", "AttachStdin": false, "AttachStdout": false, "AttachStderr": false, "ExposedPorts": { "9200/tcp": {}, "9300/tcp": {} }, "Tty": false, "OpenStdin": false, "StdinOnce": false, "Env": [ "cluster.name=docker-cluster", "bootstrap.memory_lock=true", "ES_JAVA_OPTS=-Xms4048m -Xmx4048m", "PATH=/usr/share/elasticsearch/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "ELASTIC_CONTAINER=true", "JAVA_HOME=/opt/jdk-10.0.2" ], "Cmd": [ "eswrapper" ], "ArgsEscaped": true, "Image": "docker.elastic.co/elasticsearch/elasticsearch:6.4.2", "Volumes": { "/usr/share/elasticsearch/data": {}, "/usr/share/elasticsearch/logs": {} }, "WorkingDir": "/usr/share/elasticsearch", "Entrypoint": [ "/usr/local/bin/docker-entrypoint.sh" ], "OnBuild": null, "Labels": { "com.docker.compose.config-hash": "641d62dfcad5b0ed64d90bcb930ab3a5c81476d58cbb1347c5e3efb5a8ecb9c5", "com.docker.compose.container-number": "1", "com.docker.compose.oneoff": "False", "com.docker.compose.project": "docker-elasticsearch", "com.docker.compose.service": "elasticsearch", "com.docker.compose.version": "1.22.0", "license": "Elastic License", "org.label-schema.build-date": "20180804", "org.label-schema.license": "GPLv2", "org.label-schema.name": "elasticsearch", "org.label-schema.schema-version": "1.0", "org.label-schema.url": "https://www.elastic.co/products/elasticsearch", "org.label-schema.vcs-url": "https://github.com/elastic/elasticsearch-docker", "org.label-schema.vendor": "Elastic", "org.label-schema.version": "6.4.2" } }, "NetworkSettings": { "Bridge": "", "SandboxID": "b8e754ab2281626339fe320717919e11bc44fecd98366db3e070d84622e40bca", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": { "9200/tcp": [ { "HostIp": "0.0.0.0", "HostPort": "9200" } ], "9300/tcp": null }, "SandboxKey": "/var/run/docker/netns/b8e754ab2281", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "", "Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "", "Networks": { "docker-elasticsearch_esnet": { "IPAMConfig": null, "Links": null, "Aliases": [ "elasticsearch", "03238b45703f" ], "NetworkID": "0c9ba575ec99d049041d9892b18d40fde3c85c609f0c0f42511adbf25d3946fe", "EndpointID": "bf6713f0a88daa7b72224f7ae6ec2289df731b23f094b8487943b8e0d9483f2f", "Gateway": "192.168.32.1", "IPAddress": "192.168.32.2", "IPPrefixLen": 20, "IPv6Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "MacAddress": "02:42:c0:a8:20:02", "DriverOpts": null } } } } ]

* `docker-compose --version`, if relevant

docker-compose version 1.22.0, build f46880fe

$ cat docker-compose.yml version: '2.2' services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:6.4.2 container_name: elasticsearch environment:

* Steps to reproduce

$ docker-compose up -d


## Bug Description

$ docker exec -ti 03238b45703f /bin/bash [root@03238b45703f elasticsearch]# su - elasticsearch Last login: Wed Oct 10 09:32:11 UTC 2018 on pts/0 [elasticsearch@03238b45703f ~]$ export JAVA_HOME=/opt/jdk-10.0.2

[elasticsearch@03238b45703f ~]$ elasticsearch-users useradd sensio -p testtest123 -r superuser Exception in thread "main" java.io.UncheckedIOException: could not write file [/usr/share/elasticsearch/config/users] at org.elasticsearch.xpack.security.support.SecurityFiles.writeFileAtomically(SecurityFiles.java:73) at org.elasticsearch.xpack.security.authc.file.FileUserPasswdStore.writeFile(FileUserPasswdStore.java:171) at org.elasticsearch.xpack.security.authc.file.tool.UsersTool$AddUserCommand.execute(UsersTool.java:130) at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86) at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:124) at org.elasticsearch.cli.MultiCommand.execute(MultiCommand.java:77) at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:124) at org.elasticsearch.cli.Command.main(Command.java:90) at org.elasticsearch.xpack.security.authc.file.tool.UsersTool.main(UsersTool.java:45) Caused by: java.io.IOException: Mount point not found at java.base/sun.nio.fs.LinuxFileStore.findMountEntry(LinuxFileStore.java:91) at java.base/sun.nio.fs.UnixFileStore.(UnixFileStore.java:65) at java.base/sun.nio.fs.LinuxFileStore.(LinuxFileStore.java:44) at java.base/sun.nio.fs.LinuxFileSystemProvider.getFileStore(LinuxFileSystemProvider.java:51) at java.base/sun.nio.fs.LinuxFileSystemProvider.getFileStore(LinuxFileSystemProvider.java:39) at java.base/sun.nio.fs.UnixFileSystemProvider.getFileStore(UnixFileSystemProvider.java:369) at java.base/java.nio.file.Files.getFileStore(Files.java:1479) at org.elasticsearch.env.Environment.getFileStore(Environment.java:324) at org.elasticsearch.xpack.security.support.SecurityFiles.writeFileAtomically(SecurityFiles.java:61) ... 8 more

[elasticsearch@03238b45703f ~]$ id uid=1000(elasticsearch) gid=1000(elasticsearch) groups=1000(elasticsearch),0(root) [elasticsearch@03238b45703f ~]$ ls -l /usr/share/elasticsearch/config/ total 28 -rw-rw---- 1 elasticsearch root 207 Oct 10 09:19 elasticsearch.keystore -rw-rw-r-- 1 elasticsearch root 271 Sep 26 14:17 elasticsearch.yml drwxrwx--- 2 elasticsearch root 4096 Sep 26 14:20 ingest-geoip -rw-rw---- 1 elasticsearch root 2937 Sep 26 13:30 jvm.options -rw-rw-r-- 1 elasticsearch root 272 Sep 26 14:17 log4j2.properties -rw-rw---- 1 elasticsearch root 473 Sep 26 13:38 role_mapping.yml -rw-rw---- 1 elasticsearch root 197 Sep 26 13:38 roles.yml -rw-rw---- 1 elasticsearch root 0 Sep 26 13:38 users -rw-rw---- 1 elasticsearch root 0 Sep 26 13:38 users_roles

[elasticsearch@03238b45703f ~]$ /usr/share/elasticsearch/config/ [elasticsearch@03238b45703f config]$ touch writeme [elasticsearch@03238b45703f config]$ ls -l writeme -rw-rw-r-- 1 elasticsearch elasticsearch 0 Oct 10 09:25 writeme


I've tried all 6.4.x images, they all use the java 10.0.2 same issue. Any help very appreciated.
dliappis commented 5 years ago

@fenchu Thanks for the bug report. You are likely hitting a kernel issue with Docker overlay2 (possibly: https://github.com/moby/moby/issues/34672).

I was able to reproduce your issue with an Ubuntu 16.04 vm using 4.4.0-112-generic kernel and Docker 17.12.1-ce with the default overlay2.

Upgrading to e.g. 4.18.14 (from: http://kernel.ubuntu.com/~kernel-ppa/mainline/v4.18.14/) resolved the issue.

Additionally, I was able to resolve the issue by keeping the Xenial kernel 4.4.0-112 and switching to the aufs storage driver (in /etc/docker/daemon.json as per https://docs.docker.com/storage/storagedriver/overlayfs-driver/#configure-docker-with-the-overlay-or-overlay2-storage-driver).

One other thing to note here is that you shouldn't be keeping this file in the layered filesystem and editing it by entering the container, as this change will be lost when you restart the container.

Instead, you could bind mount the (preprovisioned) file (e.g. cli syntax -v <full_host_path>/config/users:/usr/share/elasticsearch/config/users) e.g.

sensio:$2a$10$Qyduq6ew8U7H8q4aRVDCE.6eIpd5IzNVwtCGq1L4kqX4BnNK2aF..

make sure the file is still readable by Elasticsearch by giving it uid:gid 1000:0.

So TL;DR:

  1. If possible, you can try upgrading the kernel
  2. If 1. is not possible, you can try a different storage driver e.g. aufs
  3. Consider using a bind mount (i.e. bypass the Docker storage driver) for the entire config/ dir or specifically for config/users.
dliappis commented 5 years ago

The exact RCA for this issue can be found below. Kudos and many thanks to @alexsapran who researched this and came up with the detailed analysis that I am copying/pasting here:

RCA

The kernel 4.4.156 added a commit https://github.com/torvalds/linux/commit/121b09d30d48a59a0ae621b130f3b4 that added a regression with the OverlayFs's without layer that is been fixed in the subsystem's upstream but not yet ported to the latest kernel, I am thinking this would be soon-ish enouhght. The commit that fixes the regression is this one https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/vfs.git/commit/?id=d0e13f5bbe4be7c8f27736fc40503dcec04b7de0

The prerequisite that was added will guard us against the problematic kernels. So pinning the version for now to 4.4.155 is something reasonable.

Closing this issue since there is nothing more at the moment that we can do except the installer prereq check. If we need further investigation we can always reopen and revisit this.

Reproduction steps

$ mkdir test
$ cd test
$ mkdir upper lower work merged lower/dir
$ sudo mount -t overlay overlay -olowerdir=lower,upperdir=upper,workdir=work merged
$ unshare -m -p -f -U -r bash

# Now as user root

$ cd merged
$ ls -la
drwxrwxr-x 1 root root  6 Sep 28 13:47 .
drwxrwxr-x 6 root root 58 Sep 28 13:47 ..
drwxrwxr-x 2 root root  6 Sep 28 13:47 dir
$ rm -rf dir
$ ls -la
drwxrwxr-x 1 root root  6 Sep 28 13:47 .
drwxrwxr-x 6 root root 58 Sep 28 13:47 ..
$ install -o elastic -g elastic -d dir
install: cannot change owner and permissions of ‘dir’: Invalid argument
$ ls -la
drwxrwxr-x 1 root  root  17 Sep 28 13:50 .
drwxrwxr-x 6 root  root  58 Sep 28 13:47 ..
drwx------ 2 65534 65534  6 Sep 28 13:50 dir

# Just to show elastic user exists
$ id elastic
uid=1000(elastic) gid=1000(elastic) groups=1000(elastic),4(adm),10(wheel),190(systemd-journal),1001(docker)

strace-ing this failed install

11046 mkdir("dir", 0700)                = 0
11046 open("dir", O_RDONLY|O_NOCTTY|O_NONBLOCK|O_DIRECTORY|O_NOFOLLOW) = -1 EACCES (Permission denied)
11046 stat("dir", {st_mode=S_IFDIR|0700, st_size=6, ...}) = 0
11046 lchown("dir", 1000, 1000)         = -1 EPERM (Operation not permitted)
11046 open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
11046 fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0

Seems like mkdir creates the directory but the sequent open fails, but the directory is indeed created.

fenchu commented 5 years ago

@dliappis Thanks for this excellent feedback. I moved to external volume for config, but initial testing is messing directly with containers. We follow the plain ubuntu 16.04LTS and Debian Strech 9 release cycle. Great to know this will be fixed once docker is upgraded.

dliappis commented 5 years ago

You are welcome @fenchu ; note that your original comment reported the issue on Debian 9, using Kernel 4.9.0-6-amd64 kernel, so there's a difference with the above repro steps executed in Ubuntu Xenial LTS, but the underlying root kernel cause should be the same (disclaimer: haven't had the time to test the repro steps with a newer kernel on Debian 9 as I did on Ubunu 16.04).