Closed sascha-egerer closed 8 years ago
@sascha-egerer hmm could these be connections from docker engine to the volume driver? could you please run the lsof command without wc -l
and run ps aux
to see list of processes that have open handles to the socket?
$ ps aux | grep azurefile
root 11231 0.0 0.2 194932 10500 ? Ssl Feb23 0:03 /usr/bin/azurefile-dockervolumedriver
$ sudo lsof | grep azurefile
init 1 root 12w REG 8,1 3353 2055 /var/log/upstart/azurefile-dockervolumedriver.log.1 (deleted)
azurefile 11231 root cwd DIR 8,1 4096 2 /
azurefile 11231 root rtd DIR 8,1 4096 2 /
azurefile 11231 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 11231 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 11231 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 11231 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 11231 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 11231 root 1u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 root 2u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 root 3u unix 0xffff8801b6eecb00 0t0 87392 /run/docker/plugins/azurefile.sock
azurefile 11231 root 4u 0000 0,10 0 8199 anon_inode
azurefile 11231 root 5u unix 0xffff8801b6eef440 0t0 89435 /run/docker/plugins/azurefile.sock
azurefile 11231 root 6u unix 0xffff8801b6ee9680 0t0 117503 /run/docker/plugins/azurefile.sock
azurefile 11231 root 7u unix 0xffff8801b6eed640 0t0 89524 /run/docker/plugins/azurefile.sock
azurefile 11231 root 8u unix 0xffff8801b6eef080 0t0 117595 /run/docker/plugins/azurefile.sock
azurefile 11231 root 9u unix 0xffff8801b5c26180 0t0 117618 /run/docker/plugins/azurefile.sock
azurefile 11231 11236 root cwd DIR 8,1 4096 2 /
azurefile 11231 11236 root rtd DIR 8,1 4096 2 /
azurefile 11231 11236 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 11231 11236 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 11231 11236 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 11231 11236 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 11231 11236 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 11231 11236 root 1u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11236 root 2u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11236 root 3u unix 0xffff8801b6eecb00 0t0 87392 /run/docker/plugins/azurefile.sock
azurefile 11231 11236 root 4u 0000 0,10 0 8199 anon_inode
azurefile 11231 11236 root 5u unix 0xffff8801b6eef440 0t0 89435 /run/docker/plugins/azurefile.sock
azurefile 11231 11236 root 6u unix 0xffff8801b6ee9680 0t0 117503 /run/docker/plugins/azurefile.sock
azurefile 11231 11236 root 7u unix 0xffff8801b6eed640 0t0 89524 /run/docker/plugins/azurefile.sock
azurefile 11231 11236 root 8u unix 0xffff8801b6eef080 0t0 117595 /run/docker/plugins/azurefile.sock
azurefile 11231 11236 root 9u unix 0xffff8801b5c26180 0t0 117618 /run/docker/plugins/azurefile.sock
azurefile 11231 11237 root cwd DIR 8,1 4096 2 /
azurefile 11231 11237 root rtd DIR 8,1 4096 2 /
azurefile 11231 11237 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 11231 11237 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 11231 11237 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 11231 11237 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 11231 11237 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 11231 11237 root 1u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11237 root 2u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11237 root 3u unix 0xffff8801b6eecb00 0t0 87392 /run/docker/plugins/azurefile.sock
azurefile 11231 11237 root 4u 0000 0,10 0 8199 anon_inode
azurefile 11231 11237 root 5u unix 0xffff8801b6eef440 0t0 89435 /run/docker/plugins/azurefile.sock
azurefile 11231 11237 root 6u unix 0xffff8801b6ee9680 0t0 117503 /run/docker/plugins/azurefile.sock
azurefile 11231 11237 root 7u unix 0xffff8801b6eed640 0t0 89524 /run/docker/plugins/azurefile.sock
azurefile 11231 11237 root 8u unix 0xffff8801b6eef080 0t0 117595 /run/docker/plugins/azurefile.sock
azurefile 11231 11237 root 9u unix 0xffff8801b5c26180 0t0 117618 /run/docker/plugins/azurefile.sock
azurefile 11231 11238 root cwd DIR 8,1 4096 2 /
azurefile 11231 11238 root rtd DIR 8,1 4096 2 /
azurefile 11231 11238 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 11231 11238 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 11231 11238 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 11231 11238 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 11231 11238 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 11231 11238 root 1u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11238 root 2u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11238 root 3u unix 0xffff8801b6eecb00 0t0 87392 /run/docker/plugins/azurefile.sock
azurefile 11231 11238 root 4u 0000 0,10 0 8199 anon_inode
azurefile 11231 11238 root 5u unix 0xffff8801b6eef440 0t0 89435 /run/docker/plugins/azurefile.sock
azurefile 11231 11238 root 6u unix 0xffff8801b6ee9680 0t0 117503 /run/docker/plugins/azurefile.sock
azurefile 11231 11238 root 7u unix 0xffff8801b6eed640 0t0 89524 /run/docker/plugins/azurefile.sock
azurefile 11231 11238 root 8u unix 0xffff8801b6eef080 0t0 117595 /run/docker/plugins/azurefile.sock
azurefile 11231 11238 root 9u unix 0xffff8801b5c26180 0t0 117618 /run/docker/plugins/azurefile.sock
azurefile 11231 11591 root cwd DIR 8,1 4096 2 /
azurefile 11231 11591 root rtd DIR 8,1 4096 2 /
azurefile 11231 11591 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 11231 11591 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 11231 11591 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 11231 11591 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 11231 11591 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 11231 11591 root 1u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11591 root 2u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11591 root 3u unix 0xffff8801b6eecb00 0t0 87392 /run/docker/plugins/azurefile.sock
azurefile 11231 11591 root 4u 0000 0,10 0 8199 anon_inode
azurefile 11231 11591 root 5u unix 0xffff8801b6eef440 0t0 89435 /run/docker/plugins/azurefile.sock
azurefile 11231 11591 root 6u unix 0xffff8801b6ee9680 0t0 117503 /run/docker/plugins/azurefile.sock
azurefile 11231 11591 root 7u unix 0xffff8801b6eed640 0t0 89524 /run/docker/plugins/azurefile.sock
azurefile 11231 11591 root 8u unix 0xffff8801b6eef080 0t0 117595 /run/docker/plugins/azurefile.sock
azurefile 11231 11591 root 9u unix 0xffff8801b5c26180 0t0 117618 /run/docker/plugins/azurefile.sock
azurefile 11231 11592 root cwd DIR 8,1 4096 2 /
azurefile 11231 11592 root rtd DIR 8,1 4096 2 /
azurefile 11231 11592 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 11231 11592 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 11231 11592 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 11231 11592 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 11231 11592 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 11231 11592 root 1u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11592 root 2u CHR 136,4 0t0 7 /dev/pts/4
azurefile 11231 11592 root 3u unix 0xffff8801b6eecb00 0t0 87392 /run/docker/plugins/azurefile.sock
azurefile 11231 11592 root 4u 0000 0,10 0 8199 anon_inode
azurefile 11231 11592 root 5u unix 0xffff8801b6eef440 0t0 89435 /run/docker/plugins/azurefile.sock
azurefile 11231 11592 root 6u unix 0xffff8801b6ee9680 0t0 117503 /run/docker/plugins/azurefile.sock
azurefile 11231 11592 root 7u unix 0xffff8801b6eed640 0t0 89524 /run/docker/plugins/azurefile.sock
azurefile 11231 11592 root 8u unix 0xffff8801b6eef080 0t0 117595 /run/docker/plugins/azurefile.sock
azurefile 11231 11592 root 9u unix 0xffff8801b5c26180 0t0 117618 /run/docker/plugins/azurefile.sock
There is something really odd going on here. Maybe the main problem was created by docker-compose as i've the defined the volume there and i think docker-compose tries to create the volume every time doing a docker-compose up
. But i could reproduce it like this:
$ sudo service azurefile-dockervolumedriver restart
$ sudo lsof | grep azurefile | wc -l
45
$ docker volume create -d azurefile -o share=foo --name=foo
foo
$ sudo lsof | grep azurefile | wc -l
79
$ docker volume rm foo
foo
$ sudo lsof | grep azurefile | wc -l
79
$ docker volume rm foo
foo
$ sudo lsof | grep azurefile | wc -l
85
$ docker volume rm foo
foo
$ sudo lsof | grep azurefile | wc -l
91
$ sudo lsof | grep azurefile
init 1 root 13w REG 8,1 632 2055 /var/log/upstart/azurefile-dockervolumedriver.log
azurefile 14203 root cwd DIR 8,1 4096 2 /
azurefile 14203 root rtd DIR 8,1 4096 2 /
azurefile 14203 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 14203 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 14203 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 14203 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 14203 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 14203 root 1u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 root 2u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 root 3u unix 0xffff8801b6eea580 0t0 135418 /run/docker/plugins/azurefile.sock
azurefile 14203 root 4u 0000 0,10 0 8199 anon_inode
azurefile 14203 root 5u unix 0xffff8801b5c27bc0 0t0 135679 /run/docker/plugins/azurefile.sock
azurefile 14203 root 7u unix 0xffff8801b6ee8f00 0t0 134564 /run/docker/plugins/azurefile.sock
azurefile 14203 root 8u unix 0xffff8801b5c24ec0 0t0 134580 /run/docker/plugins/azurefile.sock
azurefile 14203 14205 root cwd DIR 8,1 4096 2 /
azurefile 14203 14205 root rtd DIR 8,1 4096 2 /
azurefile 14203 14205 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 14203 14205 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 14203 14205 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 14203 14205 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 14203 14205 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 14203 14205 root 1u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14205 root 2u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14205 root 3u unix 0xffff8801b6eea580 0t0 135418 /run/docker/plugins/azurefile.sock
azurefile 14203 14205 root 4u 0000 0,10 0 8199 anon_inode
azurefile 14203 14205 root 5u unix 0xffff8801b5c27bc0 0t0 135679 /run/docker/plugins/azurefile.sock
azurefile 14203 14205 root 7u unix 0xffff8801b6ee8f00 0t0 134564 /run/docker/plugins/azurefile.sock
azurefile 14203 14205 root 8u unix 0xffff8801b5c24ec0 0t0 134580 /run/docker/plugins/azurefile.sock
azurefile 14203 14206 root cwd DIR 8,1 4096 2 /
azurefile 14203 14206 root rtd DIR 8,1 4096 2 /
azurefile 14203 14206 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 14203 14206 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 14203 14206 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 14203 14206 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 14203 14206 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 14203 14206 root 1u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14206 root 2u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14206 root 3u unix 0xffff8801b6eea580 0t0 135418 /run/docker/plugins/azurefile.sock
azurefile 14203 14206 root 4u 0000 0,10 0 8199 anon_inode
azurefile 14203 14206 root 5u unix 0xffff8801b5c27bc0 0t0 135679 /run/docker/plugins/azurefile.sock
azurefile 14203 14206 root 7u unix 0xffff8801b6ee8f00 0t0 134564 /run/docker/plugins/azurefile.sock
azurefile 14203 14206 root 8u unix 0xffff8801b5c24ec0 0t0 134580 /run/docker/plugins/azurefile.sock
azurefile 14203 14207 root cwd DIR 8,1 4096 2 /
azurefile 14203 14207 root rtd DIR 8,1 4096 2 /
azurefile 14203 14207 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 14203 14207 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 14203 14207 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 14203 14207 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 14203 14207 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 14203 14207 root 1u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14207 root 2u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14207 root 3u unix 0xffff8801b6eea580 0t0 135418 /run/docker/plugins/azurefile.sock
azurefile 14203 14207 root 4u 0000 0,10 0 8199 anon_inode
azurefile 14203 14207 root 5u unix 0xffff8801b5c27bc0 0t0 135679 /run/docker/plugins/azurefile.sock
azurefile 14203 14207 root 7u unix 0xffff8801b6ee8f00 0t0 134564 /run/docker/plugins/azurefile.sock
azurefile 14203 14207 root 8u unix 0xffff8801b5c24ec0 0t0 134580 /run/docker/plugins/azurefile.sock
azurefile 14203 14232 root cwd DIR 8,1 4096 2 /
azurefile 14203 14232 root rtd DIR 8,1 4096 2 /
azurefile 14203 14232 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 14203 14232 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 14203 14232 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 14203 14232 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 14203 14232 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 14203 14232 root 1u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14232 root 2u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14232 root 3u unix 0xffff8801b6eea580 0t0 135418 /run/docker/plugins/azurefile.sock
azurefile 14203 14232 root 4u 0000 0,10 0 8199 anon_inode
azurefile 14203 14232 root 5u unix 0xffff8801b5c27bc0 0t0 135679 /run/docker/plugins/azurefile.sock
azurefile 14203 14232 root 7u unix 0xffff8801b6ee8f00 0t0 134564 /run/docker/plugins/azurefile.sock
azurefile 14203 14232 root 8u unix 0xffff8801b5c24ec0 0t0 134580 /run/docker/plugins/azurefile.sock
azurefile 14203 14233 root cwd DIR 8,1 4096 2 /
azurefile 14203 14233 root rtd DIR 8,1 4096 2 /
azurefile 14203 14233 root txt REG 8,1 9634672 137648 /usr/bin/azurefile-dockervolumedriver
azurefile 14203 14233 root mem REG 8,1 1840928 3113 /lib/x86_64-linux-gnu/libc-2.19.so
azurefile 14203 14233 root mem REG 8,1 141574 3117 /lib/x86_64-linux-gnu/libpthread-2.19.so
azurefile 14203 14233 root mem REG 8,1 149120 3313 /lib/x86_64-linux-gnu/ld-2.19.so
azurefile 14203 14233 root 0u CHR 1,3 0t0 8246 /dev/null
azurefile 14203 14233 root 1u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14233 root 2u CHR 136,2 0t0 5 /dev/pts/2
azurefile 14203 14233 root 3u unix 0xffff8801b6eea580 0t0 135418 /run/docker/plugins/azurefile.sock
azurefile 14203 14233 root 4u 0000 0,10 0 8199 anon_inode
azurefile 14203 14233 root 5u unix 0xffff8801b5c27bc0 0t0 135679 /run/docker/plugins/azurefile.sock
azurefile 14203 14233 root 7u unix 0xffff8801b6ee8f00 0t0 134564 /run/docker/plugins/azurefile.sock
azurefile 14203 14233 root 8u unix 0xffff8801b5c24ec0 0t0 134580 /run/docker/plugins/azurefile.sock
So after every $ docker volume rm foo
there are new handles that are not closed.
Maybe this is related to #20
@sascha-egerer thanks this is very insightful. I will debug.
Thanks for reporting, just fixed this in master
, feel free to compile from source until I release a new version tag.
Looks like there is still a problem. Everytime i start a container the volume is mounted again under the same path.
$ mount
/dev/sda1 on / type ext4 (rw,discard)
proc on /proc type proc (rw,noexec,nosuid,nodev)
sysfs on /sys type sysfs (rw,noexec,nosuid,nodev)
none on /sys/fs/cgroup type tmpfs (rw)
none on /sys/fs/fuse/connections type fusectl (rw)
none on /sys/kernel/debug type debugfs (rw)
none on /sys/kernel/security type securityfs (rw)
udev on /dev type devtmpfs (rw,mode=0755)
devpts on /dev/pts type devpts (rw,noexec,nosuid,gid=5,mode=0620)
tmpfs on /run type tmpfs (rw,noexec,nosuid,size=10%,mode=0755)
none on /run/lock type tmpfs (rw,noexec,nosuid,nodev,size=5242880)
none on /run/shm type tmpfs (rw,nosuid,nodev)
none on /run/user type tmpfs (rw,noexec,nosuid,nodev,size=104857600,mode=0755)
none on /sys/fs/pstore type pstore (rw)
none on /etc/network/interfaces.dynamic.d type tmpfs (rw,noexec,nosuid,nodev,size=64K)
/dev/sdb1 on /mnt type ext4 (rw)
systemd on /sys/fs/cgroup/systemd type cgroup (rw,noexec,nosuid,nodev,none,name=systemd)
//foo.file.core.windows.net/my-file-storage on /run/docker/volumedriver/azurefile/my_my-file-storage type cifs (rw)
//foo.file.core.windows.net/my-config on /run/docker/volumedriver/azurefile/my_my-config type cifs (rw)
//foo.file.core.windows.net/my-config on /run/docker/volumedriver/azurefile/my_my-config type cifs (rw)
//foo.file.core.windows.net/my-file-storage on /run/docker/volumedriver/azurefile/my_my-file-storage type cifs (rw)
//foo.file.core.windows.net/my-file-storage on /run/docker/volumedriver/azurefile/my_my-file-storage type cifs (rw)
yeah this is a separate issue, thanks for opening #23.
Looks like file handles to
/run/docker/plugins/azurefile.sock
are not closed. After (re-)starting some containers with a azurefile volume i get a "Too many open fiels" error.