clearcontainers / runtime

OCI (Open Containers Initiative) compatible runtime using Virtual Machines
Apache License 2.0
589 stars 70 forks source link

path length error using "docker run -d" #877

Open jodh-intel opened 6 years ago

jodh-intel commented 6 years ago

Description of problem

$ sudo docker run -d busybox  true

Expected result

7aee206467a3d5cf46ba5baadc8d7277f13d902c06c7587b89b5e143e1c503ab

Actual result

7aee206467a3d5cf46ba5baadc8d7277f13d902c06c7587b89b5e143e1c503ab
docker: Error response from daemon: oci runtime error: Parent directory path "/run/virtcontainers/pods/run/docker/libcontainerd/containerd/7aee206467a3d5cf46ba5baadc8d7277f13d902c06c7587b89b5e143e1c503ab/init/pid" is too long (134 characters), could not add any path for the QMP socket.

Meta details

Running cc-collect-data.sh version 3.0.12 (commit d9f04c9b648ef8bf1cc39ebba1ac091956c6a86a) at 2017-12-22.10:13:10.457858111.


Runtime is /usr/local/bin/cc-runtime.

cc-env

Output of "/usr/local/bin/cc-runtime cc-env":

[Meta]
  Version = "1.0.6"

[Runtime]
  Debug = false
  [Runtime.Version]
    Semver = "3.0.12"
    Commit = "d9f04c9b648ef8bf1cc39ebba1ac091956c6a86a"
    OCI = "1.0.0-dev"
  [Runtime.Config]
    Path = "/usr/share/defaults/clear-containers/configuration.toml"

[Hypervisor]
  MachineType = "pc"
  Version = "QEMU emulator version 2.7.1(2.7.1+git.d4a337fe91-9.cc), Copyright (c) 2003-2016 Fabrice Bellard and the QEMU Project developers"
  Path = "/usr/bin/qemu-lite-system-x86_64"
  Debug = false

[Image]
  Path = "/usr/share/clear-containers/clear-19790-containers.img"

[Kernel]
  Path = "/usr/share/clear-containers/vmlinuz-4.9.60-82.container"
  Parameters = ""

[Proxy]
  Type = "ccProxy"
  Version = "Version: 3.0.12+git.3c6daa6"
  Path = "/usr/libexec/clear-containers/cc-proxy"
  Debug = true

[Shim]
  Type = "ccShim"
  Version = "shim version: 3.0.12 (commit: d01f9a7)"
  Path = "/usr/libexec/clear-containers/cc-shim"
  Debug = true

[Agent]
  Type = "hyperstart"
  Version = "<<unknown>>"

[Host]
  Kernel = "4.10.0-42-generic"
  CCCapable = true
  [Host.Distro]
    Name = "Ubuntu"
    Version = "17.04"
  [Host.CPU]
    Vendor = "GenuineIntel"
    Model = "Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz"

Runtime config files

Runtime default config files

/etc/clear-containers/configuration.toml
/usr/share/defaults/clear-containers/configuration.toml

Runtime config file contents

Config file /etc/clear-containers/configuration.toml not found Output of "cat "/usr/share/defaults/clear-containers/configuration.toml"":

# XXX: Warning: this file is auto-generated from file "config/configuration.toml.in".

[hypervisor.qemu]
path = "/usr/bin/qemu-lite-system-x86_64"
kernel = "/usr/share/clear-containers/vmlinuz.container"
image = "/usr/share/clear-containers/clear-containers.img"
machine_type = "pc"
# Optional space-separated list of options to pass to the guest kernel.
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
# trouble running pre-2.15 glibc
kernel_params = ""

# Path to the firmware.
# If you want that qemu uses the default firmware leave this option empty
firmware = ""

# Machine accelerators
# comma-separated list of machine accelerators to pass to the hypervisor.
# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
machine_accelerators=""

# Default number of vCPUs per POD/VM:
# unspecified or 0 --> will be set to 1
# < 0              --> will be set to the actual number of physical cores
# > 0 <= 255       --> will be set to the specified number
# > 255            --> will be set to 255
default_vcpus = -1

# Bridges can be used to hot plug devices.
# Limitations:
# * Currently only pci bridges are supported
# * Until 30 devices per bridge can be hot plugged.
# * Until 5 PCI bridges can be cold plugged per VM.
#   This limitation could be a bug in qemu or in the kernel
# Default number of bridges per POD/VM:
# unspecified or 0   --> will be set to 1
# > 1 <= 5           --> will be set to the specified number
# > 5                --> will be set to 5
default_bridges = 1

# Default memory size in MiB for POD/VM.
# If unspecified then it will be set 2048 MiB.
#default_memory = 2048
disable_block_device_use = false

# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked
# This is useful when you want to reserve all the memory
# upfront or in the cases where you want memory latencies
# to be very predictable
# Default false
#enable_mem_prealloc = true

# Enable huge pages for VM RAM, default false
# Enabling this will result in the VM memory
# being allocated using huge pages.
# This is useful when you want to use vhost-user network
# stacks within the container. This will automatically 
# result in memory pre allocation
#enable_hugepages = true

# Enable swap of vm memory. Default false.
# The behaviour is undefined if mem_prealloc is also set to true
#enable_swap = true

# Debug changes the default hypervisor and kernel parameters to
# enable debug output where available.
# Default false
# these logs can be obtained in the cc-proxy logs  when the 
# proxy is set to run in debug mode
# /usr/libexec/clear-containers/cc-proxy -log debug
# or by stopping the cc-proxy service and running the cc-proxy 
# explicitly using the same command line
# 
enable_debug = true

# Disable the customizations done in the runtime when it detects
# that it is running on top a VMM. This will result in the runtime
# behaving as it would when running on bare metal.
# 
#disable_nesting_checks = true

[proxy.cc]
path = "/usr/libexec/clear-containers/cc-proxy"

# If enabled, proxy messages will be sent to the system log
# (default: disabled)
enable_debug = true

[shim.cc]
path = "/usr/libexec/clear-containers/cc-shim"

# If enabled, shim messages will be sent to the system log
# (default: disabled)
enable_debug = true

[runtime]
# If enabled, the runtime will log additional debug messages to the
# system log
# (default: disabled)
enable_debug = true

Logfiles

Runtime logs

Recent runtime problems found in system journal:

time="2017-12-22T10:10:19Z" level=info msg="launching qemu with: [-name pod-c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30 -uuid 2239af5a-a4e3-4717-b845-7f79800597dc -machine pc,accel=kvm,kernel_irqchip,nvdimm -cpu host,pmu=off -qmp unix:/run/virtcontainers/pods/c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30/2239af5a-a4e3-471,server,nowait -qmp unix:/run/virtcontainers/pods/c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30/2239af5a-a4e3-471,server,nowait -m 2048M,slots=2,maxmem=3021M -device virtio-serial-pci,disable-modern=true,id=serial0 -device virtconsole,chardev=charconsole0,id=console0 -chardev socket,id=charconsole0,path=/run/virtcontainers/pods/c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30/console.sock,server,nowait -device nvdimm,id=nv0,memdev=mem0 -object memory-backend-file,id=mem0,mem-path=/usr/share/clear-containers/clear-19790-containers.img,size=235929600 -device pci-bridge,bus=pci.0,id=pci-bridge-0,chassis_nr=1,shpc=on -device virtserialport,chardev=charch0,id=channel0,name=sh.hyper.channel.0 -chardev socket,id=charch0,path=/run/virtcontainers/pods/c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30/hyper.sock,server,nowait -device virtserialport,chardev=charch1,id=channel1,name=sh.hyper.channel.1 -chardev socket,id=charch1,path=/run/virtcontainers/pods/c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30/tty.sock,server,nowait -device virtio-9p-pci,disable-modern=true,fsdev=extra-9p-hyperShared,mount_tag=hyperShared -fsdev local,id=extra-9p-hyperShared,path=/run/hyper/shared/pods/c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30,security_model=none -netdev tap,id=network-0,vhost=on,vhostfds=3:4:5:6:7:8:9:10,fds=11:12:13:14:15:16:17:18 -device driver=virtio-net-pci,netdev=network-0,mac=02:42:ac:11:00:02,disable-modern=true,mq=on,vectors=18 -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=discard -vga none -no-user-config -nodefaults -nographic -daemonize -kernel /usr/share/clear-containers/vmlinuz-4.9.60-82.container -append root=/dev/pmem0p1 rootflags=dax,data=ordered,errors=remount-ro rw rootfstype=ext4 tsc=reliable no_timer_check rcupdate.rcu_expedited=1 i8042.direct=1 i8042.dumbkbd=1 i8042.nopnp=1 i8042.noaux=1 noreplace-smp reboot=k panic=1 console=hvc0 console=hvc1 initcall_debug iommu=off cryptomgr.notests net.ifnames=0 quiet systemd.show_status=false init=/usr/lib/systemd/systemd systemd.unit=clear-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket ip=::::::c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30::off:: -smp 4,cores=4,threads=1,sockets=1]" source=virtcontainers subsystem=qmp
time="2017-12-22T10:10:21Z" level=error msg="Container not running, impossible to signal the container" source=runtime
time="2017-12-22T10:10:21Z" level=error msg="Container ID (c30faf964962e00c4c6892df3748df1e1b7699899745bd640639036e5672bf30) does not exist" source=runtime
time="2017-12-22T10:10:35Z" level=error msg="Parent directory path \"/run/virtcontainers/pods/run/docker/libcontainerd/containerd/734bd3cbc1280d2e9ec75f87a04ed342d135cbb9bb30197bd93003e2fa2ffa14/init/pid\" is too long (134 characters), could not add any path for the QMP socket" source=runtime
time="2017-12-22T10:10:35Z" level=error msg="Container ID (734bd3cbc1280d2e9ec75f87a04ed342d135cbb9bb30197bd93003e2fa2ffa14) does not exist" source=runtime
time="2017-12-22T10:10:35Z" level=error msg="Container ID (734bd3cbc1280d2e9ec75f87a04ed342d135cbb9bb30197bd93003e2fa2ffa14) does not exist" source=runtime
time="2017-12-22T10:12:29Z" level=error msg="Parent directory path \"/run/virtcontainers/pods/run/docker/libcontainerd/containerd/7aee206467a3d5cf46ba5baadc8d7277f13d902c06c7587b89b5e143e1c503ab/init/pid\" is too long (134 characters), could not add any path for the QMP socket" source=runtime
time="2017-12-22T10:12:29Z" level=error msg="Container ID (7aee206467a3d5cf46ba5baadc8d7277f13d902c06c7587b89b5e143e1c503ab) does not exist" source=runtime
time="2017-12-22T10:12:29Z" level=error msg="Container ID (7aee206467a3d5cf46ba5baadc8d7277f13d902c06c7587b89b5e143e1c503ab) does not exist" source=runtime

Proxy logs

No recent proxy problems found in system journal.

Shim logs

No recent shim problems found in system journal.


Container manager details

Have docker

Docker

Output of "docker info":

Containers: 3
 Running: 0
 Paused: 0
 Stopped: 3
Images: 27
Server Version: 17.05.0-ce
Storage Driver: devicemapper
 Pool Name: docker-253:0-1442432-pool
 Pool Blocksize: 65.54kB
 Base Device Size: 10.74GB
 Backing Filesystem: xfs
 Data file: /dev/loop0
 Metadata file: /dev/loop1
 Data Space Used: 1.77GB
 Data Space Total: 107.4GB
 Data Space Available: 21.81GB
 Metadata Space Used: 2.499MB
 Metadata Space Total: 2.147GB
 Metadata Space Available: 2.145GB
 Thin Pool Minimum Free Space: 10.74GB
 Udev Sync Supported: true
 Deferred Removal Enabled: false
 Deferred Deletion Enabled: false
 Deferred Deleted Device Count: 0
 Data loop file: /var/lib/docker/devicemapper/devicemapper/data
 Metadata loop file: /var/lib/docker/devicemapper/devicemapper/metadata
 Library Version: 1.02.136 (2016-11-05)
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins: 
 Volume: local
 Network: bridge host macvlan null overlay
Swarm: inactive
Runtimes: cc-runtime runc
Default Runtime: cc-runtime
Init Binary: docker-init
containerd version: 9048e5e50717ea4497b757314bad98ea3763c145
runc version: 9c2d8d184e5da67c95d601382adf14862e4f2228
init version: 949e6fa
Security Options:
 apparmor
 seccomp
  Profile: default
Kernel Version: 4.10.0-42-generic
Operating System: Ubuntu 17.04
OSType: linux
Architecture: x86_64
CPUs: 4
Total Memory: 1.951GiB
Name: zesty
ID: J5RR:ZIMQ:27F3:HCG3:6QWF:JACF:3UED:5QAV:LCRD:AEDT:FNXO:RRTH
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): true
 File Descriptors: 17
 Goroutines: 22
 System Time: 2017-12-22T10:13:10.532290592Z
 EventsListeners: 0
Registry: https://index.docker.io/v1/
Experimental: false
Insecure Registries:
 127.0.0.0/8
Live Restore Enabled: false

WARNING: devicemapper: usage of loopback devices is strongly discouraged for production use.
         Use `--storage-opt dm.thinpooldev` to specify a custom block storage device.
WARNING: No swap limit support

No kubectl


Packages

Have dpkg Output of "dpkg -l|egrep "(cc-proxy|cc-runtime|cc-shim|clear-containers-image|linux-container|qemu-lite|qemu-system-x86|cc-oci-runtime)"":

ii  cc-proxy                            3.0.12+git.3c6daa6-17               amd64        
ii  cc-runtime                          3.0.12+git.d9f04c9-17               amd64        
ii  cc-runtime-bin                      3.0.12+git.d9f04c9-17               amd64        
ii  cc-runtime-config                   3.0.12+git.d9f04c9-17               amd64        
ii  cc-shim                             3.0.12+git.d01f9a7-17               amd64        
ii  clear-containers-image              19790-43                            amd64        Clear containers image
ii  linux-container                     4.9.60-82                           amd64        linux kernel optimised for container-like workloads.
ii  qemu-lite                           2.7.1+git.d4a337fe91-9              amd64        linux kernel optimised for container-like workloads.
ii  qemu-system-x86                     1:2.8+dfsg-3ubuntu2.7               amd64        QEMU full system emulation binaries (x86)

Have rpm Output of "rpm -qa|egrep "(cc-proxy|cc-runtime|cc-shim|clear-containers-image|linux-container|qemu-lite|qemu-system-x86|cc-oci-runtime)"":



jodh-intel commented 6 years ago

The path length limit is from:

const qmpSockPathSizeLimit = 107

(https://github.com/containers/virtcontainers/blob/master/qemu.go#L85)

devimc commented 6 years ago

I can't reproduce this issue

Meta details

Running cc-collect-data.sh version 3.0.12 (commit 8311a66949584c617fcbed972a1146c275c64e00) at 2017-12-22.08:13:41.751395188-0600.


Runtime is /bin/cc-runtime.

cc-env

Output of "/bin/cc-runtime cc-env":

[Meta]
  Version = "1.0.6"

[Runtime]
  Debug = false
  [Runtime.Version]
    Semver = "3.0.12"
    Commit = "8311a66949584c617fcbed972a1146c275c64e00"
    OCI = "1.0.0-dev"
  [Runtime.Config]
    Path = "/usr/share/defaults/clear-containers/configuration.toml"

[Hypervisor]
  MachineType = "pc"
  Version = "QEMU emulator version 2.7.1(2.7.1+git.d4a337fe91-9.1.cc), Copyright (c) 2003-2016 Fabrice Bellard and the QEMU Project developers"
  Path = "/usr/bin/qemu-lite-system-x86_64"
  Debug = false

[Image]
  Path = "/usr/share/clear-containers/clear-19350-containers.img"

[Kernel]
  Path = "/usr/share/clear-containers/vmlinuz-4.9.60-80.1.container"
  Parameters = ""

[Proxy]
  Type = "ccProxy"
  Version = "Version: 3.0.12-3c6daa636ebb7c981f9449f185a003f7cf9b748e"
  Path = "/usr/libexec/clear-containers/cc-proxy"
  Debug = false

[Shim]
  Type = "ccShim"
  Version = "shim version: 3.0.10 (commit: 0952966)"
  Path = "/usr/libexec/clear-containers/cc-shim"
  Debug = false

[Agent]
  Type = "hyperstart"
  Version = "<<unknown>>"

[Host]
  Kernel = "4.13.16-100.fc25.x86_64"
  CCCapable = true
  [Host.Distro]
    Name = "Fedora"
    Version = "25"
  [Host.CPU]
    Vendor = "GenuineIntel"
    Model = "Intel(R) Core(TM) i7-4700MQ CPU @ 2.40GHz"

Runtime config files

Runtime default config files

/usr/etc/clear-containers/configuration.toml
/usr/share/defaults/clear-containers/configuration.toml

Runtime config file contents

Config file /etc/clear-containers/configuration.toml not found Config file /usr/etc/clear-containers/configuration.toml not found Output of "cat "/usr/share/defaults/clear-containers/configuration.toml"":

# XXX: Warning: this file is auto-generated from file "config/configuration.toml.in".

[hypervisor.qemu]
path = "/usr/bin/qemu-lite-system-x86_64"
kernel = "/usr/share/clear-containers/vmlinuz.container"
image = "/usr/share/clear-containers/clear-containers.img"
machine_type = "pc"
# Optional space-separated list of options to pass to the guest kernel.
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
# trouble running pre-2.15 glibc
kernel_params = ""

# Path to the firmware.
# If you want that qemu uses the default firmware leave this option empty
firmware = ""

# Machine accelerators
# comma-separated list of machine accelerators to pass to the hypervisor.
# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
machine_accelerators=""

# Default number of vCPUs per POD/VM:
# unspecified or 0 --> will be set to 1
# < 0              --> will be set to the actual number of physical cores
# > 0 <= 255       --> will be set to the specified number
# > 255            --> will be set to 255
default_vcpus = -1

# Bridges can be used to hot plug devices.
# Limitations:
# * Currently only pci bridges are supported
# * Until 30 devices per bridge can be hot plugged.
# * Until 5 PCI bridges can be cold plugged per VM.
#   This limitation could be a bug in qemu or in the kernel
# Default number of bridges per POD/VM:
# unspecified or 0   --> will be set to 1
# > 1 <= 5           --> will be set to the specified number
# > 5                --> will be set to 5
default_bridges = 1

# Default memory size in MiB for POD/VM.
# If unspecified then it will be set 2048 MiB.
#default_memory = 2048
disable_block_device_use = false

# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked
# This is useful when you want to reserve all the memory
# upfront or in the cases where you want memory latencies
# to be very predictable
# Default false
#enable_mem_prealloc = true

# Enable huge pages for VM RAM, default false
# Enabling this will result in the VM memory
# being allocated using huge pages.
# This is useful when you want to use vhost-user network
# stacks within the container. This will automatically 
# result in memory pre allocation
#enable_hugepages = true

# Enable swap of vm memory. Default false.
# The behaviour is undefined if mem_prealloc is also set to true
#enable_swap = true

# Debug changes the default hypervisor and kernel parameters to
# enable debug output where available.
# Default false
# these logs can be obtained in the cc-proxy logs  when the 
# proxy is set to run in debug mode
# /usr/libexec/clear-containers/cc-proxy -log debug
# or by stopping the cc-proxy service and running the cc-proxy 
# explicitly using the same command line
# 
#enable_debug = true

# Disable the customizations done in the runtime when it detects
# that it is running on top a VMM. This will result in the runtime
# behaving as it would when running on bare metal.
# 
#disable_nesting_checks = true

[proxy.cc]
path = "/usr/libexec/clear-containers/cc-proxy"

# If enabled, proxy messages will be sent to the system log
# (default: disabled)
#enable_debug = true

[shim.cc]
path = "/usr/libexec/clear-containers/cc-shim"

# If enabled, shim messages will be sent to the system log
# (default: disabled)
#enable_debug = true

[runtime]
# If enabled, the runtime will log additional debug messages to the
# system log
# (default: disabled)
#enable_debug = true