Closed evanfoster closed 4 years ago
I did a run with full debug logs and got some better information.
# Pod ID: 65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64
# Failed container ID: c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4
# Leaked mounts:
vmss-agent-kata1-test-jfitk000000 ~ # mount | grep c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4
/dev/sda9 on /run/kata-containers/shared/sandboxes/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4/rootfs type ext4 (rw,relatime,seclabel)
tmpfs on /run/kata-containers/shared/sandboxes/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-ce656e7119bd2e6b-resolv.conf type tmpfs (rw,nosuid,nodev,seclabel,mode=755)
tmpfs on /run/kata-containers/shared/sandboxes/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-ef35fd5be2b2f2d4-hostname type tmpfs (rw,nosuid,nodev,seclabel,mode=755)
/dev/sda9 on /run/kata-containers/shared/sandboxes/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-aa31a83e3c3e3823-hosts type ext4 (rw,relatime,seclabel)
/dev/sda9 on /run/kata-containers/shared/sandboxes/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-bef093818e424930-termination-log type ext4 (rw,relatime,seclabel)
tmpfs on /run/kata-containers/shared/sandboxes/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-0bf60eeaf94cf0aa-serviceaccount type tmpfs (rw,relatime,seclabel)
# Logs (journalctl -t kata)
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.321711854Z" level=debug msg="converting /run/containers/storage/vfs-containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4/userdata/config.json" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=compatoci
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.322986659Z" level=debug msg="container rootfs: /var/lib/containers/storage/vfs/dir/e6e95191cf033295948b974ea25b6c09d13b6f5aab39d7a03c29124cc3c405e0" source=virtcontainers subsystem=oci
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.32313276Z" level=debug msg="New filesystem store backend for /var/lib/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 backend=filesystem path=/var/lib/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4 source=virtcontainers/store subsystem=store
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.32323576Z" level=debug msg="Creating root directory" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 backend=filesystem path=/var/lib/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4 source=virtcontainers/store subsystem=store
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.323321561Z" level=debug msg="Creating raw directory" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 backend=filesystem path=/var/lib/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4/raw source=virtcontainers/store subsystem=store
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.323421161Z" level=debug msg="New filesystem store backend for /run/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 backend=filesystem path=/run/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4 source=virtcontainers/store subsystem=store
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.323466061Z" level=debug msg="Creating root directory" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 backend=filesystem path=/run/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4 source=virtcontainers/store subsystem=store
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.323532861Z" level=debug msg="Creating raw directory" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 backend=filesystem path=/run/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4/raw source=virtcontainers/store subsystem=store
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.325353469Z" level=debug msg="Replacing OCI mount (/etc/resolv.conf) source /var/run/containers/storage/vfs-containers/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/userdata/resolv.conf with /run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-ce656e7119bd2e6b-resolv.conf" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.325597069Z" level=debug msg="Replacing OCI mount (/etc/hostname) source /var/run/containers/storage/vfs-containers/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/userdata/hostname with /run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-ef35fd5be2b2f2d4-hostname" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.32564947Z" level=debug msg="Replacing OCI mount (/etc/hosts) source /var/lib/kubelet/pods/891f0e51-a0ed-4804-8bb9-8f8369bc3e77/etc-hosts with /run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-aa31a83e3c3e3823-hosts" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.32570667Z" level=debug msg="Replacing OCI mount (/dev/termination-log) source /var/lib/kubelet/pods/891f0e51-a0ed-4804-8bb9-8f8369bc3e77/containers/qemu-7/0061489a with /run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-bef093818e424930-termination-log" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.32574567Z" level=debug msg="Replacing OCI mount (/var/run/secrets/kubernetes.io/serviceaccount) source /var/lib/kubelet/pods/891f0e51-a0ed-4804-8bb9-8f8369bc3e77/volumes/kubernetes.io~secret/default-token-svjv6 with /run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-0bf60eeaf94cf0aa-serviceaccount" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.326122672Z" level=info msg="Using sandbox shm" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 shm-size=67108864 source=virtcontainers subsystem=kata_agent
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.326477173Z" level=debug msg="sending request" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 name=grpc.CreateContainerRequest req="container_id:\"c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4\" exec_id:\"c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4\" storages:<driver:\"local\" source:\"local\" fstype:\"local\" options:\"mode=0777\" mount_point:\"/run/kata-containers/shared/containers/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/rootfs/local/test-volume\" > OCI:<Version:\"1.0.1-dev\" Process:<Terminal:true User:<AdditionalGids:0 AdditionalGids:1 AdditionalGids:2 AdditionalGids:3 AdditionalGids:4 AdditionalGids:6 AdditionalGids:10 AdditionalGids:11 AdditionalGids:20 AdditionalGids:26 AdditionalGids:27 > Args:\"ash\" Env:\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\" Env:\"TERM=xterm\" Env:\"TERM=xterm\" Env:\"HOSTNAME=qemu-guest-empty-dir-68ccb59f6-ss48w\" Env:\"KUBERNETES_PORT_443_TCP=tcp://192.168.0.1:443\" Env:\"KUBERNETES_PORT_443_TCP_PROTO=tcp\" Env:\"KUBERNETES_PORT_443_TCP_PORT=443\" Env:\"KUBERNETES_PORT_443_TCP_ADDR=192.168.0.1\" Env:\"KUBERNETES_SERVICE_HOST=192.168.0.1\" Env:\"KUBERNETES_SERVICE_PORT=443\" Env:\"KUBERNETES_SERVICE_PORT_HTTPS=443\" Env:\"KUBERNETES_PORT=tcp://192.168.0.1:443\" Env:\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\" Cwd:\"/\" Capabilities:<Bounding:\"CAP_CHOWN\" Bounding:\"CAP_DAC_OVERRIDE\" Bounding:\"CAP_FSETID\" Bounding:\"CAP_FOWNER\" Bounding:\"CAP_NET_RAW\" Bounding:\"CAP_SETGID\" Bounding:\"CAP_SETUID\" Bounding:\"CAP_SETPCAP\" Bounding:\"CAP_NET_BIND_SERVICE\" Bounding:\"CAP_SYS_CHROOT\" Bounding:\"CAP_KILL\" Effective:\"CAP_CHOWN\" Effective:\"CAP_DAC_OVERRIDE\" Effective:\"CAP_FSETID\" Effective:\"CAP_FOWNER\" Effective:\"CAP_NET_RAW\" Effective:\"CAP_SETGID\" Effective:\"CAP_SETUID\" Effective:\"CAP_SETPCAP\" Effective:\"CAP_NET_BIND_SERVICE\" Effective:\"CAP_SYS_CHROOT\" Effective:\"CAP_KILL\" Inheritable:\"CAP_CHOWN\" Inheritable:\"CAP_DAC_OVERRIDE\" Inheritable:\"CAP_FSETID\" Inheritable:\"CAP_FOWNER\" Inheritable:\"CAP_NET_RAW\" Inheritable:\"CAP_SETGID\" Inheritable:\"CAP_SETUID\" Inheritable:\"CAP_SETPCAP\" Inheritable:\"CAP_NET_BIND_SERVICE\" Inheritable:\"CAP_SYS_CHROOT\" Inheritable:\"CAP_KILL\" Permitted:\"CAP_CHOWN\" Permitted:\"CAP_DAC_OVERRIDE\" Permitted:\"CAP_FSETID\" Permitted:\"CAP_FOWNER\" Permitted:\"CAP_NET_RAW\" Permitted:\"CAP_SETGID\" Permitted:\"CAP_SETUID\" Permitted:\"CAP_SETPCAP\" Permitted:\"CAP_NET_BIND_SERVICE\" Permitted:\"CAP_SYS_CHROOT\" Permitted:\"CAP_KILL\" > OOMScoreAdj:997 > Root:<Path:\"/run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4/rootfs\" > Hostname:\"qemu-guest-empty-dir-68ccb59f6-ss48w\" Mounts:<destination:\"/proc\" source:\"proc\" type:\"proc\" options:\"nosuid\" options:\"noexec\" options:\"nodev\" > Mounts:<destination:\"/dev\" source:\"tmpfs\" type:\"tmpfs\" options:\"nosuid\" options:\"strictatime\" options:\"mode=755\" options:\"size=65536k\" > Mounts:<destination:\"/dev/pts\" source:\"devpts\" type:\"devpts\" options:\"nosuid\" options:\"noexec\" options:\"newinstance\" options:\"ptmxmode=0666\" options:\"mode=0620\" options:\"gid=5\" > Mounts:<destination:\"/dev/mqueue\" source:\"mqueue\" type:\"mqueue\" options:\"nosuid\" options:\"noexec\" options:\"nodev\" > Mounts:<destination:\"/sys\" source:\"sysfs\" type:\"sysfs\" options:\"nosuid\" options:\"noexec\" options:\"nodev\" options:\"ro\" > Mounts:<destination:\"/sys/fs/cgroup\" source:\"cgroup\" type:\"cgroup\" options:\"nosuid\" options:\"noexec\" options:\"nodev\" options:\"relatime\" options:\"ro\" > Mounts:<destination:\"/dev/shm\" source:\"/run/kata-containers/sandbox/shm\" type:\"bind\" options:\"rbind\" > Mounts:<destination:\"/etc/resolv.conf\" source:\"/run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-ce656e7119bd2e6b-resolv.conf\" type:\"bind\" options:\"bind\" options:\"nodev\" options:\"nosuid\" options:\"noexec\" > Mounts:<destination:\"/etc/hostname\" source:\"/run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-ef35fd5be2b2f2d4-hostname\" type:\"bind\" options:\"rw\" options:\"bind\" > Mounts:<destination:\"/test-volume\" source:\"/run/kata-containers/shared/containers/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/rootfs/local/test-volume\" type:\"local\" options:\"rw\" options:\"rbind\" options:\"rprivate\" options:\"bind\" > Mounts:<destination:\"/etc/hosts\" source:\"/run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-aa31a83e3c3e3823-hosts\" type:\"bind\" options:\"rw\" options:\"rbind\" options:\"rprivate\" options:\"bind\" > Mounts:<destination:\"/dev/termination-log\" source:\"/run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-bef093818e424930-termination-log\" type:\"bind\" options:\"rw\" options:\"rbind\" options:\"rprivate\" options:\"bind\" > Mounts:<destination:\"/var/run/secrets/kubernetes.io/serviceaccount\" source:\"/run/kata-containers/shared/containers/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4-0bf60eeaf94cf0aa-serviceaccount\" type:\"bind\" options:\"ro\" options:\"rbind\" options:\"rprivate\" options:\"bind\" > Annotations:<key:\"io.container.manager\" value:\"cri-o\" > Annotations:<key:\"io.kubernetes.container.hash\" value:\"1a9d6c05\" > Annotations:<key:\"io.kubernetes.container.name\" value:\"qemu-7\" > Annotations:<key:\"io.kubernetes.container.restartCount\" value:\"0\" > Annotations:<key:\"io.kubernetes.container.terminationMessagePath\" value:\"/dev/termination-log\" > Annotations:<key:\"io.kubernetes.container.terminationMessagePolicy\" value:\"File\" > Annotations:<key:\"io.kubernetes.cri-o.Annotations\" value:\"{\\\"io.kubernetes.container.hash\\\":\\\"1a9d6c05\\\",\\\"io.kubernetes.container.restartCount\\\":\\\"0\\\",\\\"io.kubernetes.container.terminationMessagePath\\\":\\\"/dev/termination-log\\\",\\\"io.kubernetes.container.terminationMessagePolicy\\\":\\\"File\\\",\\\"io.kubernetes.pod.terminationGracePeriod\\\":\\\"30\\\"}\" > Annotations:<key:\"io.kubernetes.cri-o.ContainerID\" value:\"c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4\" > Annotations:<key:\"io.kubernetes.cri-o.ContainerType\" value:\"container\" > Annotations:<key:\"io.kubernetes.cri-o.Created\" value:\"2020-07-13T17:07:55.288062424Z\" > Annotations:<key:\"io.kubernetes.cri-o.IP.0\" value:\"172.16.3.88\" > Annotations:<key:\"io.kubernetes.cri-o.Image\" value:\"docker.io/dmonakhov/alpine-fio@sha256:3559d2fc76edc51c4e32a80e97de1161c72625bb97fd6d531205b9efd2776cc4\" > Annotations:<key:\"io.kubernetes.cri-o.ImageName\" value:\"docker.io/dmonakhov/alpine-fio:latest\" > Annotations:<key:\"io.kubernetes.cri-o.ImageRef\" value:\"docker.io/dmonakhov/alpine-fio@sha256:3559d2fc76edc51c4e32a80e97de1161c72625bb97fd6d531205b9efd2776cc4\" > Annotations:<key:\"io.kubernetes.cri-o.Labels\" value:\"{\\\"io.kubernetes.container.name\\\":\\\"qemu-7\\\",\\\"io.kubernetes.pod.name\\\":\\\"qemu-guest-empty-dir-68ccb59f6-ss48w\\\",\\\"io.kubernetes.pod.namespace\\\":\\\"default\\\",\\\"io.kubernetes.pod.uid\\\":\\\"891f0e51-a0ed-4804-8bb9-8f8369bc3e77\\\"}\" > Annotations:<key:\"io.kubernetes.cri-o.LogPath\" value:\"/var/log/pods/default_qemu-guest-empty-dir-68ccb59f6-ss48w_891f0e51-a0ed-4804-8bb9-8f8369bc3e77/qemu-7/0.log\" > Annotations:<key:\"io.kubernetes.cri-o.Metadata\" value:\"{\\\"name\\\":\\\"qemu-7\\\"}\" > Annotations:<key:\"io.kubernetes.cri-o.MountPoint\" value:\"/var/lib/containers/storage/vfs/dir/e6e95191cf033295948b974ea25b6c09d13b6f5aab39d7a03c29124cc3c405e0\" > Annotations:<key:\"io.kubernetes.cri-o.Name\" value:\"k8s_qemu-7_qemu-guest-empty-dir-68ccb59f6-ss48w_default_891f0e51-a0ed-4804-8bb9-8f8369bc3e77_0\" > Annotations:<key:\"io.kubernetes.cri-o.ResolvPath\" value:\"/var/run/containers/storage/vfs-containers/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/userdata/resolv.conf\" > Annotations:<key:\"io.kubernetes.cri-o.SandboxID\" value:\"65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64\" > Annotations:<key:\"io.kubernetes.cri-o.SandboxName\" value:\"k8s_POD_qemu-guest-empty-dir-68ccb59f6-ss48w_default_891f0e51-a0ed-4804-8bb9-8f8369bc3e77_0\" > Annotations:<key:\"io.kubernetes.cri-o.SeccompProfilePath\" value:\"\" > Annotations:<key:\"io.kubernetes.cri-o.Stdin\" value:\"true\" > Annotations:<key:\"io.kubernetes.cri-o.StdinOnce\" value:\"false\" > Annotations:<key:\"io.kubernetes.cri-o.TTY\" value:\"true\" > Annotations:<key:\"io.kubernetes.cri-o.Volumes\" value:\"[{\\\"container_path\\\":\\\"/test-volume\\\",\\\"host_path\\\":\\\"/var/lib/kubelet/pods/891f0e51-a0ed-4804-8bb9-8f8369bc3e77/volumes/kubernetes.io~empty-dir/test-volume\\\",\\\"readonly\\\":false},{\\\"container_path\\\":\\\"/etc/hosts\\\",\\\"host_path\\\":\\\"/var/lib/kubelet/pods/891f0e51-a0ed-4804-8bb9-8f8369bc3e77/etc-hosts\\\",\\\"readonly\\\":false},{\\\"container_path\\\":\\\"/dev/termination-log\\\",\\\"host_path\\\":\\\"/var/lib/kubelet/pods/891f0e51-a0ed-4804-8bb9-8f8369bc3e77/containers/qemu-7/0061489a\\\",\\\"readonly\\\":false},{\\\"container_path\\\":\\\"/var/run/secrets/kubernetes.io/serviceaccount\\\",\\\"host_path\\\":\\\"/var/lib/kubelet/pods/891f0e51-a0ed-4804-8bb9-8f8369bc3e77/volumes/kubernetes.io~secret/default-token-svjv6\\\",\\\"readonly\\\":true}]\" > Annotations:<key:\"io.kubernetes.pod.name\" value:\"qemu-guest-empty-dir-68ccb59f6-ss48w\" > Annotations:<key:\"io.kubernetes.pod.namespace\" value:\"default\" > Annotations:<key:\"io.kubernetes.pod.terminationGracePeriod\" value:\"30\" > Annotations:<key:\"io.kubernetes.pod.uid\" value:\"891f0e51-a0ed-4804-8bb9-8f8369bc3e77\" > Linux:<Resources:<Memory:<Limit:1073741824 > CPU:<Shares:256 Quota:50000 Period:100000 > > CgroupsPath:\"/kubepods/burstable/pod891f0e51-a0ed-4804-8bb9-8f8369bc3e77/crio-c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4\" Namespaces:<Type:\"ipc\" > Namespaces:<Type:\"uts\" > Namespaces:<Type:\"mount\" > MaskedPaths:\"/proc/acpi\" MaskedPaths:\"/proc/kcore\" MaskedPaths:\"/proc/keys\" MaskedPaths:\"/proc/latency_stats\" MaskedPaths:\"/proc/timer_list\" MaskedPaths:\"/proc/timer_stats\" MaskedPaths:\"/proc/sched_debug\" MaskedPaths:\"/proc/scsi\" MaskedPaths:\"/sys/firmware\" ReadonlyPaths:\"/proc/asound\" ReadonlyPaths:\"/proc/bus\" ReadonlyPaths:\"/proc/fs\" ReadonlyPaths:\"/proc/irq\" ReadonlyPaths:\"/proc/sys\" ReadonlyPaths:\"/proc/sysrq-trigger\" > > " source=virtcontainers subsystem=kata_agent
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.335618908Z" level=debug msg="reading guest console" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent vmconsole="[ 5.418379] pci 0000:00:02.0: PCI bridge to [bus 01]"
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.33605071Z" level=debug msg="reading guest console" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent vmconsole="[ 5.418617] pci 0000:00:02.0: bridge window [io 0xc000-0xcfff]"
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.337947017Z" level=debug msg="reading guest console" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent vmconsole="[ 5.420681] pci 0000:00:02.0: bridge window [mem 0xfe800000-0xfe9fffff]"
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.339381923Z" level=debug msg="reading guest console" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent vmconsole="[ 5.422025] pci 0000:00:02.0: bridge window [mem 0xc0000000-0xc01fffff 64bit pref]"
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.448430045Z" level=debug msg="reading guest console" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent vmconsole="time=\"2020-07-13T17:07:55.437613346Z\" level=info msg=\"ignoring unexpected signal\" debug_console=false name=kata-agent pid=40 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 signal=\"child exited\" source=agent"
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.448573646Z" level=debug msg="reading guest console" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=kata_agent vmconsole="time=\"2020-07-13T17:07:55.437822275Z\" level=info msg=\"ignoring unexpected signal\" debug_console=false name=kata-agent pid=40 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 signal=\"child exited\" source=agent"
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.457076379Z" level=debug msg="Setting container state from to ready" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=container
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.457182079Z" level=debug msg="Request to hypervisor to update vCPUs" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 cpus-sandbox=5 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=sandbox
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.457217879Z" level=debug msg="Sandbox CPUs: 5" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=sandbox
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.457246679Z" level=debug msg="Request to hypervisor to update memory" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 memory-sandbox-size-byte=9663676416 sandbox=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=sandbox
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.457280579Z" level=debug msg="requested memory hotplug" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 hotplug=memory hotplug-memory-mb=1024 source=virtcontainers subsystem=qemu
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.45730898Z" level=debug msg="Requested to add memory: 1024 MB" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 hotplug=memory operation=add source=virtcontainers subsystem=qemu
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.45739538Z" level=info msg="{\"execute\":\"query-memory-devices\"}" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.458355584Z" level=info msg="{\"return\": [{\"type\": \"nvdimm\", \"data\": {\"memdev\": \"/objects/mem0\", \"hotplugged\": false, \"addr\": 4294967296, \"hotpluggable\": true, \"size\": 134217728, \"slot\": 0, \"node\": 0, \"id\": \"nv0\"}}, {\"type\": \"dimm\", \"data\": {\"memdev\": \"/objects/mem1\", \"hotplugged\": true, \"addr\": 4429185024, \"hotpluggable\": true, \"size\": 1073741824, \"slot\": 1, \"node\": 0, \"id\": \"dimmmem1\"}}, {\"type\": \"dimm\", \"data\": {\"memdev\": \"/objects/mem2\", \"hotplugged\": true, \"addr\": 5502926848, \"hotpluggable\": true, \"size\": 1073741824, \"slot\": 2, \"node\": 0, \"id\": \"dimmmem2\"}}, {\"type\": \"dimm\", \"data\": {\"memdev\": \"/objects/mem3\", \"hotplugged\": true, \"addr\": 6576668672, \"hotpluggable\": true, \"size\": 1073741824, \"slot\": 3, \"node\": 0, \"id\": \"dimmmem3\"}}, {\"type\": \"dimm\", \"data\": {\"memdev\": \"/objects/mem4\", \"hotplugged\": true, \"addr\": 7650410496, \"hotpluggable\": true, \"size\": 1073741824, \"slot\": 4, \"node\": 0, \"id\": \"dimmmem4\"}}, {\"type\": \"dimm\", \"data\": {\"memdev\": \"/objects/mem5\", \"hotplugged\": true, \"addr\": 8724152320, \"hotpluggable\": true, \"size\": 1073741824, \"slot\": 5, \"node\": 0, \"id\": \"dimmmem5\"}}, {\"type\": \"dimm\", \"data\": {\"memdev\": \"/objects/mem6\", \"hotplugged\": true, \"addr\": 9797894144, \"hotpluggable\": true, \"size\": 1073741824, \"slot\": 6, \"node\": 0, \"id\": \"dimmmem6\"}}]}" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.458679385Z" level=info msg="{\"arguments\":{\"id\":\"mem7\",\"props\":{\"mem-path\":\"/dev/shm\",\"share\":true,\"size\":1073741824},\"qom-type\":\"memory-backend-file\"},\"execute\":\"object-add\"}" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.45990189Z" level=info msg="{\"return\": {}}" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.45997459Z" level=info msg="{\"arguments\":{\"driver\":\"pc-dimm\",\"id\":\"dimmmem7\",\"memdev\":\"mem7\"},\"execute\":\"device_add\"}" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.461078594Z" level=info msg="{\"error\": {\"class\": \"GenericError\", \"desc\": \"a used vhost backend has no free memory slots left\"}}" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.461191795Z" level=error msg="Unable to hotplug memory device: QMP command failed: a used vhost backend has no free memory slots left" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.461245195Z" level=info msg="{\"arguments\":{\"id\":\"mem7\"},\"execute\":\"object-del\"}" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.461748297Z" level=info msg="{\"return\": {}}" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 source=virtcontainers subsystem=qmp
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.461805097Z" level=error msg="hotplug memory" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 error="QMP command failed: a used vhost backend has no free memory slots left" source=virtcontainers subsystem=qemu
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.461836497Z" level=debug msg="Deleting files" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 backend=filesystem path=/var/lib/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4 source=virtcontainers/store subsystem=store
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.462054998Z" level=debug msg="Deleting files" ID=65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64 backend=filesystem path=/run/vc/sbs/65b2c1c138395088afc662e962d102a1138a43e1e2024af8fd3597ff72c57a64/c7dbdcb9795cbf6011c55c20a1f6fbc8b7dd3e911bc0cb5dcf294fb0b0fdf9b4 source=virtcontainers/store subsystem=store
Jul 13 17:07:55 vmss-agent-kata1-test-jfitk000000 kata[3976491]: time="2020-07-13T17:07:55.462264399Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
I'll do some more poking around and see what I find.
Well, I think I found the problem.
This function is being called: https://github.com/kata-containers/runtime/blob/a885b1bbf9069f8cfe10b21ba4314f5ef430b29a/virtcontainers/sandbox.go#L1137-L1190
Container creation is failing at
err = s.updateResources()
if err != nil {
return nil, err
}
and the deferred function is calling removeContainer
:
defer func() {
// Rollback if error happens.
if err != nil {
s.removeContainer(c.id)
}
}()
Unfortunately, removeContainer
doesn't actually do any cleanup on the system itself: https://github.com/kata-containers/runtime/blob/a885b1bbf9069f8cfe10b21ba4314f5ef430b29a/virtcontainers/sandbox.go#L774-L791
I'm guessing that another call performing some cleanup needs to be added to the error handling in CreateContainer
.
I was able to stop leaks from happening by modifying the deferred function in CreateContainer
:
defer func() {
// Rollback if error happens.
if err != nil {
s.Logger().Warningf("Container %q could not be created, stopping it", contConfig.ID)
if err = c.stop(false); err != nil { // Should this be a force stop?
s.Logger().WithError(err).WithField("container-id", c.id).WithField("sandboxid", s.id).Warning("Could not delete container")
}
s.Logger().WithField("container-id", c.id).WithField("sandboxid", s.id).Info("Container was stopped. Removing from sandbox store")
s.removeContainer(c.id)
}
}()
I'm going to leave a pod running in a bad state for a bit and see if anything explodes.
@evanfoster, amazing! Please, open us a pull request and I'll review and have the patch backported to the correct branches!
Can do! Quick question, however. Should I be setting force
to true
when I call c.stop
in this case? Not sure what the general feeling is for things like that.
Yes, IMHO, we do should force it. @devimc, what do you think?
Description of problem
When using the same setup as #2795, I found that sandbox mounts weren't being cleaned up, leading to a massive number of mountpoints (20,000 mounts in ~2 hours). For example:
I tested with @fidencio 's fix for #2719 (https://github.com/cri-o/cri-o/pull/3924) but continued to have the same issue.
I'm not 100% sure, but I believe this is only an issue for containers in pods that are affected by #2795.
Expected result
Container sandboxes are cleaned up as each container is deleted.
Actual result
Sandbox mounts leak.
I have appended some interesting logs to the end of the output of
kata-collect-data.sh
.Show kata-collect-data.sh details
# Meta details Running `kata-collect-data.sh` version `1.11.2-adobe (commit 9dd46e7244ec94345a3181427da818c4ae49b9a9-dirty)` at `2020-07-07.19:43:40.798586627+0000`. --- Runtime is `/opt/kata/bin/kata-runtime`. # `kata-env` Output of "`/opt/kata/bin/kata-runtime kata-env`": ```toml [Meta] Version = "1.0.24" [Runtime] Debug = false Trace = false DisableGuestSeccomp = true DisableNewNetNs = false SandboxCgroupOnly = false Path = "/opt/kata/bin/kata-runtime" [Runtime.Version] OCI = "1.0.1-dev" [Runtime.Version.Version] Semver = "1.11.2-adobe" Major = 1 Minor = 11 Patch = 2 Commit = "9dd46e7244ec94345a3181427da818c4ae49b9a9-dirty" [Runtime.Config] Path = "/opt/kata/share/defaults/kata-containers/configuration-qemu-virtiofs.toml" [Hypervisor] MachineType = "pc" Version = "QEMU emulator version 4.1.0 (kata-static)\nCopyright (c) 2003-2019 Fabrice Bellard and the QEMU Project developers" Path = "/opt/kata/bin/qemu-virtiofs-system-x86_64" BlockDeviceDriver = "virtio-scsi" EntropySource = "/dev/urandom" SharedFS = "virtio-fs" VirtioFSDaemon = "/opt/kata/bin/virtiofsd" Msize9p = 8192 MemorySlots = 500 PCIeRootPort = 0 HotplugVFIOOnRootBus = false Debug = false UseVSock = true [Image] Path = "/opt/kata/share/kata-containers/kata-containers-image_clearlinux_1.11.2_agent_abb7149e49.img" [Kernel] Path = "/opt/kata/share/kata-containers/vmlinuz-virtio-fs-dev-74-virtiofs" Parameters = "systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket scsi_mod.scan=none" [Initrd] Path = "" [Proxy] Type = "noProxy" Path = "" Debug = false [Proxy.Version] Semver = "" Major = 0 Minor = 0 Patch = 0 Commit = "" [Shim] Type = "kataShim" Path = "/opt/kata/libexec/kata-containers/kata-shim" Debug = false [Shim.Version] Semver = "1.11.2-5ccc2cdabbb5fed33124c0b87ccecd058f7adc19" Major = 1 Minor = 11 Patch = 2 Commit = "<>"
[Agent]
Type = "kata"
Debug = false
Trace = false
TraceMode = ""
TraceType = ""
[Host]
Kernel = "4.19.106-flatcar"
Architecture = "amd64"
VMContainerCapable = true
SupportVSocks = true
[Host.Distro]
Name = "Flatcar Container Linux by Kinvolk"
Version = "2345.3.0"
[Host.CPU]
Vendor = "GenuineIntel"
Model = "Intel(R) Xeon(R) CPU E5-2673 v4 @ 2.30GHz"
[Netmon]
Path = "/opt/kata/libexec/kata-containers/kata-netmon"
Debug = false
Enable = false
[Netmon.Version]
Semver = "1.11.2-adobe"
Major = 1
Minor = 11
Patch = 2
Commit = "<>"
```
---
# Runtime config files
## Runtime default config files
```
/etc/kata-containers/configuration.toml
/opt/kata/share/defaults/kata-containers/configuration.toml
```
## Runtime config file contents
Output of "`cat "/etc/kata-containers/configuration.toml"`":
```toml
# Copyright (c) 2017-2019 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
# XXX: WARNING: this file is auto-generated.
# XXX:
# XXX: Source file: "cli/config/configuration-qemu-virtiofs.toml.in"
# XXX: Project:
# XXX: Name: Kata Containers
# XXX: Type: kata
[hypervisor.qemu]
path = "/opt/kata/bin/qemu-virtiofs-system-x86_64"
kernel = "/opt/kata/share/kata-containers/vmlinuz-virtiofs.container"
image = "/opt/kata/share/kata-containers/kata-containers.img"
machine_type = "pc"
# Optional space-separated list of options to pass to the guest kernel.
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
# trouble running pre-2.15 glibc.
#
# WARNING: - any parameter specified here will take priority over the default
# parameter value of the same name used to start the virtual machine.
# Do not set values here unless you understand the impact of doing so as you
# may stop the virtual machine from booting.
# To see the list of default parameters, enable hypervisor debug, create a
# container and look for 'default-kernel-parameters' log entries.
kernel_params = ""
# Path to the firmware.
# If you want that qemu uses the default firmware leave this option empty
firmware = ""
# Machine accelerators
# comma-separated list of machine accelerators to pass to the hypervisor.
# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
machine_accelerators=""
# Default number of vCPUs per SB/VM:
# unspecified or 0 --> will be set to 1
# < 0 --> will be set to the actual number of physical cores
# > 0 <= number of physical cores --> will be set to the specified number
# > number of physical cores --> will be set to the actual number of physical cores
default_vcpus = 1
# Default maximum number of vCPUs per SB/VM:
# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
# of vCPUs supported by KVM if that number is exceeded
# > 0 <= number of physical cores --> will be set to the specified number
# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number
# of vCPUs supported by KVM if that number is exceeded
# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
# the actual number of physical cores is greater than it.
# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
# can be added to a SB/VM, but the memory footprint will be big. Another example, with
# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
# unless you know what are you doing.
default_maxvcpus = 0
# Bridges can be used to hot plug devices.
# Limitations:
# * Currently only pci bridges are supported
# * Until 30 devices per bridge can be hot plugged.
# * Until 5 PCI bridges can be cold plugged per VM.
# This limitation could be a bug in qemu or in the kernel
# Default number of bridges per SB/VM:
# unspecified or 0 --> will be set to 1
# > 1 <= 5 --> will be set to the specified number
# > 5 --> will be set to 5
default_bridges = 1
# Default memory size in MiB for SB/VM.
# If unspecified then it will be set 2048 MiB.
default_memory = 2048
#
# Default memory slots per SB/VM.
# If unspecified then it will be set 10.
# This is will determine the times that memory will be hotadded to sandbox/VM.
memory_slots = 500
# The size in MiB will be plused to max memory of hypervisor.
# It is the memory address space for the NVDIMM devie.
# If set block storage driver (block_device_driver) to "nvdimm",
# should set memory_offset to the size of block device.
# Default 0
#memory_offset = 0
# Disable block device from being used for a container's rootfs.
# In case of a storage driver like devicemapper where a container's
# root file system is backed by a block device, the block device is passed
# directly to the hypervisor for performance reasons.
# This flag prevents the block device from being passed to the hypervisor,
# 9pfs is used instead to pass the rootfs.
disable_block_device_use = false
# Shared file system type:
# - virtio-fs (default)
# - virtio-9p
shared_fs = "virtio-fs"
# Path to vhost-user-fs daemon.
virtio_fs_daemon = "/opt/kata/bin/virtiofsd"
# Default size of DAX cache in MiB
virtio_fs_cache_size = 0
# Extra args for virtiofsd daemon
#
# Format example:
# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
#
# see `virtiofsd -h` for possible options.
virtio_fs_extra_args = []
# Cache mode:
#
# - none
# Metadata, data, and pathname lookup are not cached in guest. They are
# always fetched from host and any changes are immediately pushed to host.
#
# - auto
# Metadata and pathname lookup cache expires after a configured amount of
# time (default is 1 second). Data is cached while the file is open (close
# to open consistency).
#
# - always
# Metadata, data, and pathname lookup are cached in guest and never expire.
virtio_fs_cache = "always"
# Block storage driver to be used for the hypervisor in case the container
# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
# or nvdimm.
block_device_driver = "virtio-scsi"
# Specifies cache-related options will be set to block devices or not.
# Default false
#block_device_cache_set = true
# Specifies cache-related options for block devices.
# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
# Default false
#block_device_cache_direct = true
# Specifies cache-related options for block devices.
# Denotes whether flush requests for the device are ignored.
# Default false
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
#
enable_iothreads = false
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked
# This is useful when you want to reserve all the memory
# upfront or in the cases where you want memory latencies
# to be very predictable
# Default false
#enable_mem_prealloc = true
# Enable huge pages for VM RAM, default false
# Enabling this will result in the VM memory
# being allocated using huge pages.
# This is useful when you want to use vhost-user network
# stacks within the container. This will automatically
# result in memory pre allocation
#enable_hugepages = true
# Enable vhost-user storage device, default false
# Enabling this will result in some Linux reserved block type
# major range 240-254 being chosen to represent vhost-user devices.
enable_vhost_user_store = false
# The base directory specifically used for vhost-user devices.
# Its sub-path "block" is used for block devices; "block/sockets" is
# where we expect vhost-user sockets to live; "block/devices" is where
# simulated block device nodes for vhost-user devices to live.
vhost_user_store_path = "/var/run/kata-containers/vhost-user"
# Enable file based guest memory support. The default is an empty string which
# will disable this feature. In the case of virtio-fs, this is enabled
# automatically and '/dev/shm' is used as the backing folder.
# This option will be ignored if VM templating is enabled.
#file_mem_backend = ""
# Enable swap of vm memory. Default false.
# The behaviour is undefined if mem_prealloc is also set to true
#enable_swap = true
# This option changes the default hypervisor and kernel parameters
# to enable debug output where available. This extra output is added
# to the proxy logs, but only when proxy debug is also enabled.
#
# Default false
#enable_debug = true
# Disable the customizations done in the runtime when it detects
# that it is running on top a VMM. This will result in the runtime
# behaving as it would when running on bare metal.
#
#disable_nesting_checks = true
# This is the msize used for 9p shares. It is the number of bytes
# used for 9p packet payload.
#msize_9p = 8192
# If true and vsocks are supported, use vsocks to communicate directly
# with the agent and no proxy is started, otherwise use unix
# sockets and start a proxy to communicate with the agent.
# Default false
use_vsock = true
# If false and nvdimm is supported, use nvdimm device to plug guest image.
# Otherwise virtio-block device is used.
# Default false
#disable_image_nvdimm = true
# VFIO devices are hotplugged on a bridge by default.
# Enable hotplugging on root bus. This may be required for devices with
# a large PCI bar, as this is a current limitation with hotplugging on
# a bridge. This value is valid for "pc" machine type.
# Default false
#hotplug_vfio_on_root_bus = true
# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
# security (vhost-net runs ring0) for network I/O performance.
#disable_vhost_net = true
#
# Default entropy source.
# The path to a host source of entropy (including a real hardware RNG)
# /dev/urandom and /dev/random are two main options.
# Be aware that /dev/random is a blocking source of entropy. If the host
# runs out of entropy, the VMs boot time will increase leading to get startup
# timeouts.
# The source of entropy /dev/urandom is non-blocking and provides a
# generally acceptable source of entropy. It should work well for pretty much
# all practical purposes.
#entropy_source= "/dev/urandom"
# Path to OCI hook binaries in the *guest rootfs*.
# This does not affect host-side hooks which must instead be added to
# the OCI spec passed to the runtime.
#
# You can create a rootfs with hooks by customizing the osbuilder scripts:
# https://github.com/kata-containers/osbuilder
#
# Hooks must be stored in a subdirectory of guest_hook_path according to their
# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}".
# The agent will scan these directories for executable files and add them, in
# lexicographical order, to the lifecycle of the guest container.
# Hooks are executed in the runtime namespace of the guest. See the official documentation:
# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
# Warnings will be logged if any error is encountered will scanning for hooks,
# but it will not abort container execution.
#guest_hook_path = "/usr/share/oci/hooks"
[factory]
# VM templating support. Once enabled, new VMs are created from template
# using vm cloning. They will share the same initial kernel, initramfs and
# agent memory by mapping it readonly. It helps speeding up new container
# creation and saves a lot of memory if there are many kata containers running
# on the same host.
#
# When disabled, new VMs are created from scratch.
#
# Note: Requires "initrd=" to be set ("image=" is not supported).
#
# Default false
#enable_template = true
# Specifies the path of template.
#
# Default "/run/vc/vm/template"
#template_path = "/run/vc/vm/template"
# The number of caches of VMCache:
# unspecified or == 0 --> VMCache is disabled
# > 0 --> will be set to the specified number
#
# VMCache is a function that creates VMs as caches before using it.
# It helps speed up new container creation.
# The function consists of a server and some clients communicating
# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto.
# The VMCache server will create some VMs and cache them by factory cache.
# It will convert the VM to gRPC format and transport it when gets
# requestion from clients.
# Factory grpccache is the VMCache client. It will request gRPC format
# VM and convert it back to a VM. If VMCache function is enabled,
# kata-runtime will request VM from factory grpccache when it creates
# a new sandbox.
#
# Default 0
#vm_cache_number = 0
# Specify the address of the Unix socket that is used by VMCache.
#
# Default /var/run/kata-containers/cache.sock
#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
[proxy.kata]
path = "/opt/kata/libexec/kata-containers/kata-proxy"
# If enabled, proxy messages will be sent to the system log
# (default: disabled)
#enable_debug = true
[shim.kata]
path = "/opt/kata/libexec/kata-containers/kata-shim"
# If enabled, shim messages will be sent to the system log
# (default: disabled)
#enable_debug = true
# If enabled, the shim will create opentracing.io traces and spans.
# (See https://www.jaegertracing.io/docs/getting-started).
#
# Note: By default, the shim runs in a separate network namespace. Therefore,
# to allow it to send trace details to the Jaeger agent running on the host,
# it is necessary to set 'disable_new_netns=true' so that it runs in the host
# network namespace.
#
# (default: disabled)
#enable_tracing = true
[agent.kata]
# If enabled, make the agent display debug-level messages.
# (default: disabled)
#enable_debug = true
# Enable agent tracing.
#
# If enabled, the default trace mode is "dynamic" and the
# default trace type is "isolated". The trace mode and type are set
# explicity with the `trace_type=` and `trace_mode=` options.
#
# Notes:
#
# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
# will NOT activate agent tracing.
#
# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
# full details.
#
# (default: disabled)
#enable_tracing = true
#
#trace_mode = "dynamic"
#trace_type = "isolated"
# Comma separated list of kernel modules and their parameters.
# These modules will be loaded in the guest kernel using modprobe(8).
# The following example can be used to load two kernel modules with parameters
# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
# The first word is considered as the module name and the rest as its parameters.
# Container will not be started when:
# * A kernel module is specified and the modprobe command is not installed in the guest
# or it fails loading the module.
# * The module is not available in the guest or it doesn't met the guest kernel
# requirements, like architecture and version.
#
kernel_modules=[]
[netmon]
# If enabled, the network monitoring process gets started when the
# sandbox is created. This allows for the detection of some additional
# network being added to the existing network namespace, after the
# sandbox has been created.
# (default: disabled)
#enable_netmon = true
# Specify the path to the netmon binary.
path = "/opt/kata/libexec/kata-containers/kata-netmon"
# If enabled, netmon messages will be sent to the system log
# (default: disabled)
#enable_debug = true
[runtime]
# If enabled, the runtime will log additional debug messages to the
# system log
# (default: disabled)
#enable_debug = true
#
# Internetworking model
# Determines how the VM should be connected to the
# the container network interface
# Options:
#
# - bridged (Deprecated)
# Uses a linux bridge to interconnect the container interface to
# the VM. Works for most cases except macvlan and ipvlan.
# ***NOTE: This feature has been deprecated with plans to remove this
# feature in the future. Please use other network models listed below.
#
# - macvtap
# Used when the Container network interface can be bridged using
# macvtap.
#
# - none
# Used when customize network. Only creates a tap device. No veth pair.
#
# - tcfilter
# Uses tc filter rules to redirect traffic from the network interface
# provided by plugin to a tap interface connected to the VM.
#
internetworking_model="tcfilter"
# disable guest seccomp
# Determines whether container seccomp profiles are passed to the virtual
# machine and applied by the kata agent. If set to true, seccomp is not applied
# within the guest
# (default: true)
disable_guest_seccomp=true
# If enabled, the runtime will create opentracing.io traces and spans.
# (See https://www.jaegertracing.io/docs/getting-started).
# (default: disabled)
#enable_tracing = true
# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
# `disable_new_netns` conflicts with `enable_netmon`
# `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only
# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
# (like OVS) directly.
# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
# (default: false)
#disable_new_netns = true
# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
# The container cgroups in the host are not created, just one single cgroup per sandbox.
# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
# The sandbox cgroup is constrained if there is no container type annotation.
# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
sandbox_cgroup_only=false
# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
# be created on the host and shared via 9p. This is far slower, but allows sharing of files from host to guest.
disable_guest_empty_dir = false
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.
# Supported experimental features:
# (default: [])
experimental=[]
```
Output of "`cat "/opt/kata/share/defaults/kata-containers/configuration.toml"`":
```toml
# Copyright (c) 2017-2019 Intel Corporation
#
# SPDX-License-Identifier: Apache-2.0
#
# XXX: WARNING: this file is auto-generated.
# XXX:
# XXX: Source file: "cli/config/configuration-qemu.toml.in"
# XXX: Project:
# XXX: Name: Kata Containers
# XXX: Type: kata
[hypervisor.qemu]
path = "/opt/kata/bin/qemu-system-x86_64"
kernel = "/opt/kata/share/kata-containers/vmlinuz.container"
initrd = "/opt/kata/share/kata-containers/kata-containers-initrd.img"
image = "/opt/kata/share/kata-containers/kata-containers.img"
machine_type = "pc"
# Optional space-separated list of options to pass to the guest kernel.
# For example, use `kernel_params = "vsyscall=emulate"` if you are having
# trouble running pre-2.15 glibc.
#
# WARNING: - any parameter specified here will take priority over the default
# parameter value of the same name used to start the virtual machine.
# Do not set values here unless you understand the impact of doing so as you
# may stop the virtual machine from booting.
# To see the list of default parameters, enable hypervisor debug, create a
# container and look for 'default-kernel-parameters' log entries.
kernel_params = ""
# Path to the firmware.
# If you want that qemu uses the default firmware leave this option empty
firmware = ""
# Machine accelerators
# comma-separated list of machine accelerators to pass to the hypervisor.
# For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
machine_accelerators=""
# Default number of vCPUs per SB/VM:
# unspecified or 0 --> will be set to 1
# < 0 --> will be set to the actual number of physical cores
# > 0 <= number of physical cores --> will be set to the specified number
# > number of physical cores --> will be set to the actual number of physical cores
default_vcpus = 1
# Default maximum number of vCPUs per SB/VM:
# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number
# of vCPUs supported by KVM if that number is exceeded
# > 0 <= number of physical cores --> will be set to the specified number
# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number
# of vCPUs supported by KVM if that number is exceeded
# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when
# the actual number of physical cores is greater than it.
# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU
# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs
# can be added to a SB/VM, but the memory footprint will be big. Another example, with
# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of
# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable,
# unless you know what are you doing.
default_maxvcpus = 0
# Bridges can be used to hot plug devices.
# Limitations:
# * Currently only pci bridges are supported
# * Until 30 devices per bridge can be hot plugged.
# * Until 5 PCI bridges can be cold plugged per VM.
# This limitation could be a bug in qemu or in the kernel
# Default number of bridges per SB/VM:
# unspecified or 0 --> will be set to 1
# > 1 <= 5 --> will be set to the specified number
# > 5 --> will be set to 5
default_bridges = 1
# Default memory size in MiB for SB/VM.
# If unspecified then it will be set 2048 MiB.
default_memory = 2048
#
# Default memory slots per SB/VM.
# If unspecified then it will be set 10.
# This is will determine the times that memory will be hotadded to sandbox/VM.
#memory_slots = 10
# The size in MiB will be plused to max memory of hypervisor.
# It is the memory address space for the NVDIMM devie.
# If set block storage driver (block_device_driver) to "nvdimm",
# should set memory_offset to the size of block device.
# Default 0
#memory_offset = 0
# Specifies virtio-mem will be enabled or not.
# Please note that this option should be used with the command
# "echo 1 > /proc/sys/vm/overcommit_memory".
# Default false
#enable_virtio_mem = true
# Disable block device from being used for a container's rootfs.
# In case of a storage driver like devicemapper where a container's
# root file system is backed by a block device, the block device is passed
# directly to the hypervisor for performance reasons.
# This flag prevents the block device from being passed to the hypervisor,
# 9pfs is used instead to pass the rootfs.
disable_block_device_use = false
# Shared file system type:
# - virtio-9p (default)
# - virtio-fs
shared_fs = "virtio-9p"
# Path to vhost-user-fs daemon.
virtio_fs_daemon = "/opt/kata/bin/virtiofsd"
# Default size of DAX cache in MiB
virtio_fs_cache_size = 1024
# Extra args for virtiofsd daemon
#
# Format example:
# ["-o", "arg1=xxx,arg2", "-o", "hello world", "--arg3=yyy"]
#
# see `virtiofsd -h` for possible options.
virtio_fs_extra_args = []
# Cache mode:
#
# - none
# Metadata, data, and pathname lookup are not cached in guest. They are
# always fetched from host and any changes are immediately pushed to host.
#
# - auto
# Metadata and pathname lookup cache expires after a configured amount of
# time (default is 1 second). Data is cached while the file is open (close
# to open consistency).
#
# - always
# Metadata, data, and pathname lookup are cached in guest and never expire.
virtio_fs_cache = "always"
# Block storage driver to be used for the hypervisor in case the container
# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
# or nvdimm.
block_device_driver = "virtio-scsi"
# Specifies cache-related options will be set to block devices or not.
# Default false
#block_device_cache_set = true
# Specifies cache-related options for block devices.
# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled.
# Default false
#block_device_cache_direct = true
# Specifies cache-related options for block devices.
# Denotes whether flush requests for the device are ignored.
# Default false
#block_device_cache_noflush = true
# Enable iothreads (data-plane) to be used. This causes IO to be
# handled in a separate IO thread. This is currently only implemented
# for SCSI.
#
enable_iothreads = false
# Enable pre allocation of VM RAM, default false
# Enabling this will result in lower container density
# as all of the memory will be allocated and locked
# This is useful when you want to reserve all the memory
# upfront or in the cases where you want memory latencies
# to be very predictable
# Default false
#enable_mem_prealloc = true
# Enable huge pages for VM RAM, default false
# Enabling this will result in the VM memory
# being allocated using huge pages.
# This is useful when you want to use vhost-user network
# stacks within the container. This will automatically
# result in memory pre allocation
#enable_hugepages = true
# Enable vhost-user storage device, default false
# Enabling this will result in some Linux reserved block type
# major range 240-254 being chosen to represent vhost-user devices.
enable_vhost_user_store = false
# The base directory specifically used for vhost-user devices.
# Its sub-path "block" is used for block devices; "block/sockets" is
# where we expect vhost-user sockets to live; "block/devices" is where
# simulated block device nodes for vhost-user devices to live.
vhost_user_store_path = "/var/run/kata-containers/vhost-user"
# Enable file based guest memory support. The default is an empty string which
# will disable this feature. In the case of virtio-fs, this is enabled
# automatically and '/dev/shm' is used as the backing folder.
# This option will be ignored if VM templating is enabled.
#file_mem_backend = ""
# Enable swap of vm memory. Default false.
# The behaviour is undefined if mem_prealloc is also set to true
#enable_swap = true
# This option changes the default hypervisor and kernel parameters
# to enable debug output where available. This extra output is added
# to the proxy logs, but only when proxy debug is also enabled.
#
# Default false
#enable_debug = true
# Disable the customizations done in the runtime when it detects
# that it is running on top a VMM. This will result in the runtime
# behaving as it would when running on bare metal.
#
#disable_nesting_checks = true
# This is the msize used for 9p shares. It is the number of bytes
# used for 9p packet payload.
#msize_9p = 8192
# If true and vsocks are supported, use vsocks to communicate directly
# with the agent and no proxy is started, otherwise use unix
# sockets and start a proxy to communicate with the agent.
# Default false
#use_vsock = true
# If false and nvdimm is supported, use nvdimm device to plug guest image.
# Otherwise virtio-block device is used.
# Default is false
#disable_image_nvdimm = true
# VFIO devices are hotplugged on a bridge by default.
# Enable hotplugging on root bus. This may be required for devices with
# a large PCI bar, as this is a current limitation with hotplugging on
# a bridge. This value is valid for "pc" machine type.
# Default false
#hotplug_vfio_on_root_bus = true
# Before hot plugging a PCIe device, you need to add a pcie_root_port device.
# Use this parameter when using some large PCI bar devices, such as Nvidia GPU
# The value means the number of pcie_root_port
# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35"
# Default 0
#pcie_root_port = 2
# If vhost-net backend for virtio-net is not desired, set to true. Default is false, which trades off
# security (vhost-net runs ring0) for network I/O performance.
#disable_vhost_net = true
#
# Default entropy source.
# The path to a host source of entropy (including a real hardware RNG)
# /dev/urandom and /dev/random are two main options.
# Be aware that /dev/random is a blocking source of entropy. If the host
# runs out of entropy, the VMs boot time will increase leading to get startup
# timeouts.
# The source of entropy /dev/urandom is non-blocking and provides a
# generally acceptable source of entropy. It should work well for pretty much
# all practical purposes.
#entropy_source= "/dev/urandom"
# Path to OCI hook binaries in the *guest rootfs*.
# This does not affect host-side hooks which must instead be added to
# the OCI spec passed to the runtime.
#
# You can create a rootfs with hooks by customizing the osbuilder scripts:
# https://github.com/kata-containers/osbuilder
#
# Hooks must be stored in a subdirectory of guest_hook_path according to their
# hook type, i.e. "guest_hook_path/{prestart,postart,poststop}".
# The agent will scan these directories for executable files and add them, in
# lexicographical order, to the lifecycle of the guest container.
# Hooks are executed in the runtime namespace of the guest. See the official documentation:
# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks
# Warnings will be logged if any error is encountered will scanning for hooks,
# but it will not abort container execution.
#guest_hook_path = "/usr/share/oci/hooks"
[factory]
# VM templating support. Once enabled, new VMs are created from template
# using vm cloning. They will share the same initial kernel, initramfs and
# agent memory by mapping it readonly. It helps speeding up new container
# creation and saves a lot of memory if there are many kata containers running
# on the same host.
#
# When disabled, new VMs are created from scratch.
#
# Note: Requires "initrd=" to be set ("image=" is not supported).
#
# Default false
#enable_template = true
# Specifies the path of template.
#
# Default "/run/vc/vm/template"
#template_path = "/run/vc/vm/template"
# The number of caches of VMCache:
# unspecified or == 0 --> VMCache is disabled
# > 0 --> will be set to the specified number
#
# VMCache is a function that creates VMs as caches before using it.
# It helps speed up new container creation.
# The function consists of a server and some clients communicating
# through Unix socket. The protocol is gRPC in protocols/cache/cache.proto.
# The VMCache server will create some VMs and cache them by factory cache.
# It will convert the VM to gRPC format and transport it when gets
# requestion from clients.
# Factory grpccache is the VMCache client. It will request gRPC format
# VM and convert it back to a VM. If VMCache function is enabled,
# kata-runtime will request VM from factory grpccache when it creates
# a new sandbox.
#
# Default 0
#vm_cache_number = 0
# Specify the address of the Unix socket that is used by VMCache.
#
# Default /var/run/kata-containers/cache.sock
#vm_cache_endpoint = "/var/run/kata-containers/cache.sock"
[proxy.kata]
path = "/opt/kata/libexec/kata-containers/kata-proxy"
# If enabled, proxy messages will be sent to the system log
# (default: disabled)
#enable_debug = true
[shim.kata]
path = "/opt/kata/libexec/kata-containers/kata-shim"
# If enabled, shim messages will be sent to the system log
# (default: disabled)
#enable_debug = true
# If enabled, the shim will create opentracing.io traces and spans.
# (See https://www.jaegertracing.io/docs/getting-started).
#
# Note: By default, the shim runs in a separate network namespace. Therefore,
# to allow it to send trace details to the Jaeger agent running on the host,
# it is necessary to set 'disable_new_netns=true' so that it runs in the host
# network namespace.
#
# (default: disabled)
#enable_tracing = true
[agent.kata]
# If enabled, make the agent display debug-level messages.
# (default: disabled)
#enable_debug = true
# Enable agent tracing.
#
# If enabled, the default trace mode is "dynamic" and the
# default trace type is "isolated". The trace mode and type are set
# explicity with the `trace_type=` and `trace_mode=` options.
#
# Notes:
#
# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly
# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing`
# will NOT activate agent tracing.
#
# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for
# full details.
#
# (default: disabled)
#enable_tracing = true
#
#trace_mode = "dynamic"
#trace_type = "isolated"
# Comma separated list of kernel modules and their parameters.
# These modules will be loaded in the guest kernel using modprobe(8).
# The following example can be used to load two kernel modules with parameters
# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"]
# The first word is considered as the module name and the rest as its parameters.
# Container will not be started when:
# * A kernel module is specified and the modprobe command is not installed in the guest
# or it fails loading the module.
# * The module is not available in the guest or it doesn't met the guest kernel
# requirements, like architecture and version.
#
kernel_modules=[]
[netmon]
# If enabled, the network monitoring process gets started when the
# sandbox is created. This allows for the detection of some additional
# network being added to the existing network namespace, after the
# sandbox has been created.
# (default: disabled)
#enable_netmon = true
# Specify the path to the netmon binary.
path = "/opt/kata/libexec/kata-containers/kata-netmon"
# If enabled, netmon messages will be sent to the system log
# (default: disabled)
#enable_debug = true
[runtime]
# If enabled, the runtime will log additional debug messages to the
# system log
# (default: disabled)
#enable_debug = true
#
# Internetworking model
# Determines how the VM should be connected to the
# the container network interface
# Options:
#
# - macvtap
# Used when the Container network interface can be bridged using
# macvtap.
#
# - none
# Used when customize network. Only creates a tap device. No veth pair.
#
# - tcfilter
# Uses tc filter rules to redirect traffic from the network interface
# provided by plugin to a tap interface connected to the VM.
#
internetworking_model="tcfilter"
# disable guest seccomp
# Determines whether container seccomp profiles are passed to the virtual
# machine and applied by the kata agent. If set to true, seccomp is not applied
# within the guest
# (default: true)
disable_guest_seccomp=true
# If enabled, the runtime will create opentracing.io traces and spans.
# (See https://www.jaegertracing.io/docs/getting-started).
# (default: disabled)
#enable_tracing = true
# If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
# This option may have some potential impacts to your host. It should only be used when you know what you're doing.
# `disable_new_netns` conflicts with `enable_netmon`
# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
# (like OVS) directly.
# If you are using docker, `disable_new_netns` only works with `docker run --net=none`
# (default: false)
#disable_new_netns = true
# if enabled, the runtime will add all the kata processes inside one dedicated cgroup.
# The container cgroups in the host are not created, just one single cgroup per sandbox.
# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox.
# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation.
# The sandbox cgroup is constrained if there is no container type annotation.
# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType
sandbox_cgroup_only=false
# If enabled, the runtime will not create Kubernetes emptyDir mounts on the guest filesystem. Instead, emptyDir mounts will
# be created on the host and shared via 9p. This is far slower, but allows sharing of files from host to guest.
disable_guest_empty_dir=false
# Enabled experimental feature list, format: ["a", "b"].
# Experimental features are features not stable enough for production,
# they may break compatibility, and are prepared for a big version bump.
# Supported experimental features:
# (default: [])
experimental=[]
```
Config file `/usr/share/defaults/kata-containers/configuration.toml` not found
---
# KSM throttler
## version
Output of "` --version`":
```
/opt/kata/bin/kata-collect-data.sh: line 178: --version: command not found
```
## systemd service
# Image details
```yaml
---
osbuilder:
url: "https://github.com/kata-containers/osbuilder"
version: "unknown"
rootfs-creation-time: "2020-07-02T15:02:45.860272195+0000Z"
description: "osbuilder rootfs"
file-format-version: "0.0.2"
architecture: "x86_64"
base-distro:
name: "Clear"
version: "33450"
packages:
default:
- "chrony"
- "iptables-bin"
- "kmod-bin"
- "libudev0-shim"
- "systemd"
- "util-linux-bin"
extra:
agent:
url: "https://github.com/kata-containers/agent"
name: "kata-agent"
version: "1.11.2-abb7149e49ea3b6bbb23526e8562d6aa9c181e35"
agent-is-init-daemon: "no"
```
---
# Initrd details
No initrd
---
# Logfiles
## Runtime logs
No recent runtime problems found in system journal.
## Proxy logs
No recent proxy problems found in system journal.
## Shim logs
No recent shim problems found in system journal.
## Throttler logs
No recent throttler problems found in system journal.
---
# Container manager details
Have `docker`, but it's not being used. Removing this information.
Have `kubectl`
## Kubernetes
Output of "`kubectl version`":
```
Client Version: version.Info{Major:"1", Minor:"17", GitVersion:"v1.17.6", GitCommit:"d32e40e20d167e103faf894261614c5b45c44198", GitTreeState:"clean", BuildDate:"2020-05-20T13:16:24Z", GoVersion:"go1.13.9", Compiler:"gc", Platform:"linux/amd64"}
Error from server (NotFound): the server could not find the requested resource
```
Output of "`kubectl config view`":
```
apiVersion: v1
clusters: null
contexts: null
current-context: ""
kind: Config
preferences: {}
users: null
```
Output of "`systemctl show kubelet`":
```
Type=simple
Restart=on-failure
NotifyAccess=none
RestartUSec=5s
TimeoutStartUSec=1min 30s
TimeoutStopUSec=1min 30s
RuntimeMaxUSec=infinity
WatchdogUSec=0
WatchdogTimestampMonotonic=0
RootDirectoryStartOnly=no
RemainAfterExit=no
GuessMainPID=yes
MainPID=4331
ControlPID=0
FileDescriptorStoreMax=0
NFileDescriptorStore=0
StatusErrno=0
Result=success
UID=[not set]
GID=[not set]
NRestarts=0
ExecMainStartTimestamp=Tue 2020-07-07 19:20:31 UTC
ExecMainStartTimestampMonotonic=309860860
ExecMainExitTimestampMonotonic=0
ExecMainPID=4331
ExecMainCode=0
ExecMainStatus=0
ExecStartPre={ path=/bin/bash ; argv[]=/bin/bash -c if [[ $(/bin/mount | /bin/grep /sys/fs/bpf -c) -eq 0 ]]; then /bin/mount bpffs /sys/fs/bpf -t bpf; fi ; ignore_errors=no ; start_time=[Tue 2020-07-07 19:20:31 UTC] ; stop_time=[Tue 2020-07-07 19:20:31 UTC] ; pid=4320 ; code=exited ; status=0 }
ExecStartPre={ path=/bin/bash ; argv[]=/bin/bash -c until [[ $(hostname) != 'localhost' ]]; do sleep 1; done ; ignore_errors=no ; start_time=[Tue 2020-07-07 19:20:31 UTC] ; stop_time=[Tue 2020-07-07 19:20:31 UTC] ; pid=4325 ; code=exited ; status=0 }
ExecStartPre={ path=/bin/bash ; argv[]=/bin/bash /opt/ethos/bin/kubelet-master-setup.sh ; ignore_errors=yes ; start_time=[Tue 2020-07-07 19:20:31 UTC] ; stop_time=[Tue 2020-07-07 19:20:31 UTC] ; pid=4329 ; code=exited ; status=127 }
ExecStart={ path=/opt/bin/kubelet ; argv[]=/opt/bin/kubelet --cert-dir=/etc/kubernetes/certs --config=/etc/kubernetes/kubelet.yaml --image-pull-progress-deadline=10m --kubeconfig=/etc/kubernetes/kubeconfig/kubelet.kubeconfig --network-plugin=cni --root-dir=/var/lib/kubelet --v=2 $KUBELET_ARGS ; ignore_errors=no ; start_time=[Tue 2020-07-07 19:20:31 UTC] ; stop_time=[n/a] ; pid=4331 ; code=(null) ; status=0/0 }
Slice=system.slice
ControlGroup=/system.slice/kubelet.service
MemoryCurrent=102268928
CPUUsageNSec=79259769193
TasksCurrent=42
IPIngressBytes=18446744073709551615
IPIngressPackets=18446744073709551615
IPEgressBytes=18446744073709551615
IPEgressPackets=18446744073709551615
Delegate=no
CPUAccounting=yes
CPUWeight=[not set]
StartupCPUWeight=[not set]
CPUShares=[not set]
StartupCPUShares=[not set]
CPUQuotaPerSecUSec=infinity
IOAccounting=no
IOWeight=[not set]
StartupIOWeight=[not set]
BlockIOAccounting=no
BlockIOWeight=[not set]
StartupBlockIOWeight=[not set]
MemoryAccounting=yes
MemoryMin=0
MemoryLow=0
MemoryHigh=infinity
MemoryMax=infinity
MemorySwapMax=infinity
MemoryLimit=infinity
DevicePolicy=auto
TasksAccounting=yes
TasksMax=131071
IPAccounting=no
EnvironmentFiles=/run/ethos/kubelet-args (ignore_errors=no)
UMask=0022
LimitCPU=infinity
LimitCPUSoft=infinity
LimitFSIZE=infinity
LimitFSIZESoft=infinity
LimitDATA=infinity
LimitDATASoft=infinity
LimitSTACK=infinity
LimitSTACKSoft=8388608
LimitCORE=infinity
LimitCORESoft=infinity
LimitRSS=infinity
LimitRSSSoft=infinity
LimitNOFILE=524288
LimitNOFILESoft=1024
LimitAS=infinity
LimitASSoft=infinity
LimitNPROC=257423
LimitNPROCSoft=257423
LimitMEMLOCK=65536
LimitMEMLOCKSoft=65536
LimitLOCKS=infinity
LimitLOCKSSoft=infinity
LimitSIGPENDING=257423
LimitSIGPENDINGSoft=257423
LimitMSGQUEUE=819200
LimitMSGQUEUESoft=819200
LimitNICE=0
LimitNICESoft=0
LimitRTPRIO=0
LimitRTPRIOSoft=0
LimitRTTIME=infinity
LimitRTTIMESoft=infinity
OOMScoreAdjust=0
Nice=0
IOSchedulingClass=0
IOSchedulingPriority=0
CPUSchedulingPolicy=0
CPUSchedulingPriority=0
TimerSlackNSec=50000
CPUSchedulingResetOnFork=no
NonBlocking=no
StandardInput=null
StandardInputData=
StandardOutput=journal
StandardError=inherit
TTYReset=no
TTYVHangup=no
TTYVTDisallocate=no
SyslogPriority=30
SyslogLevelPrefix=yes
SyslogLevel=6
SyslogFacility=3
LogLevelMax=-1
LogRateLimitIntervalUSec=0
LogRateLimitBurst=0
SecureBits=0
CapabilityBoundingSet=cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend
AmbientCapabilities=
DynamicUser=no
RemoveIPC=no
MountFlags=
PrivateTmp=no
PrivateDevices=no
ProtectKernelTunables=no
ProtectKernelModules=no
ProtectControlGroups=no
PrivateNetwork=no
PrivateUsers=no
PrivateMounts=no
ProtectHome=no
ProtectSystem=no
SameProcessGroup=no
UtmpMode=init
IgnoreSIGPIPE=yes
NoNewPrivileges=no
SystemCallErrorNumber=0
LockPersonality=no
RuntimeDirectoryPreserve=no
RuntimeDirectoryMode=0755
StateDirectoryMode=0755
CacheDirectoryMode=0755
LogsDirectoryMode=0755
ConfigurationDirectoryMode=0755
MemoryDenyWriteExecute=no
RestrictRealtime=no
RestrictNamespaces=no
MountAPIVFS=no
KeyringMode=private
KillMode=process
KillSignal=15
FinalKillSignal=9
SendSIGKILL=yes
SendSIGHUP=no
WatchdogSignal=6
Id=kubelet.service
Names=kubelet.service
Requires=system.slice sysinit.target coreos-metadata.service crio.service configure-docker.service configure-kubelet.service download-certificates.service docker.service
Wants=configure-kubelet.service
WantedBy=multi-user.target
Conflicts=shutdown.target
Before=multi-user.target shutdown.target
After=configure-docker.service mnt-nvme.mount coreos-metadata.service nvidia-driver.service system.slice docker.service sysinit.target basic.target download-certificates.service systemd-journald.socket configure-kubelet.service crio.service
Description=Kubernetes Kubelet
LoadState=loaded
ActiveState=active
SubState=running
FragmentPath=/etc/systemd/system/kubelet.service
DropInPaths=/etc/systemd/system/kubelet.service.d/11-ecr-credentials.conf
UnitFileState=enabled
UnitFilePreset=enabled
StateChangeTimestamp=Tue 2020-07-07 19:20:31 UTC
StateChangeTimestampMonotonic=309860913
InactiveExitTimestamp=Tue 2020-07-07 19:20:31 UTC
InactiveExitTimestampMonotonic=309841745
ActiveEnterTimestamp=Tue 2020-07-07 19:20:31 UTC
ActiveEnterTimestampMonotonic=309860913
ActiveExitTimestamp=Tue 2020-07-07 19:19:46 UTC
ActiveExitTimestampMonotonic=264485400
InactiveEnterTimestamp=Tue 2020-07-07 19:19:46 UTC
InactiveEnterTimestampMonotonic=264500020
CanStart=yes
CanStop=yes
CanReload=no
CanIsolate=no
StopWhenUnneeded=no
RefuseManualStart=no
RefuseManualStop=no
AllowIsolate=no
DefaultDependencies=yes
OnFailureJobMode=replace
IgnoreOnIsolate=no
NeedDaemonReload=no
JobTimeoutUSec=infinity
JobRunningTimeoutUSec=infinity
JobTimeoutAction=none
ConditionResult=yes
AssertResult=yes
ConditionTimestamp=Tue 2020-07-07 19:20:31 UTC
ConditionTimestampMonotonic=309839607
AssertTimestamp=Tue 2020-07-07 19:20:31 UTC
AssertTimestampMonotonic=309839608
Transient=no
Perpetual=no
StartLimitIntervalUSec=10s
StartLimitBurst=5
StartLimitAction=none
FailureAction=none
FailureActionExitStatus=-1
SuccessAction=none
SuccessActionExitStatus=-1
InvocationID=424a1e9ac5da43d79f2f4d3a064a3a9f
CollectMode=inactive
```
Have `crio`
## crio
Output of "`crio --version`":
```
crio version 1.17.4
commit: "d237e8716fa901928905460fdf3b8280770f0b51"
```
Output of "`systemctl show crio`":
```
Type=notify
Restart=always
NotifyAccess=main
RestartUSec=100ms
TimeoutStartUSec=infinity
TimeoutStopUSec=1min 30s
RuntimeMaxUSec=infinity
WatchdogUSec=0
WatchdogTimestampMonotonic=0
RootDirectoryStartOnly=no
RemainAfterExit=no
GuessMainPID=yes
MainPID=4141
ControlPID=0
FileDescriptorStoreMax=0
NFileDescriptorStore=0
StatusErrno=0
Result=success
UID=[not set]
GID=[not set]
NRestarts=0
ExecMainStartTimestamp=Tue 2020-07-07 19:20:30 UTC
ExecMainStartTimestampMonotonic=309325626
ExecMainExitTimestampMonotonic=0
ExecMainPID=4141
ExecMainCode=0
ExecMainStatus=0
ExecStartPre={ path=/opt/ethos/bin/crio-setup.sh ; argv[]=/opt/ethos/bin/crio-setup.sh ; ignore_errors=no ; start_time=[Tue 2020-07-07 19:20:30 UTC] ; stop_time=[Tue 2020-07-07 19:20:30 UTC] ; pid=4119 ; code=exited ; status=0 }
ExecStart={ path=/opt/bin/crio ; argv[]=/opt/bin/crio $CRIO_FLAGS ; ignore_errors=no ; start_time=[Tue 2020-07-07 19:20:30 UTC] ; stop_time=[n/a] ; pid=4141 ; code=(null) ; status=0/0 }
ExecReload={ path=/bin/kill ; argv[]=/bin/kill -s HUP $MAINPID ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Slice=system.slice
ControlGroup=/system.slice/crio.service
MemoryCurrent=6169927680
CPUUsageNSec=178232167673
TasksCurrent=276
IPIngressBytes=18446744073709551615
IPIngressPackets=18446744073709551615
IPEgressBytes=18446744073709551615
IPEgressPackets=18446744073709551615
Delegate=no
CPUAccounting=yes
CPUWeight=[not set]
StartupCPUWeight=[not set]
CPUShares=[not set]
StartupCPUShares=[not set]
CPUQuotaPerSecUSec=infinity
IOAccounting=no
IOWeight=[not set]
StartupIOWeight=[not set]
BlockIOAccounting=no
BlockIOWeight=[not set]
StartupBlockIOWeight=[not set]
MemoryAccounting=yes
MemoryMin=0
MemoryLow=0
MemoryHigh=infinity
MemoryMax=infinity
MemorySwapMax=infinity
MemoryLimit=infinity
DevicePolicy=auto
TasksAccounting=yes
TasksMax=infinity
IPAccounting=no
EnvironmentFiles=/etc/crio/crio.env (ignore_errors=no)
UMask=0022
LimitCPU=infinity
LimitCPUSoft=infinity
LimitFSIZE=infinity
LimitFSIZESoft=infinity
LimitDATA=infinity
LimitDATASoft=infinity
LimitSTACK=infinity
LimitSTACKSoft=8388608
LimitCORE=infinity
LimitCORESoft=infinity
LimitRSS=infinity
LimitRSSSoft=infinity
LimitNOFILE=1048576
LimitNOFILESoft=1048576
LimitAS=infinity
LimitASSoft=infinity
LimitNPROC=1048576
LimitNPROCSoft=1048576
LimitMEMLOCK=65536
LimitMEMLOCKSoft=65536
LimitLOCKS=infinity
LimitLOCKSSoft=infinity
LimitSIGPENDING=257423
LimitSIGPENDINGSoft=257423
LimitMSGQUEUE=819200
LimitMSGQUEUESoft=819200
LimitNICE=0
LimitNICESoft=0
LimitRTPRIO=0
LimitRTPRIOSoft=0
LimitRTTIME=infinity
LimitRTTIMESoft=infinity
OOMScoreAdjust=-999
Nice=0
IOSchedulingClass=0
IOSchedulingPriority=0
CPUSchedulingPolicy=0
CPUSchedulingPriority=0
TimerSlackNSec=50000
CPUSchedulingResetOnFork=no
NonBlocking=no
StandardInput=null
StandardInputData=
StandardOutput=journal
StandardError=inherit
TTYReset=no
TTYVHangup=no
TTYVTDisallocate=no
SyslogPriority=30
SyslogLevelPrefix=yes
SyslogLevel=6
SyslogFacility=3
LogLevelMax=-1
LogRateLimitIntervalUSec=0
LogRateLimitBurst=0
SecureBits=0
CapabilityBoundingSet=cap_chown cap_dac_override cap_dac_read_search cap_fowner cap_fsetid cap_kill cap_setgid cap_setuid cap_setpcap cap_linux_immutable cap_net_bind_service cap_net_broadcast cap_net_admin cap_net_raw cap_ipc_lock cap_ipc_owner cap_sys_module cap_sys_rawio cap_sys_chroot cap_sys_ptrace cap_sys_pacct cap_sys_admin cap_sys_boot cap_sys_nice cap_sys_resource cap_sys_time cap_sys_tty_config cap_mknod cap_lease cap_audit_write cap_audit_control cap_setfcap cap_mac_override cap_mac_admin cap_syslog cap_wake_alarm cap_block_suspend
AmbientCapabilities=
DynamicUser=no
RemoveIPC=no
MountFlags=
PrivateTmp=no
PrivateDevices=no
ProtectKernelTunables=no
ProtectKernelModules=no
ProtectControlGroups=no
PrivateNetwork=no
PrivateUsers=no
PrivateMounts=no
ProtectHome=no
ProtectSystem=no
SameProcessGroup=no
UtmpMode=init
IgnoreSIGPIPE=yes
NoNewPrivileges=no
SystemCallErrorNumber=0
LockPersonality=no
RuntimeDirectoryPreserve=no
RuntimeDirectoryMode=0755
StateDirectoryMode=0755
CacheDirectoryMode=0755
LogsDirectoryMode=0755
ConfigurationDirectoryMode=0755
MemoryDenyWriteExecute=no
RestrictRealtime=no
RestrictNamespaces=no
MountAPIVFS=no
KeyringMode=private
KillMode=control-group
KillSignal=15
FinalKillSignal=9
SendSIGKILL=yes
SendSIGHUP=no
WatchdogSignal=6
Id=crio.service
Names=crio.service
Requires=lvm2-lvmetad.service network-online.target cri-logging-driver-watch.service system.slice sysinit.target
RequiredBy=kubelet.service
WantedBy=crio-shutdown.service multi-user.target
Conflicts=shutdown.target
Before=multi-user.target nvidia-driver.service shutdown.target kubelet.service crio-shutdown.service
After=network-online.target lvm2-lvmetad.service systemd-journald.socket basic.target system.slice sysinit.target cri-logging-driver-watch.service
Documentation=https://github.com/kubernetes-sigs/cri-o/blob/master/contrib/systemd/crio.service
Description=Open Container Initiative Daemon
LoadState=loaded
ActiveState=active
SubState=running
FragmentPath=/etc/systemd/system/crio.service
UnitFileState=enabled
UnitFilePreset=enabled
StateChangeTimestamp=Tue 2020-07-07 19:20:31 UTC
StateChangeTimestampMonotonic=309838091
InactiveExitTimestamp=Tue 2020-07-07 19:20:30 UTC
InactiveExitTimestampMonotonic=309250325
ActiveEnterTimestamp=Tue 2020-07-07 19:20:31 UTC
ActiveEnterTimestampMonotonic=309838091
ActiveExitTimestamp=Tue 2020-07-07 19:19:46 UTC
ActiveExitTimestampMonotonic=264501971
InactiveEnterTimestamp=Tue 2020-07-07 19:20:30 UTC
InactiveEnterTimestampMonotonic=309247179
CanStart=yes
CanStop=yes
CanReload=yes
CanIsolate=no
StopWhenUnneeded=no
RefuseManualStart=no
RefuseManualStop=no
AllowIsolate=no
DefaultDependencies=yes
OnFailureJobMode=replace
IgnoreOnIsolate=no
NeedDaemonReload=no
JobTimeoutUSec=infinity
JobRunningTimeoutUSec=infinity
JobTimeoutAction=none
ConditionResult=yes
AssertResult=yes
ConditionTimestamp=Tue 2020-07-07 19:20:30 UTC
ConditionTimestampMonotonic=309248184
AssertTimestamp=Tue 2020-07-07 19:20:30 UTC
AssertTimestampMonotonic=309248185
Transient=no
Perpetual=no
StartLimitIntervalUSec=10s
StartLimitBurst=5
StartLimitAction=none
FailureAction=none
FailureActionExitStatus=-1
SuccessAction=none
SuccessActionExitStatus=-1
InvocationID=9a608158c56845da9f72b72d5feb2db7
CollectMode=inactive
```
Output of "`cat /etc/crio/crio.conf`":
```
# The CRI-O configuration file specifies all of the available configuration
# options and command-line flags for the crio(8) OCI Kubernetes Container Runtime
# daemon, but in a TOML format that can be more easily modified and versioned.
#
# Please refer to crio.conf(5) for details of all configuration options.
# CRI-O supports partial configuration reload during runtime, which can be
# done by sending SIGHUP to the running process. Currently supported options
# are explicitly mentioned with: 'This option supports live configuration
# reload'.
# CRI-O reads its storage defaults from the containers-storage.conf(5) file
# located at /etc/containers/storage.conf. Modify this storage configuration if
# you want to change the system's defaults. If you want to modify storage just
# for CRI-O, you can change the storage configuration options here.
[crio]
# Path to the "root directory". CRI-O stores all of its data, including
# containers images, in this directory.
#root = "/home/sascha/.local/share/containers/storage"
# Path to the "run directory". CRI-O stores all of its state in this directory.
#runroot = "/tmp/1000"
# Storage driver used to manage the storage of images and containers. Please
# refer to containers-storage.conf(5) to see all available storage drivers.
storage_driver = "vfs"
# List to pass options to the storage driver. Please refer to
# containers-storage.conf(5) to see all available storage options.
#storage_option = [
#]
# If set to false, in-memory locking will be used instead of file-based locking.
# **Deprecated** this option will be removed in the future.
file_locking = false
# Path to the lock file.
# **Deprecated** this option will be removed in the future.
file_locking_path = "/run/crio.lock"
# The crio.api table contains settings for the kubelet/gRPC interface.
[crio.api]
# Path to AF_LOCAL socket on which CRI-O will listen.
listen = "/var/run/crio/crio.sock"
# IP address on which the stream server will listen.
stream_address = "127.0.0.1"
# The port on which the stream server will listen.
stream_port = "0"
# Enable encrypted TLS transport of the stream server.
stream_enable_tls = false
# Path to the x509 certificate file used to serve the encrypted stream. This
# file can change, and CRI-O will automatically pick up the changes within 5
# minutes.
stream_tls_cert = ""
# Path to the key file used to serve the encrypted stream. This file can
# change, and CRI-O will automatically pick up the changes within 5 minutes.
stream_tls_key = ""
# Path to the x509 CA(s) file used to verify and authenticate client
# communication with the encrypted stream. This file can change, and CRI-O will
# automatically pick up the changes within 5 minutes.
stream_tls_ca = ""
# Maximum grpc send message size in bytes. If not set or <=0, then CRI-O will default to 16 * 1024 * 1024.
grpc_max_send_msg_size = 16777216
# Maximum grpc receive message size. If not set or <= 0, then CRI-O will default to 16 * 1024 * 1024.
grpc_max_recv_msg_size = 16777216
# The crio.runtime table contains settings pertaining to the OCI runtime used
# and options for how to set up and manage the OCI runtime.
[crio.runtime]
# A list of ulimits to be set in containers by default, specified as
# "=:", for example:
# "nofile=1024:2048"
# If nothing is set here, settings will be inherited from the CRI-O daemon
#default_ulimits = [
#]
# default_runtime is the _name_ of the OCI runtime to be used as the default.
# The name is matched against the runtimes map below.
default_runtime = "runc"
# If true, the runtime will not use pivot_root, but instead use MS_MOVE.
no_pivot = false
# Path to the conmon binary, used for monitoring the OCI runtime.
# Ethos: The default value of `/usr/local/libexec/crio/conmon` is on the read-only
# filesystem. This binary is provided at the new value, `/opt/bin/conmon`
conmon = "/opt/bin/conmon"
# Cgroup setting for conmon
conmon_cgroup = "pod"
# Environment variable list for the conmon process, used for passing necessary
# environment variables to conmon or the runtime.
conmon_env = [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
]
# If true, SELinux will be used for pod separation on the host.
# Ethos: selinux must be disabled for kata to currently function.
selinux = false
# Path to the seccomp.json profile which is used as the default seccomp profile
# for the runtime. If not specified, then the internal default seccomp profile
# will be used.
seccomp_profile = "/etc/crio/seccomp.json"
# Used to change the name of the default AppArmor profile of CRI-O. The default
# profile name is "crio-default-" followed by the version string of CRI-O.
apparmor_profile = "crio-default"
# Cgroup management implementation used for the runtime.
cgroup_manager = "cgroupfs"
# List of default capabilities for containers. If it is empty or commented out,
# only the capabilities defined in the containers json file by the user/kube
# will be added.
# default_capabilities = [
# "CHOWN",
# "DAC_OVERRIDE",
# "FSETID",
# "FOWNER",
# "NET_RAW",
# "SETGID",
# "SETUID",
# "SETPCAP",
# "NET_BIND_SERVICE",
# "SYS_CHROOT",
# "KILL",
# ]
# List of default sysctls. If it is empty or commented out, only the sysctls
# defined in the container json file by the user/kube will be added.
default_sysctls = [
]
# List of additional devices. specified as
# "::", for example: "--device=/dev/sdc:/dev/xvdc:rwm".
#If it is empty or commented out, only the devices
# defined in the container json file by the user/kube will be added.
additional_devices = [
]
# Path to OCI hooks directories for automatically executed hooks.
hooks_dir = [
"/etc/containers/oci/hooks.d"
]
# List of default mounts for each container. **Deprecated:** this option will
# be removed in future versions in favor of default_mounts_file.
default_mounts = [
]
# Path to the file specifying the defaults mounts for each container. The
# format of the config is /SRC:/DST, one mount per line. Notice that CRI-O reads
# its default mounts from the following two files:
#
# 1) /etc/containers/mounts.conf (i.e., default_mounts_file): This is the
# override file, where users can either add in their own default mounts, or
# override the default mounts shipped with the package.
#
# 2) /usr/share/containers/mounts.conf: This is the default file read for
# mounts. If you want CRI-O to read from a different, specific mounts file,
# you can change the default_mounts_file. Note, if this is done, CRI-O will
# only add mounts it finds in this file.
#
#default_mounts_file = ""
# Maximum number of processes allowed in a container.
pids_limit = 1024
# Maximum sized allowed for the container log file. Negative numbers indicate
# that no size limit is imposed. If it is positive, it must be >= 8192 to
# match/exceed conmon's read buffer. The file is truncated and re-opened so the
# limit is never exceeded.
log_size_max = -1
# Whether container output should be logged to journald in addition to the kuberentes log file
log_to_journald = false
# Path to directory in which container exit files are written to by conmon.
container_exits_dir = "/var/run/crio/exits"
# Path to directory for container attach sockets.
container_attach_socket_dir = "/var/run/crio"
# If set to true, all containers will run in read-only mode.
read_only = false
# Changes the verbosity of the logs based on the level it is set to. Options
# are fatal, panic, error, warn, info, and debug. This option supports live
# configuration reload.
log_level = "error"
# The default log directory where all logs will go unless directly specified by the kubelet
log_dir = "/var/log/crio/pods"
# The UID mappings for the user namespace of each container. A range is
# specified in the form containerUID:HostUID:Size. Multiple ranges must be
# separated by comma.
uid_mappings = ""
# The GID mappings for the user namespace of each container. A range is
# specified in the form containerGID:HostGID:Size. Multiple ranges must be
# separated by comma.
gid_mappings = ""
# The minimal amount of time in seconds to wait before issuing a timeout
# regarding the proper termination of the container.
ctr_stop_timeout = 0
# The "crio.runtime.runtimes" table defines a list of OCI compatible runtimes.
# The runtime to use is picked based on the runtime_handler provided by the CRI.
# If no runtime_handler is provided, the runtime will be picked based on the level
# of trust of the workload.
# ManageNetworkNSLifecycle determines whether we pin and remove network namespace
# and manage its lifecycle.
# Ethos: `manage_network_ns_lifecycle` is added according to Kata docs
# https://github.com/kata-containers/packaging/blob/master/kata-deploy/scripts/kata-deploy.sh#L53-L72
# ManageNetworkNSLifecycle determines whether we pin and remove network namespace
# and manage its lifecycle
manage_network_ns_lifecycle = true
[crio.runtime.runtimes.runc]
runtime_path = "/usr/bin/runc"
runtime_type = "oci"
# runtime_type = "vm" is (probably) meant to be used when CRI-O is trying to run things made
# for containerd. If the runtime can't support OCI, then you can use the "vm" type to run
# it anyways.
# We use the "vm" runtime type here because virtio-fs performs terribly under "oci".
# I have absolutely no idea WHY this is the case because the docs don't provide any details,
# but I was told to try it in this Github issue comment and it worked:
# https://github.com/cri-o/cri-o/issues/3581#issuecomment-615467744
# It says "containerd" below, but that's just a shim Kata created to work under containerd,
# which we are pretending to be.
[crio.runtime.runtimes.kata-qemu]
runtime_path = "/opt/kata/bin/containerd-shim-kata-v2"
runtime_type = "vm"
# The crio.image table contains settings pertaining to the management of OCI images.
#
# CRI-O reads its configured registries defaults from the system wide
# containers-registries.conf(5) located in /etc/containers/registries.conf. If
# you want to modify just CRI-O, you can change the registries configuration in
# this file. Otherwise, leave insecure_registries and registries commented out to
# use the system's defaults from /etc/containers/registries.conf.
[crio.image]
# Default transport for pulling images from a remote container storage.
default_transport = "docker://"
# The path to a file containing credentials necessary for pulling images from
# secure registries. The file is similar to that of /var/lib/kubelet/config.json
global_auth_file = ""
# The image used to instantiate infra containers.
# This option supports live configuration reload.
pause_image = "k8s.gcr.io/pause:3.1"
# The path to a file containing credentials specific for pulling the pause_image from
# above. The file is similar to that of /var/lib/kubelet/config.json
# This option supports live configuration reload.
pause_image_auth_file = ""
# The command to run to have a container stay in the paused state.
# This option supports live configuration reload.
pause_command = "/pause"
# Path to the file which decides what sort of policy we use when deciding
# whether or not to trust an image that we've pulled. It is not recommended that
# this option be used, as the default behavior of using the system-wide default
# policy (i.e., /etc/containers/policy.json) is most often preferred. Please
# refer to containers-policy.json(5) for more details.
signature_policy = ""
# Controls how image volumes are handled. The valid values are mkdir, bind and
# ignore; the latter will ignore volumes entirely.
image_volumes = "mkdir"
# List of registries to be used when pulling an unqualified image (e.g.,
# "alpine:latest"). By default, registries is set to "docker.io" for
# compatibility reasons. Depending on your workload and usecase you may add more
# registries (e.g., "quay.io", "registry.fedoraproject.org",
# "registry.opensuse.org", etc.).
registries = [
"docker.io"
]
# The crio.network table containers settings pertaining to the management of
# CNI plugins.
[crio.network]
# Path to the directory where CNI configuration files are located.
network_dir = "/etc/cni/net.d/"
# Paths to directories where CNI plugin binaries are located.
plugin_dirs = [
"/opt/cni/bin/",
]
```
Have `containerd`, but it's not being used. Removing this information.
---
# Packages
No `dpkg`
No `rpm`
---
Here are some interesting logs:
```
Jul 07 19:23:39 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:39.551920801Z" level=warning msg="Could not remove container share dir" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="no such file or directory" sandbox=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 share-dir=/run/kata-containers/shared/sandboxes/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/43c8918c151282863ada75b404ad09a203e204e1bacc31cb7bce416bb5adb583 source=virtcontainers subsystem=container
Jul 07 19:23:39 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:39.613104734Z" level=warning msg="Could not remove container share dir" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="no such file or directory" sandbox=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 share-dir=/run/kata-containers/shared/sandboxes/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/18eb3898b20b76b1e1e5a59972830284a9f98771b34881bab1464c3532203692 source=virtcontainers subsystem=container
Jul 07 19:23:39 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:39.689115998Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:39 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:39.765917565Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:39 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:39.825004193Z" level=warning msg="Could not remove container share dir" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="no such file or directory" sandbox=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 share-dir=/run/kata-containers/shared/sandboxes/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 source=virtcontainers subsystem=container
Jul 07 19:23:39 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:39.841578729Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:39 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:39.913913085Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:39 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:39.983010835Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.433067905Z" level=error msg="Could not read qemu pid file" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="open /run/vc/vm/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/pid: no such file or directory" source=virtcontainers subsystem=qemu
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.439315218Z" level=error msg="Could not read qemu pid file" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="open /run/vc/vm/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/pid: no such file or directory" source=virtcontainers subsystem=qemu
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.443983028Z" level=error msg="Could not read qemu pid file" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="open /run/vc/vm/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/pid: no such file or directory" source=virtcontainers subsystem=qemu
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.450406242Z" level=error msg="Could not read qemu pid file" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="open /run/vc/vm/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/pid: no such file or directory" source=virtcontainers subsystem=qemu
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.455938754Z" level=error msg="Could not read qemu pid file" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="open /run/vc/vm/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/pid: no such file or directory" source=virtcontainers subsystem=qemu
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.460417464Z" level=error msg="Could not read qemu pid file" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="open /run/vc/vm/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/pid: no such file or directory" source=virtcontainers subsystem=qemu
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.464333672Z" level=error msg="Could not read qemu pid file" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="open /run/vc/vm/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/pid: no such file or directory" source=virtcontainers subsystem=qemu
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.467494979Z" level=error msg="Could not read qemu pid file" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="open /run/vc/vm/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/pid: no such file or directory" source=virtcontainers subsystem=qemu
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.46772428Z" level=warning msg="sandbox cgroups path is empty" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 sandbox=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 source=virtcontainers subsystem=sandbox
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.504457259Z" level=error msg="failed to cleanup vm path /run/kata-containers/shared/sandboxes/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020" ID=18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020 error="unlinkat /run/kata-containers/shared/sandboxes/18b5183160fc040e003876fc9c9531d1200989f7fd6223a69f231a6b23ff1020/mounts/b0470ad7a6b70be6b8365a6358bf805edda687f8f1b1eb88bf02428252fce434-b84680ab634462df-serviceaccount: device or resource busy" source=virtcontainers subsystem=kata_agent
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.514954181Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.578383118Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.585244733Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.619687907Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.664178203Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.669924815Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.714727712Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.765279521Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.772312136Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.837361576Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.902739217Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:40 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:40.983729992Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:23:41 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:23:41.056122847Z" level=error msg="post event" error="failed to publish event: exit status 1"
Jul 07 19:24:31 vmss-agent-kata1-test-jfitk000000 kata[6326]: time="2020-07-07T19:24:31.633182456Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
Jul 07 19:48:37 vmss-agent-kata1-test-jfitk000000 kata[19635]: time="2020-07-07T19:48:37.069073056Z" level=error msg="Unable to add memory device mem7: QMP command failed: a used vhost backend has no free memory slots left" ID=c9533b4572784b2f368462c6ef534198485b0e27a3f838d4b037e3f802cc40bc source=virtcontainers subsystem=qmp
Jul 07 19:48:37 vmss-agent-kata1-test-jfitk000000 kata[19635]: time="2020-07-07T19:48:37.069723958Z" level=error msg="hotplug memory" ID=c9533b4572784b2f368462c6ef534198485b0e27a3f838d4b037e3f802cc40bc error="QMP command failed: a used vhost backend has no free memory slots left" source=virtcontainers subsystem=qemu
Jul 07 19:48:37 vmss-agent-kata1-test-jfitk000000 kata[19635]: time="2020-07-07T19:48:37.069824758Z" level=warning msg="failed to cleanup rootfs mount" error="no such file or directory"
```