search

containers / crun

A fast and lightweight fully featured OCI runtime and C library for running containers
GNU General Public License v2.0
3.02k stars 306 forks source link

"creating cgroup directory `/sys/fs/cgroup/machine.slice/libpod-***.scope`: No such file or directory" - nested root podman --privileged container under LXC #678

Closed ntkme closed 3 years ago

ntkme commented 3 years ago

After manually building crun with patch #676, now I'm seeing a different error for nested root podman --privileged container under LXC. Again, the same command with --runtime=runc works fine.

sudo podman --log-level DEBUG run --rm -it --privileged --runtime=/usr/local/bin/crun --cgroup-manager systemd ghcr.io/ntkme/systemd-podman 
INFO[0000] podman filtering at log level debug          
DEBU[0000] Called run.PersistentPreRunE(podman --log-level DEBUG run --rm -it --privileged --runtime=/usr/local/bin/crun --cgroup-manager systemd ghcr.io/ntkme/systemd-podman) 
DEBU[0000] Reading configuration file "/usr/share/containers/containers.conf" 
DEBU[0000] Merged system config "/usr/share/containers/containers.conf": &{Containers:{Devices:[] Volumes:[] ApparmorProfile:containers-default-0.33.4 Annotations:[] CgroupNS:host Cgroups:enabled DefaultCapabilities:[CHOWN DAC_OVERRIDE FOWNER FSETID KILL NET_BIND_SERVICE SETFCAP SETGID SETPCAP SETUID SYS_CHROOT] DefaultSysctls:[net.ipv4.ping_group_range=0 0] DefaultUlimits:[] DefaultMountsFile: DNSServers:[] DNSOptions:[] DNSSearches:[] EnableKeyring:true EnableLabeling:false Env:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin TERM=xterm] EnvHost:false HTTPProxy:true Init:false InitPath: IPCNS:private LogDriver:k8s-file LogSizeMax:-1 NetNS:bridge NoHosts:false PidsLimit:2048 PidNS:private SeccompProfile:/usr/share/containers/seccomp.json ShmSize:65536k TZ: Umask:0022 UTSNS:private UserNS:host UserNSSize:65536} Engine:{ImageBuildFormat:oci CgroupCheck:false CgroupManager:systemd ConmonEnvVars:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin] ConmonPath:[/usr/libexec/podman/conmon /usr/local/libexec/podman/conmon /usr/local/lib/podman/conmon /usr/bin/conmon /usr/sbin/conmon /usr/local/bin/conmon /usr/local/sbin/conmon /run/current-system/sw/bin/conmon] DetachKeys:ctrl-p,ctrl-q EnablePortReservation:true Env:[] EventsLogFilePath:/run/libpod/events/events.log EventsLogger:journald HooksDir:[/usr/share/containers/oci/hooks.d] ImageDefaultTransport:docker:// InfraCommand: InfraImage:k8s.gcr.io/pause:3.2 InitPath:/usr/libexec/podman/catatonit LockType:shm MultiImageArchive:false Namespace: NetworkCmdPath: NetworkCmdOptions:[] NoPivotRoot:false NumLocks:2048 OCIRuntime:crun OCIRuntimes:map[crun:[/usr/bin/crun /usr/sbin/crun /usr/local/bin/crun /usr/local/sbin/crun /sbin/crun /bin/crun /run/current-system/sw/bin/crun] kata:[/usr/bin/kata-runtime /usr/sbin/kata-runtime /usr/local/bin/kata-runtime /usr/local/sbin/kata-runtime /sbin/kata-runtime /bin/kata-runtime /usr/bin/kata-qemu /usr/bin/kata-fc] runc:[/usr/bin/runc /usr/sbin/runc /usr/local/bin/runc /usr/local/sbin/runc /sbin/runc /bin/runc /usr/lib/cri-o-runc/sbin/runc /run/current-system/sw/bin/runc]] PullPolicy:missing Remote:false RemoteURI: RemoteIdentity: ActiveService: ServiceDestinations:map[] RuntimePath:[] RuntimeSupportsJSON:[crun runc] RuntimeSupportsNoCgroups:[crun] RuntimeSupportsKVM:[kata kata-runtime kata-qemu kata-fc] SetOptions:{StorageConfigRunRootSet:false StorageConfigGraphRootSet:false StorageConfigGraphDriverNameSet:false StaticDirSet:false VolumePathSet:false TmpDirSet:false} SignaturePolicyPath:/etc/containers/policy.json SDNotify:false StateType:3 StaticDir:/var/lib/containers/storage/libpod StopTimeout:10 TmpDir:/run/libpod VolumePath:/var/lib/containers/storage/volumes VolumePlugins:map[]} Network:{CNIPluginDirs:[/usr/libexec/cni /usr/lib/cni /usr/local/lib/cni /opt/cni/bin] DefaultNetwork:podman NetworkConfigDir:/etc/cni/net.d/}} 
DEBU[0000] Reading configuration file "/etc/containers/containers.conf" 
DEBU[0000] Merged system config "/etc/containers/containers.conf": &{Containers:{Devices:[] Volumes:[] ApparmorProfile:containers-default-0.33.4 Annotations:[] CgroupNS:host Cgroups:enabled DefaultCapabilities:[CHOWN DAC_OVERRIDE FOWNER FSETID KILL NET_BIND_SERVICE SETFCAP SETGID SETPCAP SETUID SYS_CHROOT] DefaultSysctls:[net.ipv4.ping_group_range=0 0] DefaultUlimits:[] DefaultMountsFile: DNSServers:[] DNSOptions:[] DNSSearches:[] EnableKeyring:false EnableLabeling:false Env:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin TERM=xterm] EnvHost:false HTTPProxy:true Init:false InitPath: IPCNS:private LogDriver:k8s-file LogSizeMax:-1 NetNS:bridge NoHosts:false PidsLimit:2048 PidNS:private SeccompProfile:/usr/share/containers/seccomp.json ShmSize:65536k TZ: Umask:0022 UTSNS:private UserNS:host UserNSSize:65536} Engine:{ImageBuildFormat:oci CgroupCheck:false CgroupManager:systemd ConmonEnvVars:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin] ConmonPath:[/usr/libexec/podman/conmon /usr/local/libexec/podman/conmon /usr/local/lib/podman/conmon /usr/bin/conmon /usr/sbin/conmon /usr/local/bin/conmon /usr/local/sbin/conmon /run/current-system/sw/bin/conmon] DetachKeys:ctrl-p,ctrl-q EnablePortReservation:true Env:[] EventsLogFilePath:/run/libpod/events/events.log EventsLogger:journald HooksDir:[/usr/share/containers/oci/hooks.d] ImageDefaultTransport:docker:// InfraCommand: InfraImage:k8s.gcr.io/pause:3.2 InitPath:/usr/libexec/podman/catatonit LockType:shm MultiImageArchive:false Namespace: NetworkCmdPath: NetworkCmdOptions:[] NoPivotRoot:false NumLocks:2048 OCIRuntime:runc OCIRuntimes:map[crun:[/usr/bin/crun /usr/sbin/crun /usr/local/bin/crun /usr/local/sbin/crun /sbin/crun /bin/crun /run/current-system/sw/bin/crun] kata:[/usr/bin/kata-runtime /usr/sbin/kata-runtime /usr/local/bin/kata-runtime /usr/local/sbin/kata-runtime /sbin/kata-runtime /bin/kata-runtime /usr/bin/kata-qemu /usr/bin/kata-fc] runc:[/usr/bin/runc /usr/sbin/runc /usr/local/bin/runc /usr/local/sbin/runc /sbin/runc /bin/runc /usr/lib/cri-o-runc/sbin/runc /run/current-system/sw/bin/runc]] PullPolicy:missing Remote:false RemoteURI: RemoteIdentity: ActiveService: ServiceDestinations:map[] RuntimePath:[] RuntimeSupportsJSON:[crun runc] RuntimeSupportsNoCgroups:[crun] RuntimeSupportsKVM:[kata kata-runtime kata-qemu kata-fc] SetOptions:{StorageConfigRunRootSet:false StorageConfigGraphRootSet:false StorageConfigGraphDriverNameSet:false StaticDirSet:false VolumePathSet:false TmpDirSet:false} SignaturePolicyPath:/etc/containers/policy.json SDNotify:false StateType:3 StaticDir:/var/lib/containers/storage/libpod StopTimeout:10 TmpDir:/run/libpod VolumePath:/var/lib/containers/storage/volumes VolumePlugins:map[]} Network:{CNIPluginDirs:[/usr/libexec/cni /usr/lib/cni /usr/local/lib/cni /opt/cni/bin] DefaultNetwork:podman NetworkConfigDir:/etc/cni/net.d/}} 
DEBU[0000] Using conmon: "/usr/bin/conmon"              
DEBU[0000] Initializing boltdb state at /var/lib/containers/storage/libpod/bolt_state.db 
DEBU[0000] Using graph driver                           
DEBU[0000] Using graph root /var/lib/containers/storage 
DEBU[0000] Using run root /run/containers/storage       
DEBU[0000] Using static dir /var/lib/containers/storage/libpod 
DEBU[0000] Using tmp dir /run/libpod                    
DEBU[0000] Using volume path /var/lib/containers/storage/volumes 
DEBU[0000] Set libpod namespace to ""                   
INFO[0000] [graphdriver] using prior storage driver: btrfs 
DEBU[0000] Initializing event backend journald          
INFO[0000] Error initializing configured OCI runtime kata: no valid executable found for OCI runtime kata: invalid argument 
DEBU[0000] using runtime "/usr/bin/crun"                
DEBU[0000] using runtime "/usr/bin/runc"                
DEBU[0000] using runtime "/usr/local/bin/crun"          
INFO[0000] Found CNI network podman (type=ptp) at /etc/cni/net.d/87-podman-ptp.conflist 
DEBU[0000] Default CNI network name podman is unchangeable 
INFO[0000] Setting parallel job count to 25             
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]ghcr.io/ntkme/systemd-podman:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]ghcr.io/ntkme/systemd-podman:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@59c75a69953eefdb0f2c0f6039cbe572eac05542286a168706b061f2e46cbcfc" 
DEBU[0000] exporting opaque data as blob "sha256:59c75a69953eefdb0f2c0f6039cbe572eac05542286a168706b061f2e46cbcfc" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]ghcr.io/ntkme/systemd-podman:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@59c75a69953eefdb0f2c0f6039cbe572eac05542286a168706b061f2e46cbcfc" 
DEBU[0000] exporting opaque data as blob "sha256:59c75a69953eefdb0f2c0f6039cbe572eac05542286a168706b061f2e46cbcfc" 
DEBU[0000] Image has volume at "/var/lib/containers"    
DEBU[0000] Adding anonymous image volume at "/var/lib/containers" 
DEBU[0000] using systemd mode: true                     
DEBU[0000] No hostname set; container's hostname will default to runtime default 
DEBU[0000] Loading default seccomp profile              
DEBU[0000] Allocated lock 0 for container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@59c75a69953eefdb0f2c0f6039cbe572eac05542286a168706b061f2e46cbcfc" 
DEBU[0000] exporting opaque data as blob "sha256:59c75a69953eefdb0f2c0f6039cbe572eac05542286a168706b061f2e46cbcfc" 
DEBU[0000] created container "1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be" 
DEBU[0000] container "1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be" has work directory "/var/lib/containers/storage/btrfs-containers/1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be/userdata" 
DEBU[0000] container "1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be" has run directory "/run/containers/storage/btrfs-containers/1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be/userdata" 
DEBU[0000] Creating new volume 4a2db510c21b967788e0a0aa4df8e18c45a105892732e0f39570d158c914d114 for container 
DEBU[0000] Validating options for local driver          
DEBU[0000] Handling terminal attach                     
DEBU[0000] Made network namespace at /run/netns/cni-f33c47ca-bb63-9cae-e13c-598ce484bb3a for container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be 
DEBU[0000] mounted container "1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be" at "/var/lib/containers/storage/btrfs/subvolumes/128c42e9de4b3fb4d3d7b59b830a874111119c11a71312db22ceba2aa221ebd0" 
DEBU[0000] Going to mount named volume 4a2db510c21b967788e0a0aa4df8e18c45a105892732e0f39570d158c914d114 
DEBU[0000] Copying up contents from container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be to volume 4a2db510c21b967788e0a0aa4df8e18c45a105892732e0f39570d158c914d114 
INFO[0000] Got pod network &{Name:dazzling_engelbart Namespace:dazzling_engelbart ID:1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be NetNS:/run/netns/cni-f33c47ca-bb63-9cae-e13c-598ce484bb3a Networks:[{Name:podman Ifname:eth0}] RuntimeConfig:map[podman:{IP: MAC: PortMappings:[] Bandwidth: IpRanges:[]}] Aliases:map[]} 
INFO[0000] About to add CNI network podman (type=ptp)   
INFO[0000] About to copy up into volume 4a2db510c21b967788e0a0aa4df8e18c45a105892732e0f39570d158c914d114 
DEBU[0000] Created root filesystem for container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be at /var/lib/containers/storage/btrfs/subvolumes/128c42e9de4b3fb4d3d7b59b830a874111119c11a71312db22ceba2aa221ebd0 
DEBU[0000] [0] CNI result: &{0.4.0 [{Name:veth27390bef Mac:26:9c:76:19:fd:4e Sandbox:} {Name:eth0 Mac:e2:cd:02:52:e1:91 Sandbox:/run/netns/cni-f33c47ca-bb63-9cae-e13c-598ce484bb3a}] [{Version:4 Interface:0xc00039fb88 Address:{IP:172.16.16.76 Mask:ffffff00} Gateway:172.16.16.1}] [{Dst:{IP:0.0.0.0 Mask:00000000} GW:}] {[]  [] []}} 
DEBU[0000] Workdir "/" resolved to host path "/var/lib/containers/storage/btrfs/subvolumes/128c42e9de4b3fb4d3d7b59b830a874111119c11a71312db22ceba2aa221ebd0" 
DEBU[0000] /etc/system-fips does not exist on host, not mounting FIPS mode subscription 
DEBU[0000] Setting CGroups for container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be to machine.slice:libpod:1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be 
DEBU[0000] reading hooks from /usr/share/containers/oci/hooks.d 
DEBU[0000] Created OCI spec for container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be at /var/lib/containers/storage/btrfs-containers/1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be/userdata/config.json 
DEBU[0000] /usr/bin/conmon messages will be logged to syslog 
DEBU[0000] running conmon: /usr/bin/conmon               args="[--api-version 1 -c 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be -u 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be -r /usr/local/bin/crun -b /var/lib/containers/storage/btrfs-containers/1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be/userdata -p /run/containers/storage/btrfs-containers/1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be/userdata/pidfile -n dazzling_engelbart --exit-dir /run/libpod/exits --socket-dir-path /run/libpod/socket -s -l k8s-file:/var/lib/containers/storage/btrfs-containers/1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be/userdata/ctr.log --log-level debug --syslog -t --no-new-keyring --conmon-pidfile /run/containers/storage/btrfs-containers/1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg debug --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --runtime --exit-command-arg /usr/local/bin/crun --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg --syslog --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be]"
INFO[0000] Running conmon under slice machine.slice and unitName libpod-conmon-1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be.scope 
DEBU[0000] Received: -1                                 
DEBU[0000] Cleaning up container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be 
DEBU[0000] Tearing down network namespace at /run/netns/cni-f33c47ca-bb63-9cae-e13c-598ce484bb3a for container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be 
INFO[0000] Got pod network &{Name:dazzling_engelbart Namespace:dazzling_engelbart ID:1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be NetNS:/run/netns/cni-f33c47ca-bb63-9cae-e13c-598ce484bb3a Networks:[{Name:podman Ifname:}] RuntimeConfig:map[podman:{IP: MAC: PortMappings:[] Bandwidth: IpRanges:[]}] Aliases:map[]} 
INFO[0000] About to del CNI network podman (type=ptp)   
DEBU[0000] unmounted container "1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be" 
DEBU[0000] Removing container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be 
DEBU[0000] Removing all exec sessions for container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be 
DEBU[0000] Cleaning up container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be 
DEBU[0000] Network is already cleaned up, skipping...   
DEBU[0000] Container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be storage is already unmounted, skipping... 
DEBU[0000] Container 1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be storage is already unmounted, skipping... 
DEBU[0000] ExitCode msg: "creating cgroup directory `/sys/fs/cgroup/machine.slice/libpod-1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be.scope`: no such file or directory: oci not found" 
Error: creating cgroup directory `/sys/fs/cgroup/machine.slice/libpod-1aa9e0d970b1e81e65f3a1cd34278138b849dc5ad4c80453c6ae4ee54dbde9be.scope`: No such file or directory: OCI not found

strace.log

The error is coming from here:

https://github.com/containers/crun/blob/4cc7fa1124cce75dc26e12186d9cbeabded2b710/src/libcrun/cgroup.c#L536-L546

giuseppe commented 3 years ago

is systemd available inside the container?

ntkme commented 3 years ago
  1. systemd is running in the parent LXC container.
  2. systemd (/sbin/init) is also the start up command inside the child podman container. The same error happens if I switch to use just docker.io/library/alpine.
ntkme commented 3 years ago

Did more tests, runc works with either --cgroup-manager systemd or --cgroup-manager cgroupfs:

sudo podman --log-level DEBUG run --rm -it --privileged --runtime=runc --cgroup-manager systemd docker.io/library/alpine
sudo podman --log-level DEBUG run --rm -it --privileged --runtime=runc --cgroup-manager cgroupfs docker.io/library/alpine

crun fails with either --cgroup-manager systemd or --cgroup-manager cgroupfs:

sudo podman --log-level DEBUG run --rm -it --privileged --runtime=/usr/local/bin/crun --cgroup-manager systemd docker.io/library/alpine 
INFO[0000] podman filtering at log level debug          
DEBU[0000] Called run.PersistentPreRunE(podman --log-level DEBUG run --rm -it --privileged --runtime=/usr/local/bin/crun --cgroup-manager systemd docker.io/library/alpine) 
DEBU[0000] Reading configuration file "/usr/share/containers/containers.conf" 
DEBU[0000] Merged system config "/usr/share/containers/containers.conf": &{Containers:{Devices:[] Volumes:[] ApparmorProfile:containers-default-0.33.4 Annotations:[] CgroupNS:host Cgroups:enabled DefaultCapabilities:[CHOWN DAC_OVERRIDE FOWNER FSETID KILL NET_BIND_SERVICE SETFCAP SETGID SETPCAP SETUID SYS_CHROOT] DefaultSysctls:[net.ipv4.ping_group_range=0 0] DefaultUlimits:[] DefaultMountsFile: DNSServers:[] DNSOptions:[] DNSSearches:[] EnableKeyring:true EnableLabeling:false Env:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin TERM=xterm] EnvHost:false HTTPProxy:true Init:false InitPath: IPCNS:private LogDriver:k8s-file LogSizeMax:-1 NetNS:bridge NoHosts:false PidsLimit:2048 PidNS:private SeccompProfile:/usr/share/containers/seccomp.json ShmSize:65536k TZ: Umask:0022 UTSNS:private UserNS:host UserNSSize:65536} Engine:{ImageBuildFormat:oci CgroupCheck:false CgroupManager:systemd ConmonEnvVars:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin] ConmonPath:[/usr/libexec/podman/conmon /usr/local/libexec/podman/conmon /usr/local/lib/podman/conmon /usr/bin/conmon /usr/sbin/conmon /usr/local/bin/conmon /usr/local/sbin/conmon /run/current-system/sw/bin/conmon] DetachKeys:ctrl-p,ctrl-q EnablePortReservation:true Env:[] EventsLogFilePath:/run/libpod/events/events.log EventsLogger:journald HooksDir:[/usr/share/containers/oci/hooks.d] ImageDefaultTransport:docker:// InfraCommand: InfraImage:k8s.gcr.io/pause:3.2 InitPath:/usr/libexec/podman/catatonit LockType:shm MultiImageArchive:false Namespace: NetworkCmdPath: NetworkCmdOptions:[] NoPivotRoot:false NumLocks:2048 OCIRuntime:crun OCIRuntimes:map[crun:[/usr/bin/crun /usr/sbin/crun /usr/local/bin/crun /usr/local/sbin/crun /sbin/crun /bin/crun /run/current-system/sw/bin/crun] kata:[/usr/bin/kata-runtime /usr/sbin/kata-runtime /usr/local/bin/kata-runtime /usr/local/sbin/kata-runtime /sbin/kata-runtime /bin/kata-runtime /usr/bin/kata-qemu /usr/bin/kata-fc] runc:[/usr/bin/runc /usr/sbin/runc /usr/local/bin/runc /usr/local/sbin/runc /sbin/runc /bin/runc /usr/lib/cri-o-runc/sbin/runc /run/current-system/sw/bin/runc]] PullPolicy:missing Remote:false RemoteURI: RemoteIdentity: ActiveService: ServiceDestinations:map[] RuntimePath:[] RuntimeSupportsJSON:[crun runc] RuntimeSupportsNoCgroups:[crun] RuntimeSupportsKVM:[kata kata-runtime kata-qemu kata-fc] SetOptions:{StorageConfigRunRootSet:false StorageConfigGraphRootSet:false StorageConfigGraphDriverNameSet:false StaticDirSet:false VolumePathSet:false TmpDirSet:false} SignaturePolicyPath:/etc/containers/policy.json SDNotify:false StateType:3 StaticDir:/var/lib/containers/storage/libpod StopTimeout:10 TmpDir:/run/libpod VolumePath:/var/lib/containers/storage/volumes VolumePlugins:map[]} Network:{CNIPluginDirs:[/usr/libexec/cni /usr/lib/cni /usr/local/lib/cni /opt/cni/bin] DefaultNetwork:podman NetworkConfigDir:/etc/cni/net.d/}} 
DEBU[0000] Reading configuration file "/etc/containers/containers.conf" 
DEBU[0000] Merged system config "/etc/containers/containers.conf": &{Containers:{Devices:[] Volumes:[] ApparmorProfile:containers-default-0.33.4 Annotations:[] CgroupNS:host Cgroups:enabled DefaultCapabilities:[CHOWN DAC_OVERRIDE FOWNER FSETID KILL NET_BIND_SERVICE SETFCAP SETGID SETPCAP SETUID SYS_CHROOT] DefaultSysctls:[net.ipv4.ping_group_range=0 0] DefaultUlimits:[] DefaultMountsFile: DNSServers:[] DNSOptions:[] DNSSearches:[] EnableKeyring:false EnableLabeling:false Env:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin TERM=xterm] EnvHost:false HTTPProxy:true Init:false InitPath: IPCNS:private LogDriver:k8s-file LogSizeMax:-1 NetNS:bridge NoHosts:false PidsLimit:2048 PidNS:private SeccompProfile:/usr/share/containers/seccomp.json ShmSize:65536k TZ: Umask:0022 UTSNS:private UserNS:host UserNSSize:65536} Engine:{ImageBuildFormat:oci CgroupCheck:false CgroupManager:systemd ConmonEnvVars:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin] ConmonPath:[/usr/libexec/podman/conmon /usr/local/libexec/podman/conmon /usr/local/lib/podman/conmon /usr/bin/conmon /usr/sbin/conmon /usr/local/bin/conmon /usr/local/sbin/conmon /run/current-system/sw/bin/conmon] DetachKeys:ctrl-p,ctrl-q EnablePortReservation:true Env:[] EventsLogFilePath:/run/libpod/events/events.log EventsLogger:journald HooksDir:[/usr/share/containers/oci/hooks.d] ImageDefaultTransport:docker:// InfraCommand: InfraImage:k8s.gcr.io/pause:3.2 InitPath:/usr/libexec/podman/catatonit LockType:shm MultiImageArchive:false Namespace: NetworkCmdPath: NetworkCmdOptions:[] NoPivotRoot:false NumLocks:2048 OCIRuntime:runc OCIRuntimes:map[crun:[/usr/bin/crun /usr/sbin/crun /usr/local/bin/crun /usr/local/sbin/crun /sbin/crun /bin/crun /run/current-system/sw/bin/crun] kata:[/usr/bin/kata-runtime /usr/sbin/kata-runtime /usr/local/bin/kata-runtime /usr/local/sbin/kata-runtime /sbin/kata-runtime /bin/kata-runtime /usr/bin/kata-qemu /usr/bin/kata-fc] runc:[/usr/bin/runc /usr/sbin/runc /usr/local/bin/runc /usr/local/sbin/runc /sbin/runc /bin/runc /usr/lib/cri-o-runc/sbin/runc /run/current-system/sw/bin/runc]] PullPolicy:missing Remote:false RemoteURI: RemoteIdentity: ActiveService: ServiceDestinations:map[] RuntimePath:[] RuntimeSupportsJSON:[crun runc] RuntimeSupportsNoCgroups:[crun] RuntimeSupportsKVM:[kata kata-runtime kata-qemu kata-fc] SetOptions:{StorageConfigRunRootSet:false StorageConfigGraphRootSet:false StorageConfigGraphDriverNameSet:false StaticDirSet:false VolumePathSet:false TmpDirSet:false} SignaturePolicyPath:/etc/containers/policy.json SDNotify:false StateType:3 StaticDir:/var/lib/containers/storage/libpod StopTimeout:10 TmpDir:/run/libpod VolumePath:/var/lib/containers/storage/volumes VolumePlugins:map[]} Network:{CNIPluginDirs:[/usr/libexec/cni /usr/lib/cni /usr/local/lib/cni /opt/cni/bin] DefaultNetwork:podman NetworkConfigDir:/etc/cni/net.d/}} 
DEBU[0000] Using conmon: "/usr/bin/conmon"              
DEBU[0000] Initializing boltdb state at /var/lib/containers/storage/libpod/bolt_state.db 
DEBU[0000] Using graph driver                           
DEBU[0000] Using graph root /var/lib/containers/storage 
DEBU[0000] Using run root /run/containers/storage       
DEBU[0000] Using static dir /var/lib/containers/storage/libpod 
DEBU[0000] Using tmp dir /run/libpod                    
DEBU[0000] Using volume path /var/lib/containers/storage/volumes 
DEBU[0000] Set libpod namespace to ""                   
INFO[0000] [graphdriver] using prior storage driver: btrfs 
DEBU[0000] Initializing event backend journald          
DEBU[0000] using runtime "/usr/bin/crun"                
DEBU[0000] using runtime "/usr/bin/runc"                
INFO[0000] Error initializing configured OCI runtime kata: no valid executable found for OCI runtime kata: invalid argument 
DEBU[0000] using runtime "/usr/local/bin/crun"          
INFO[0000] Found CNI network podman (type=ptp) at /etc/cni/net.d/87-podman-ptp.conflist 
DEBU[0000] Default CNI network name podman is unchangeable 
INFO[0000] Setting parallel job count to 25             
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]docker.io/library/alpine:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]docker.io/library/alpine:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] exporting opaque data as blob "sha256:6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]docker.io/library/alpine:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] exporting opaque data as blob "sha256:6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] using systemd mode: false                    
DEBU[0000] No hostname set; container's hostname will default to runtime default 
DEBU[0000] Loading default seccomp profile              
DEBU[0000] Allocated lock 0 for container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] exporting opaque data as blob "sha256:6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] created container "7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8" 
DEBU[0000] container "7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8" has work directory "/var/lib/containers/storage/btrfs-containers/7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8/userdata" 
DEBU[0000] container "7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8" has run directory "/run/containers/storage/btrfs-containers/7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8/userdata" 
DEBU[0000] Handling terminal attach                     
DEBU[0000] Made network namespace at /run/netns/cni-fe83ba9c-2274-ab28-7722-363c7b8edf72 for container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 
DEBU[0000] mounted container "7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8" at "/var/lib/containers/storage/btrfs/subvolumes/2a344a119fc233b0963ff55105ef469f7bc65aadffa337e95021d41a64cfeb81" 
DEBU[0000] Created root filesystem for container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 at /var/lib/containers/storage/btrfs/subvolumes/2a344a119fc233b0963ff55105ef469f7bc65aadffa337e95021d41a64cfeb81 
INFO[0000] Got pod network &{Name:dazzling_hermann Namespace:dazzling_hermann ID:7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 NetNS:/run/netns/cni-fe83ba9c-2274-ab28-7722-363c7b8edf72 Networks:[{Name:podman Ifname:eth0}] RuntimeConfig:map[podman:{IP: MAC: PortMappings:[] Bandwidth: IpRanges:[]}] Aliases:map[]} 
INFO[0000] About to add CNI network podman (type=ptp)   
DEBU[0000] [0] CNI result: &{0.4.0 [{Name:veth27f3acf4 Mac:7a:28:49:f9:c0:14 Sandbox:} {Name:eth0 Mac:d2:9b:c9:9e:0f:a7 Sandbox:/run/netns/cni-fe83ba9c-2274-ab28-7722-363c7b8edf72}] [{Version:4 Interface:0xc000130b38 Address:{IP:172.16.16.83 Mask:ffffff00} Gateway:172.16.16.1}] [{Dst:{IP:0.0.0.0 Mask:00000000} GW:}] {[]  [] []}} 
DEBU[0000] Workdir "/" resolved to host path "/var/lib/containers/storage/btrfs/subvolumes/2a344a119fc233b0963ff55105ef469f7bc65aadffa337e95021d41a64cfeb81" 
DEBU[0000] /etc/system-fips does not exist on host, not mounting FIPS mode subscription 
DEBU[0000] Setting CGroups for container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 to machine.slice:libpod:7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 
DEBU[0000] reading hooks from /usr/share/containers/oci/hooks.d 
DEBU[0000] Created OCI spec for container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 at /var/lib/containers/storage/btrfs-containers/7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8/userdata/config.json 
DEBU[0000] /usr/bin/conmon messages will be logged to syslog 
DEBU[0000] running conmon: /usr/bin/conmon               args="[--api-version 1 -c 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 -u 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 -r /usr/local/bin/crun -b /var/lib/containers/storage/btrfs-containers/7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8/userdata -p /run/containers/storage/btrfs-containers/7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8/userdata/pidfile -n dazzling_hermann --exit-dir /run/libpod/exits --socket-dir-path /run/libpod/socket -s -l k8s-file:/var/lib/containers/storage/btrfs-containers/7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8/userdata/ctr.log --log-level debug --syslog -t --no-new-keyring --conmon-pidfile /run/containers/storage/btrfs-containers/7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg debug --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --runtime --exit-command-arg /usr/local/bin/crun --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg --syslog --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8]"
INFO[0000] Running conmon under slice machine.slice and unitName libpod-conmon-7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8.scope 
DEBU[0000] Received: -1                                 
DEBU[0000] Cleaning up container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 
DEBU[0000] Tearing down network namespace at /run/netns/cni-fe83ba9c-2274-ab28-7722-363c7b8edf72 for container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 
INFO[0000] Got pod network &{Name:dazzling_hermann Namespace:dazzling_hermann ID:7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 NetNS:/run/netns/cni-fe83ba9c-2274-ab28-7722-363c7b8edf72 Networks:[{Name:podman Ifname:}] RuntimeConfig:map[podman:{IP: MAC: PortMappings:[] Bandwidth: IpRanges:[]}] Aliases:map[]} 
INFO[0000] About to del CNI network podman (type=ptp)   
DEBU[0000] unmounted container "7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8" 
DEBU[0000] Removing container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 
DEBU[0000] Removing all exec sessions for container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 
DEBU[0000] Cleaning up container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 
DEBU[0000] Network is already cleaned up, skipping...   
DEBU[0000] Container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 storage is already unmounted, skipping... 
DEBU[0000] Container 7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8 storage is already unmounted, skipping... 
DEBU[0000] ExitCode msg: "creating cgroup directory `/sys/fs/cgroup/machine.slice/libpod-7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8.scope`: no such file or directory: oci not found" 
Error: creating cgroup directory `/sys/fs/cgroup/machine.slice/libpod-7b844e8928936172c0319e274723585860dcae226172a0268929dae31aac22a8.scope`: No such file or directory: OCI not found
sudo podman --log-level DEBUG run --rm -it --privileged --runtime=/usr/local/bin/crun --cgroup-manager cgroupfs docker.io/library/alpine 
INFO[0000] podman filtering at log level debug          
DEBU[0000] Called run.PersistentPreRunE(podman --log-level DEBUG run --rm -it --privileged --runtime=/usr/local/bin/crun --cgroup-manager cgroupfs docker.io/library/alpine) 
DEBU[0000] Reading configuration file "/usr/share/containers/containers.conf" 
DEBU[0000] Merged system config "/usr/share/containers/containers.conf": &{Containers:{Devices:[] Volumes:[] ApparmorProfile:containers-default-0.33.4 Annotations:[] CgroupNS:host Cgroups:enabled DefaultCapabilities:[CHOWN DAC_OVERRIDE FOWNER FSETID KILL NET_BIND_SERVICE SETFCAP SETGID SETPCAP SETUID SYS_CHROOT] DefaultSysctls:[net.ipv4.ping_group_range=0 0] DefaultUlimits:[] DefaultMountsFile: DNSServers:[] DNSOptions:[] DNSSearches:[] EnableKeyring:true EnableLabeling:false Env:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin TERM=xterm] EnvHost:false HTTPProxy:true Init:false InitPath: IPCNS:private LogDriver:k8s-file LogSizeMax:-1 NetNS:bridge NoHosts:false PidsLimit:2048 PidNS:private SeccompProfile:/usr/share/containers/seccomp.json ShmSize:65536k TZ: Umask:0022 UTSNS:private UserNS:host UserNSSize:65536} Engine:{ImageBuildFormat:oci CgroupCheck:false CgroupManager:systemd ConmonEnvVars:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin] ConmonPath:[/usr/libexec/podman/conmon /usr/local/libexec/podman/conmon /usr/local/lib/podman/conmon /usr/bin/conmon /usr/sbin/conmon /usr/local/bin/conmon /usr/local/sbin/conmon /run/current-system/sw/bin/conmon] DetachKeys:ctrl-p,ctrl-q EnablePortReservation:true Env:[] EventsLogFilePath:/run/libpod/events/events.log EventsLogger:journald HooksDir:[/usr/share/containers/oci/hooks.d] ImageDefaultTransport:docker:// InfraCommand: InfraImage:k8s.gcr.io/pause:3.2 InitPath:/usr/libexec/podman/catatonit LockType:shm MultiImageArchive:false Namespace: NetworkCmdPath: NetworkCmdOptions:[] NoPivotRoot:false NumLocks:2048 OCIRuntime:crun OCIRuntimes:map[crun:[/usr/bin/crun /usr/sbin/crun /usr/local/bin/crun /usr/local/sbin/crun /sbin/crun /bin/crun /run/current-system/sw/bin/crun] kata:[/usr/bin/kata-runtime /usr/sbin/kata-runtime /usr/local/bin/kata-runtime /usr/local/sbin/kata-runtime /sbin/kata-runtime /bin/kata-runtime /usr/bin/kata-qemu /usr/bin/kata-fc] runc:[/usr/bin/runc /usr/sbin/runc /usr/local/bin/runc /usr/local/sbin/runc /sbin/runc /bin/runc /usr/lib/cri-o-runc/sbin/runc /run/current-system/sw/bin/runc]] PullPolicy:missing Remote:false RemoteURI: RemoteIdentity: ActiveService: ServiceDestinations:map[] RuntimePath:[] RuntimeSupportsJSON:[crun runc] RuntimeSupportsNoCgroups:[crun] RuntimeSupportsKVM:[kata kata-runtime kata-qemu kata-fc] SetOptions:{StorageConfigRunRootSet:false StorageConfigGraphRootSet:false StorageConfigGraphDriverNameSet:false StaticDirSet:false VolumePathSet:false TmpDirSet:false} SignaturePolicyPath:/etc/containers/policy.json SDNotify:false StateType:3 StaticDir:/var/lib/containers/storage/libpod StopTimeout:10 TmpDir:/run/libpod VolumePath:/var/lib/containers/storage/volumes VolumePlugins:map[]} Network:{CNIPluginDirs:[/usr/libexec/cni /usr/lib/cni /usr/local/lib/cni /opt/cni/bin] DefaultNetwork:podman NetworkConfigDir:/etc/cni/net.d/}} 
DEBU[0000] Reading configuration file "/etc/containers/containers.conf" 
DEBU[0000] Merged system config "/etc/containers/containers.conf": &{Containers:{Devices:[] Volumes:[] ApparmorProfile:containers-default-0.33.4 Annotations:[] CgroupNS:host Cgroups:enabled DefaultCapabilities:[CHOWN DAC_OVERRIDE FOWNER FSETID KILL NET_BIND_SERVICE SETFCAP SETGID SETPCAP SETUID SYS_CHROOT] DefaultSysctls:[net.ipv4.ping_group_range=0 0] DefaultUlimits:[] DefaultMountsFile: DNSServers:[] DNSOptions:[] DNSSearches:[] EnableKeyring:false EnableLabeling:false Env:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin TERM=xterm] EnvHost:false HTTPProxy:true Init:false InitPath: IPCNS:private LogDriver:k8s-file LogSizeMax:-1 NetNS:bridge NoHosts:false PidsLimit:2048 PidNS:private SeccompProfile:/usr/share/containers/seccomp.json ShmSize:65536k TZ: Umask:0022 UTSNS:private UserNS:host UserNSSize:65536} Engine:{ImageBuildFormat:oci CgroupCheck:false CgroupManager:systemd ConmonEnvVars:[PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin] ConmonPath:[/usr/libexec/podman/conmon /usr/local/libexec/podman/conmon /usr/local/lib/podman/conmon /usr/bin/conmon /usr/sbin/conmon /usr/local/bin/conmon /usr/local/sbin/conmon /run/current-system/sw/bin/conmon] DetachKeys:ctrl-p,ctrl-q EnablePortReservation:true Env:[] EventsLogFilePath:/run/libpod/events/events.log EventsLogger:journald HooksDir:[/usr/share/containers/oci/hooks.d] ImageDefaultTransport:docker:// InfraCommand: InfraImage:k8s.gcr.io/pause:3.2 InitPath:/usr/libexec/podman/catatonit LockType:shm MultiImageArchive:false Namespace: NetworkCmdPath: NetworkCmdOptions:[] NoPivotRoot:false NumLocks:2048 OCIRuntime:runc OCIRuntimes:map[crun:[/usr/bin/crun /usr/sbin/crun /usr/local/bin/crun /usr/local/sbin/crun /sbin/crun /bin/crun /run/current-system/sw/bin/crun] kata:[/usr/bin/kata-runtime /usr/sbin/kata-runtime /usr/local/bin/kata-runtime /usr/local/sbin/kata-runtime /sbin/kata-runtime /bin/kata-runtime /usr/bin/kata-qemu /usr/bin/kata-fc] runc:[/usr/bin/runc /usr/sbin/runc /usr/local/bin/runc /usr/local/sbin/runc /sbin/runc /bin/runc /usr/lib/cri-o-runc/sbin/runc /run/current-system/sw/bin/runc]] PullPolicy:missing Remote:false RemoteURI: RemoteIdentity: ActiveService: ServiceDestinations:map[] RuntimePath:[] RuntimeSupportsJSON:[crun runc] RuntimeSupportsNoCgroups:[crun] RuntimeSupportsKVM:[kata kata-runtime kata-qemu kata-fc] SetOptions:{StorageConfigRunRootSet:false StorageConfigGraphRootSet:false StorageConfigGraphDriverNameSet:false StaticDirSet:false VolumePathSet:false TmpDirSet:false} SignaturePolicyPath:/etc/containers/policy.json SDNotify:false StateType:3 StaticDir:/var/lib/containers/storage/libpod StopTimeout:10 TmpDir:/run/libpod VolumePath:/var/lib/containers/storage/volumes VolumePlugins:map[]} Network:{CNIPluginDirs:[/usr/libexec/cni /usr/lib/cni /usr/local/lib/cni /opt/cni/bin] DefaultNetwork:podman NetworkConfigDir:/etc/cni/net.d/}} 
DEBU[0000] Using conmon: "/usr/bin/conmon"              
DEBU[0000] Initializing boltdb state at /var/lib/containers/storage/libpod/bolt_state.db 
DEBU[0000] Using graph driver                           
DEBU[0000] Using graph root /var/lib/containers/storage 
DEBU[0000] Using run root /run/containers/storage       
DEBU[0000] Using static dir /var/lib/containers/storage/libpod 
DEBU[0000] Using tmp dir /run/libpod                    
DEBU[0000] Using volume path /var/lib/containers/storage/volumes 
DEBU[0000] Set libpod namespace to ""                   
INFO[0000] [graphdriver] using prior storage driver: btrfs 
DEBU[0000] Initializing event backend journald          
DEBU[0000] using runtime "/usr/bin/crun"                
DEBU[0000] using runtime "/usr/bin/runc"                
INFO[0000] Error initializing configured OCI runtime kata: no valid executable found for OCI runtime kata: invalid argument 
DEBU[0000] using runtime "/usr/local/bin/crun"          
INFO[0000] Found CNI network podman (type=ptp) at /etc/cni/net.d/87-podman-ptp.conflist 
DEBU[0000] Default CNI network name podman is unchangeable 
INFO[0000] Setting parallel job count to 25             
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]docker.io/library/alpine:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]docker.io/library/alpine:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] exporting opaque data as blob "sha256:6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]docker.io/library/alpine:latest" 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] exporting opaque data as blob "sha256:6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] using systemd mode: false                    
DEBU[0000] No hostname set; container's hostname will default to runtime default 
DEBU[0000] Loading default seccomp profile              
DEBU[0000] Allocated lock 0 for container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e 
DEBU[0000] parsed reference into "[btrfs@/var/lib/containers/storage+/run/containers/storage]@6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] exporting opaque data as blob "sha256:6dbb9cc54074106d46d4ccb330f2a40a682d49dda5f4844962b7dce9fe44aaec" 
DEBU[0000] created container "5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e" 
DEBU[0000] container "5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e" has work directory "/var/lib/containers/storage/btrfs-containers/5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e/userdata" 
DEBU[0000] container "5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e" has run directory "/run/containers/storage/btrfs-containers/5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e/userdata" 
DEBU[0000] Handling terminal attach                     
DEBU[0000] mounted container "5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e" at "/var/lib/containers/storage/btrfs/subvolumes/9a8aba20cae02e1dda2eefc10b5005174d4101767346ef52ba02b576613c95fc" 
DEBU[0000] Made network namespace at /run/netns/cni-5e0e72a2-e795-a918-5204-6a628b6f8117 for container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e 
DEBU[0000] Created root filesystem for container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e at /var/lib/containers/storage/btrfs/subvolumes/9a8aba20cae02e1dda2eefc10b5005174d4101767346ef52ba02b576613c95fc 
INFO[0000] Got pod network &{Name:wizardly_cori Namespace:wizardly_cori ID:5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e NetNS:/run/netns/cni-5e0e72a2-e795-a918-5204-6a628b6f8117 Networks:[{Name:podman Ifname:eth0}] RuntimeConfig:map[podman:{IP: MAC: PortMappings:[] Bandwidth: IpRanges:[]}] Aliases:map[]} 
INFO[0000] About to add CNI network podman (type=ptp)   
DEBU[0000] [0] CNI result: &{0.4.0 [{Name:vetheba2b6a4 Mac:66:fb:54:bd:c5:d3 Sandbox:} {Name:eth0 Mac:2a:d2:ea:a8:34:fb Sandbox:/run/netns/cni-5e0e72a2-e795-a918-5204-6a628b6f8117}] [{Version:4 Interface:0xc0000d4258 Address:{IP:172.16.16.82 Mask:ffffff00} Gateway:172.16.16.1}] [{Dst:{IP:0.0.0.0 Mask:00000000} GW:}] {[]  [] []}} 
DEBU[0000] Workdir "/" resolved to host path "/var/lib/containers/storage/btrfs/subvolumes/9a8aba20cae02e1dda2eefc10b5005174d4101767346ef52ba02b576613c95fc" 
DEBU[0000] /etc/system-fips does not exist on host, not mounting FIPS mode subscription 
DEBU[0000] Setting CGroup path for container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e to /libpod_parent/libpod-5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e 
DEBU[0000] reading hooks from /usr/share/containers/oci/hooks.d 
DEBU[0000] Created OCI spec for container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e at /var/lib/containers/storage/btrfs-containers/5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e/userdata/config.json 
DEBU[0000] /usr/bin/conmon messages will be logged to syslog 
DEBU[0000] running conmon: /usr/bin/conmon               args="[--api-version 1 -c 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e -u 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e -r /usr/local/bin/crun -b /var/lib/containers/storage/btrfs-containers/5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e/userdata -p /run/containers/storage/btrfs-containers/5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e/userdata/pidfile -n wizardly_cori --exit-dir /run/libpod/exits --socket-dir-path /run/libpod/socket -l k8s-file:/var/lib/containers/storage/btrfs-containers/5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e/userdata/ctr.log --log-level debug --syslog -t --no-new-keyring --conmon-pidfile /run/containers/storage/btrfs-containers/5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /var/lib/containers/storage --exit-command-arg --runroot --exit-command-arg /run/containers/storage --exit-command-arg --log-level --exit-command-arg debug --exit-command-arg --cgroup-manager --exit-command-arg cgroupfs --exit-command-arg --tmpdir --exit-command-arg /run/libpod --exit-command-arg --runtime --exit-command-arg /usr/local/bin/crun --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg --syslog --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --rm --exit-command-arg 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e]"
WARN[0000] Failed to add conmon to cgroupfs sandbox cgroup: error creating cgroup path for debug: mkdir /sys/fs/cgroup/debug/libpod_parent: read-only file system 
DEBU[0000] Received: -1                                 
DEBU[0000] Cleaning up container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e 
DEBU[0000] Tearing down network namespace at /run/netns/cni-5e0e72a2-e795-a918-5204-6a628b6f8117 for container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e 
INFO[0000] Got pod network &{Name:wizardly_cori Namespace:wizardly_cori ID:5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e NetNS:/run/netns/cni-5e0e72a2-e795-a918-5204-6a628b6f8117 Networks:[{Name:podman Ifname:}] RuntimeConfig:map[podman:{IP: MAC: PortMappings:[] Bandwidth: IpRanges:[]}] Aliases:map[]} 
INFO[0000] About to del CNI network podman (type=ptp)   
DEBU[0000] unmounted container "5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e" 
DEBU[0000] Removing container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e 
DEBU[0000] Removing all exec sessions for container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e 
DEBU[0000] Cleaning up container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e 
DEBU[0000] Network is already cleaned up, skipping...   
DEBU[0000] Container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e storage is already unmounted, skipping... 
DEBU[0000] Container 5d85917f227c25802b2f4962fbaf74bec800ffc0a48176f39ccbc39672089d5e storage is already unmounted, skipping... 
DEBU[0000] ExitCode msg: "writing file `devices.allow`: operation not permitted: oci permission denied" 
Error: writing file `devices.allow`: Operation not permitted: OCI permission denied
rhatdan commented 3 years ago

I think in the second comment you meant crun fails.

@giuseppe PTAL

giuseppe commented 3 years ago

Do you have a recipe for how I can reproduce locally? Is the host running LXC on Ubuntu?

ntkme commented 3 years ago

This environment is marketed by Google as "Linux on Chromebook", it looks like below:

└── Chrome OS host
    └── Chrome OS virtual machine (termina)
        └── LXC container - Debian running systemd (penguin)
            └── `sudo podman --runtime crun run`

I don't have Ubuntu to test whether is can be reproduced there. So I collected some information about this environment below. Please let me know if there is anything also you want to see.

(termina) chronos@localhost ~ $ uname -r
5.4.99-12983-g8b7876ab9f5e
(termina) chronos@localhost ~ $ lxc version
Client version: 3.17
Server version: 3.17
(termina) chronos@localhost ~ $ lxc config show penguin --expanded 
architecture: x86_64
config:
  boot.autostart: "false"
  boot.host_shutdown_timeout: "9"
  image.architecture: amd64
  image.description: Debian buster amd64 (20210206_14:16)
  image.os: Debian
  image.release: buster
  image.serial: "20210206_14:16"
  image.type: squashfs
  raw.idmap: |-
    both 1000 1000
    both 655360 655360
    both 665357 665357
    both 1001 1001
  security.nesting: "true"
  security.syscalls.blacklist: keyctl errno 38
  volatile.base_image: 89260331cf49c476c9cf40b3d38f86c594ff5baba022981c4d44309b95b7c5a5
  volatile.eth0.host_name: veth290cadd0
  volatile.eth0.hwaddr: 00:16:3e:30:d1:67
  volatile.eth0.name: eth0
  volatile.idmap.base: "0"
  volatile.idmap.current: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000},{"Isuid":true,"Isgid":true,"Hostid":1000,"Nsid":1000,"Maprange":1},{"Isuid":true,"Isgid":true,"Hostid":1001,"Nsid":1001,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1001002,"Nsid":1002,"Maprange":654358},{"Isuid":true,"Isgid":true,"Hostid":655360,"Nsid":655360,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1655361,"Nsid":655361,"Maprange":9996},{"Isuid":true,"Isgid":true,"Hostid":665357,"Nsid":665357,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1665358,"Nsid":665358,"Maprange":999334642},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000},{"Isuid":true,"Isgid":true,"Hostid":1000,"Nsid":1000,"Maprange":1},{"Isuid":true,"Isgid":true,"Hostid":1001,"Nsid":1001,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1001002,"Nsid":1002,"Maprange":654358},{"Isuid":true,"Isgid":true,"Hostid":655360,"Nsid":655360,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1655361,"Nsid":655361,"Maprange":9996},{"Isuid":true,"Isgid":true,"Hostid":665357,"Nsid":665357,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1665358,"Nsid":665358,"Maprange":999334642}]'
  volatile.idmap.next: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000},{"Isuid":true,"Isgid":true,"Hostid":1000,"Nsid":1000,"Maprange":1},{"Isuid":true,"Isgid":true,"Hostid":1001,"Nsid":1001,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1001002,"Nsid":1002,"Maprange":654358},{"Isuid":true,"Isgid":true,"Hostid":655360,"Nsid":655360,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1655361,"Nsid":655361,"Maprange":9996},{"Isuid":true,"Isgid":true,"Hostid":665357,"Nsid":665357,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1665358,"Nsid":665358,"Maprange":999334642},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000},{"Isuid":true,"Isgid":true,"Hostid":1000,"Nsid":1000,"Maprange":1},{"Isuid":true,"Isgid":true,"Hostid":1001,"Nsid":1001,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1001002,"Nsid":1002,"Maprange":654358},{"Isuid":true,"Isgid":true,"Hostid":655360,"Nsid":655360,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1655361,"Nsid":655361,"Maprange":9996},{"Isuid":true,"Isgid":true,"Hostid":665357,"Nsid":665357,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1665358,"Nsid":665358,"Maprange":999334642}]'
  volatile.last_state.idmap: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000},{"Isuid":true,"Isgid":true,"Hostid":1000,"Nsid":1000,"Maprange":1},{"Isuid":true,"Isgid":true,"Hostid":1001,"Nsid":1001,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1001002,"Nsid":1002,"Maprange":654358},{"Isuid":true,"Isgid":true,"Hostid":655360,"Nsid":655360,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1655361,"Nsid":655361,"Maprange":9996},{"Isuid":true,"Isgid":true,"Hostid":665357,"Nsid":665357,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":1665358,"Nsid":665358,"Maprange":999334642},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000},{"Isuid":true,"Isgid":true,"Hostid":1000,"Nsid":1000,"Maprange":1},{"Isuid":true,"Isgid":true,"Hostid":1001,"Nsid":1001,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1001002,"Nsid":1002,"Maprange":654358},{"Isuid":true,"Isgid":true,"Hostid":655360,"Nsid":655360,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1655361,"Nsid":655361,"Maprange":9996},{"Isuid":true,"Isgid":true,"Hostid":665357,"Nsid":665357,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":1665358,"Nsid":665358,"Maprange":999334642}]'
  volatile.last_state.power: RUNNING
devices:
  /dev/dri/card0:
    major: "226"
    minor: "0"
    mode: "0666"
    path: /dev/dri/card0
    type: unix-char
  /dev/dri/renderD128:
    major: "226"
    minor: "128"
    mode: "0666"
    path: /dev/dri/renderD128
    type: unix-char
  /dev/kvm:
    major: "10"
    minor: "232"
    mode: "0666"
    path: /dev/kvm
    type: unix-char
  /dev/snd/controlC0:
    major: "116"
    minor: "0"
    mode: "0666"
    path: /dev/snd/controlC0
    type: unix-char
  /dev/snd/pcmC0D0c:
    major: "116"
    minor: "24"
    mode: "0666"
    path: /dev/snd/pcmC0D0c
    type: unix-char
  /dev/snd/pcmC0D0p:
    major: "116"
    minor: "16"
    mode: "0666"
    path: /dev/snd/pcmC0D0p
    type: unix-char
  /dev/snd/pcmC0D1c:
    major: "116"
    minor: "25"
    mode: "0666"
    path: /dev/snd/pcmC0D1c
    type: unix-char
  /dev/snd/seq:
    major: "116"
    minor: "1"
    mode: "0666"
    path: /dev/snd/seq
    type: unix-char
  /dev/snd/timer:
    major: "116"
    minor: "33"
    mode: "0666"
    path: /dev/snd/timer
    type: unix-char
  container_token:
    path: /dev/.container_token
    required: "false"
    source: /run/tokens/penguin_token
    type: disk
  cros_containers:
    path: /opt/google/cros-containers
    source: /opt/google/cros-containers
    type: disk
  cros_milestone:
    path: /dev/.cros_milestone
    source: /run/cros_milestone
    type: disk
  eth0:
    nictype: bridged
    parent: lxdbr0
    type: nic
  external:
    path: /mnt/external
    source: /mnt/external
    type: disk
  fuse:
    mode: "0666"
    source: /dev/fuse
    type: unix-char
  host-ip:
    path: /dev/.host_ip
    source: /run/host_ip
    type: disk
  root:
    path: /
    pool: default
    type: disk
  shared:
    path: /mnt/chromeos
    source: /mnt/shared
    type: disk
  ssh_authorized_keys:
    path: /dev/.ssh/ssh_authorized_keys
    required: "false"
    source: /run/sshd/penguin/authorized_keys
    type: disk
  ssh_host_key:
    path: /dev/.ssh/ssh_host_key
    required: "false"
    source: /run/sshd/penguin/ssh_host_key
    type: disk
  sshd_config:
    path: /dev/.ssh/sshd_config
    source: /usr/share/container_sshd_config
    type: disk
  tun:
    mode: "0666"
    source: /dev/net/tun
    type: unix-char
  usb:
    mode: "0666"
    type: usb
  wl0:
    mode: "0666"
    source: /dev/wl0
    type: unix-char
ephemeral: false
profiles:
- default
stateful: false
description: ""
giuseppe commented 3 years ago

thanks, I wonder if cgroups are usable at all.

When you run a container with runc and you specify some limits (e.g. --memory 100M) does it work correctly?

Could you show me the output for these commands in the LXC container:

ntkme commented 3 years ago

With sudo, runc works with --memory=100M but shows a warning about cgroup:

sudo podman run --rm -it --runtime=runc --memory=100M docker.io/library/alpine
Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap.
/ #

Without sudo, runc fails with --memory=100M:

$ podman run --rm -it --runtime=runc --memory=100M docker.io/library/alpine
Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap.
Error: OCI runtime error: container_linux.go:367: starting container process caused: process_linux.go:495: container init caused: process_linux.go:458: setting cgroup config for procHooks process caused: cannot set memory limit: container could not join or create cgroup
$ cat /proc/self/mountinfo
119 64 0:44 /lxd/storage-pools/default/containers/penguin/rootfs / rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=267,subvol=/lxd/storage-pools/default/containers/penguin
120 119 0:55 / /dev rw,relatime - tmpfs none rw,size=492k,mode=755,uid=1000000,gid=1000000
121 119 0:54 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw
122 119 0:56 / /sys rw,relatime - sysfs sysfs rw
123 121 0:47 /proc/cpuinfo /proc/cpuinfo rw,nosuid,nodev,relatime - fuse.lxcfs lxcfs rw,user_id=0,group_id=0,allow_other
124 121 0:47 /proc/diskstats /proc/diskstats rw,nosuid,nodev,relatime - fuse.lxcfs lxcfs rw,user_id=0,group_id=0,allow_other
125 121 0:47 /proc/meminfo /proc/meminfo rw,nosuid,nodev,relatime - fuse.lxcfs lxcfs rw,user_id=0,group_id=0,allow_other
126 121 0:47 /proc/stat /proc/stat rw,nosuid,nodev,relatime - fuse.lxcfs lxcfs rw,user_id=0,group_id=0,allow_other
127 121 0:47 /proc/uptime /proc/uptime rw,nosuid,nodev,relatime - fuse.lxcfs lxcfs rw,user_id=0,group_id=0,allow_other
128 120 0:6 /full /dev/full rw,nosuid,noexec,relatime - devtmpfs devtmpfs rw,size=3378976k,nr_inodes=844744,mode=755
129 120 0:6 /null /dev/null rw,nosuid,noexec,relatime - devtmpfs devtmpfs rw,size=3378976k,nr_inodes=844744,mode=755
130 120 0:6 /random /dev/random rw,nosuid,noexec,relatime - devtmpfs devtmpfs rw,size=3378976k,nr_inodes=844744,mode=755
131 120 0:6 /tty /dev/tty rw,nosuid,noexec,relatime - devtmpfs devtmpfs rw,size=3378976k,nr_inodes=844744,mode=755
132 120 0:6 /urandom /dev/urandom rw,nosuid,noexec,relatime - devtmpfs devtmpfs rw,size=3378976k,nr_inodes=844744,mode=755
133 120 0:6 /zero /dev/zero rw,nosuid,noexec,relatime - devtmpfs devtmpfs rw,size=3378976k,nr_inodes=844744,mode=755
134 120 0:6 /fuse /dev/fuse rw,nosuid,noexec,relatime - devtmpfs devtmpfs rw,size=3378976k,nr_inodes=844744,mode=755
135 120 0:6 /net/tun /dev/net/tun rw,nosuid,noexec,relatime - devtmpfs devtmpfs rw,size=3378976k,nr_inodes=844744,mode=755
136 121 0:5 /sys/fs/binfmt_misc /proc/sys/fs/binfmt_misc rw,nosuid,nodev,noexec,relatime - proc proc rw
137 122 0:24 /fs/fuse/connections /sys/fs/fuse/connections rw,nosuid,nodev,noexec,relatime - sysfs sys rw
138 122 0:24 /kernel/debug /sys/kernel/debug rw,nosuid,nodev,noexec,relatime - sysfs sys rw
139 122 0:24 /kernel/security /sys/kernel/security rw,nosuid,nodev,noexec,relatime - sysfs sys rw
140 120 0:54 / /dev/.lxc/proc rw,relatime - proc proc rw
141 120 0:56 / /dev/.lxc/sys rw,relatime - sysfs sys rw
142 120 0:52 / /dev/lxd rw,relatime - tmpfs tmpfs rw,size=100k,mode=755
143 120 0:51 /penguin /dev/.lxd-mounts rw,relatime master:1 - tmpfs tmpfs rw,size=100k,mode=711
144 120 0:27 /tokens/penguin_token /dev/.container_token rw,nosuid,nodev,noexec,relatime - tmpfs run rw,mode=755
145 120 0:27 /cros_milestone /dev/.cros_milestone rw,nosuid,nodev,noexec,relatime - tmpfs run rw,mode=755
146 120 0:27 /host_ip /dev/.host_ip rw,nosuid,nodev,noexec,relatime - tmpfs run rw,mode=755
147 120 0:27 /sshd/penguin/authorized_keys /dev/.ssh/ssh_authorized_keys rw,nosuid,nodev,noexec,relatime - tmpfs run rw,mode=755
148 120 0:27 /sshd/penguin/ssh_host_key /dev/.ssh/ssh_host_key rw,nosuid,nodev,noexec,relatime - tmpfs run rw,mode=755
149 120 259:0 /usr/share/container_sshd_config /dev/.ssh/sshd_config ro,relatime - ext4 /dev/root ro
150 119 0:43 / /mnt/chromeos rw,nosuid,nodev,relatime - 9p 9p rw,sync,dirsync,access=any,trans=fd,rfd=26,wfd=26
151 119 0:26 / /mnt/external rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755
152 119 254:0 / /opt/google/cros-containers ro,relatime - ext4 /dev/vda ro
153 120 0:44 /lxd/devices/penguin/unix.-dev-dri-card0.dev-dri-card0 /dev/dri/card0 rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
154 120 0:44 /lxd/devices/penguin/unix.-dev-dri-renderD128.dev-dri-renderD128 /dev/dri/renderD128 rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
155 120 0:44 /lxd/devices/penguin/unix.-dev-kvm.dev-kvm /dev/kvm rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
156 120 0:44 /lxd/devices/penguin/unix.-dev-snd-controlC0.dev-snd-controlC0 /dev/snd/controlC0 rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
157 120 0:44 /lxd/devices/penguin/unix.-dev-snd-pcmC0D0c.dev-snd-pcmC0D0c /dev/snd/pcmC0D0c rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
158 120 0:44 /lxd/devices/penguin/unix.-dev-snd-pcmC0D0p.dev-snd-pcmC0D0p /dev/snd/pcmC0D0p rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
159 120 0:44 /lxd/devices/penguin/unix.-dev-snd-pcmC0D1c.dev-snd-pcmC0D1c /dev/snd/pcmC0D1c rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
160 120 0:44 /lxd/devices/penguin/unix.-dev-snd-seq.dev-snd-seq /dev/snd/seq rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
161 120 0:44 /lxd/devices/penguin/unix.-dev-snd-timer.dev-snd-timer /dev/snd/timer rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
162 134 0:44 /lxd/devices/penguin/unix.fuse.dev-fuse /dev/fuse rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
163 135 0:44 /lxd/devices/penguin/unix.tun.dev-net-tun /dev/net/tun rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
164 120 0:44 /lxd/devices/penguin/unix.wl0.dev-wl0 /dev/wl0 rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
165 120 0:44 /lxd/devices/penguin/unix.usb.dev-bus-usb-001-001 /dev/bus/usb/001/001 rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
166 120 0:44 /lxd/devices/penguin/unix.usb.dev-bus-usb-002-001 /dev/bus/usb/002/001 rw,relatime - btrfs /dev/vdb rw,discard,space_cache,user_subvol_rm_allowed,subvolid=5,subvol=/
167 120 0:29 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666
65 120 0:57 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=1000005,mode=620,ptmxmode=666,max=1024
66 120 0:57 /ptmx /dev/ptmx rw,nosuid,noexec,relatime - devpts devpts rw,gid=1000005,mode=620,ptmxmode=666,max=1024
55 120 0:58 / /dev/shm rw,nosuid,nodev - tmpfs tmpfs rw,uid=1000000,gid=1000000
56 119 0:59 / /run rw,nosuid,nodev - tmpfs tmpfs rw,size=1352588k,nr_inodes=819200,mode=755,uid=1000000,gid=1000000
57 56 0:60 / /run/lock rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,size=5120k,uid=1000000,gid=1000000
58 122 0:61 / /sys/fs/cgroup ro,nosuid,nodev,noexec - tmpfs tmpfs ro,size=4096k,nr_inodes=1024,mode=755,uid=1000000,gid=1000000
59 58 0:42 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,name=systemd
60 58 0:35 / /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,devices
61 58 0:39 / /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,net_cls,net_prio
62 58 0:36 / /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,freezer
63 58 0:40 / /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,perf_event
67 58 0:33 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpu,cpuacct
68 58 0:41 / /sys/fs/cgroup/pids rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,pids
69 58 0:32 / /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,blkio
71 58 0:37 / /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,hugetlb
72 58 0:38 / /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,memory
73 58 0:34 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpuset
74 120 0:53 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw
278 56 0:62 / /run/user/1000 rw,nosuid,nodev,relatime - tmpfs tmpfs rw,size=676292k,nr_inodes=169073,mode=700,uid=1000,gid=1000
428 56 0:59 /netns /run/netns rw,nosuid,nodev shared:150 - tmpfs tmpfs rw,size=1352588k,nr_inodes=819200,mode=755,uid=1000000,gid=1000000
$ ls -l /sys/fs/cgroup
total 0
drwxrwxr-x 6 nobody root  0 May 24 01:07 blkio
lrwxrwxrwx 1 root   root 11 May 22 14:57 cpu -> cpu,cpuacct
lrwxrwxrwx 1 root   root 11 May 22 14:57 cpuacct -> cpu,cpuacct
drwxrwxr-x 6 nobody root  0 May 24 01:07 cpu,cpuacct
drwxrwxr-x 3 nobody root  0 May 24 01:07 cpuset
drwxr-xr-x 2 root   root 40 May 22 14:57 debug
drwxrwxr-x 6 nobody root  0 May 24 01:07 devices
drwxrwxr-x 3 nobody root  0 May 24 01:07 freezer
drwxrwxr-x 3 nobody root  0 May 24 01:07 hugetlb
drwxrwxr-x 6 nobody root  0 May 24 01:07 memory
lrwxrwxrwx 1 root   root 16 May 22 14:57 net_cls -> net_cls,net_prio
drwxrwxr-x 3 nobody root  0 May 24 01:07 net_cls,net_prio
lrwxrwxrwx 1 root   root 16 May 22 14:57 net_prio -> net_cls,net_prio
drwxrwxr-x 3 nobody root  0 May 24 01:07 perf_event
drwxrwxr-x 6 nobody root  0 May 24 01:07 pids
drwxrwxr-x 7 nobody root  0 May 24 01:07 systemd
$ cat /proc/cgroups
#subsys_name    hierarchy       num_cgroups     enabled
cpuset  3       5       1
cpu     2       10      1
cpuacct 2       10      1
blkio   1       10      1
memory  7       32      1
devices 4       22      1
freezer 5       5       1
net_cls 8       5       1
perf_event      9       5       1
net_prio        8       5       1
hugetlb 6       5       1
pids    10      25      1
debug   0       1       1
giuseppe commented 3 years ago

thanks!

Can you also show me cat /proc/self/cgroup?

ntkme commented 3 years ago 
$ cat /proc/self/cgroup
11:name=systemd:/user.slice/user-1000.slice/user@1000.service/app.slice/cros-garcon.service
10:pids:/user.slice/user-1000.slice/user@1000.service
9:perf_event:/
8:net_cls,net_prio:/
7:memory:/user.slice/user-1000.slice/user@1000.service
6:hugetlb:/
5:freezer:/
4:devices:/user.slice
3:cpuset:/
2:cpu,cpuacct:/user.slice
1:blkio:/user.slice
0::/
giuseppe commented 3 years ago

are you able to try a patch? Does the following patch solve the issue you are seeing?

diff --git a/src/libcrun/cgroup.c b/src/libcrun/cgroup.c
index b30cd27..24279a2 100644
--- a/src/libcrun/cgroup.c
+++ b/src/libcrun/cgroup.c
@@ -1014,6 +1014,9 @@ systemd_finalize (struct libcrun_cgroup_args *args, libcrun_error_t *err)
           subpath = strchr (subsystem, ':') + 1;
           *(subpath - 1) = '\0';

+          if (subsystem[0] == '\0')
+            continue;
+
           if (strcmp (subpath, *path))
             {
               ret = enter_cgroup_subsystem (pid, subsystem, *path, true, err);
ntkme commented 3 years ago

I think we're one step forward, with the patch now --cgroup-manager systemd fails the same way as --cgroup-manager cgroupfs:

$ sudo podman run --rm -it --privileged --runtime=/usr/local/bin/crun docker.io/library/alpine
Error: writing file `devices.allow`: Operation not permitted: OCI permission denied
giuseppe commented 3 years ago

can you show me the output for cat /proc/self/uid_map and cat /proc/self/gid_map ?

giuseppe commented 3 years ago

if the output is different than 0 0 4294967295 please try with this patch:

diff --git a/src/libcrun/cgroup.c b/src/libcrun/cgroup.c
index b30cd27..c3cc6dc 100644
--- a/src/libcrun/cgroup.c
+++ b/src/libcrun/cgroup.c
@@ -1014,6 +1014,9 @@ systemd_finalize (struct libcrun_cgroup_args *args, libcrun_error_t *err)
           subpath = strchr (subsystem, ':') + 1;
           *(subpath - 1) = '\0';

+          if (subsystem[0] == '\0')
+            continue;
+
           if (strcmp (subpath, *path))
             {
               ret = enter_cgroup_subsystem (pid, subsystem, *path, true, err);
@@ -2729,10 +2732,31 @@ static int
 write_devices_resources (int dirfd, bool cgroup2, runtime_spec_schema_defs_linux_device_cgroup **devs, size_t devs_len,
                          libcrun_error_t *err)
 {
+  int ret;
+
   if (cgroup2)
-    return write_devices_resources_v2 (dirfd, devs, devs_len, err);
+    ret = write_devices_resources_v2 (dirfd, devs, devs_len, err);
+  else
+    ret = write_devices_resources_v1 (dirfd, devs, devs_len, err);
+  if (UNLIKELY (ret < 0))
+    {
+      libcrun_error_t tmp_err = NULL;
+      int rootless;
+
+      rootless = is_rootless (&tmp_err);
+      if (UNLIKELY (rootless < 0))
+        {
+          crun_error_release (&tmp_err);
+          return ret;
+        }

-  return write_devices_resources_v1 (dirfd, devs, devs_len, err);
+      if (rootless)
+        {
+          crun_error_release (err);
+          ret = 0;
+        }
+    }
+  return ret;
 }

 /* use for cgroupv2 files with .min, .max, .low, or .high suffix */
ntkme commented 3 years ago

@giuseppe The patch worked!

The uid and gid map both looked like below:

         0    1000000       1000
      1000       1000          1
      1001       1001          1
      1002    1001002     654358
    655360     655360          1
    655361    1655361       9996
    665357     665357          1
    665358    1665358  999334642
giuseppe commented 3 years ago

thanks for confirming it!

Opened a PR