search

weaveworks / ignite

Ignite a Firecracker microVM
https://ignite.readthedocs.org
Apache License 2.0
3.49k stars 222 forks source link

FATA[0000] unknown runtime "" error on ignite stop/kill/attach #991

Closed limegreendender2 closed 11 months ago

limegreendender2 commented 11 months ago

I followed the install instructions and the last good command was

root@firecracker:/etc/firecracker/manifests# ignite ps
VM ID                                   IMAGE                           KERNEL                                  SIZE    CPUS    MEMORY   CREATED  STATUS          IPS     PORTS   NAME
0fd7d383-e547-4c59-918d-e238100f6ff6    weaveworks/ignite-ubuntu:latest weaveworks/ignite-kernel:5.10.51        3.0 GB  2       800.0 MB 3d ago   Up <nil>                        smoke-test

Unfortunately the attach and subsequent commands fail.

root@firecracker:/etc/firecracker/manifests#  ignite attach smoke-test
FATA[0000] unknown runtime ""

Ignite version

root@firecracker:/etc/firecracker/manifests# ignite version
Ignite version: version.Info{Major:"0", Minor:"10", GitVersion:"v0.10.0", GitCommit:"4540abeb9ba6daba32a72ef2b799095c71ebacb0", GitTreeState:"clean", BuildDate:"2021-07-19T20:52:59Z", GoVersion:"go1.16.3", Compiler:"gc", Platform:"linux/amd64", SandboxImage:version.Image{Name:"weaveworks/ignite", Tag:"v0.10.0", Delimeter:":"}, KernelImage:version.Image{Name:"weaveworks/ignite-kernel", Tag:"5.10.51", Delimeter:":"}}
Firecracker version: v0.22.4

root@firecracker:~# lscpu | grep Virtualization
Virtualization:                  AMD-V

root@firecracker:~# lsmod | grep kvm
kvm_amd               151552  12
kvm                  1019904  1 kvm_amd
ccp                    98304  1 kvm_amd
irqbypass              16384  12 vfio_pci_core,kvm

root@firecracker:~# containerd --version
containerd containerd.io 1.6.18 2456e983eb9e37e47538f59ea18f2043c9a73640

I've tried this on two environments - both with the same results WSL2 with KVM enabled kernel and Debian Bullseye with kernel 5.15.74-1-pve running under proxmox.

strace prior to error message

[pid  3630] newfstatat(AT_FDCWD, "/var/lib/firecracker/vm/0fd7d383-e547-4c59-918d-e238100f6ff6/metadata.json", {st_mode=S_IFREG|0644, st_size=917, ...}, 0) = 0
[pid  3630] openat(AT_FDCWD, "/var/lib/firecracker/vm/0fd7d383-e547-4c59-918d-e238100f6ff6/metadata.json", O_RDONLY|O_CLOEXEC) = 7
[pid  3630] epoll_ctl(4, EPOLL_CTL_ADD, 7, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=465838016, u64=140145248706496}}) = -1 EPERM (Operation not permitted)
[pid  3630] epoll_ctl(4, EPOLL_CTL_DEL, 7, 0xc0001df33c) = -1 EPERM (Operation not permitted)
[pid  3630] fstat(7, {st_mode=S_IFREG|0644, st_size=917, ...}) = 0
[pid  3630] read(7, "{\n  \"kind\": \"VM\",\n  \"apiVersion\""..., 918) = 917
[pid  3630] read(7, "", 1)              = 0
[pid  3630] close(7)                    = 0
[pid  3630] openat(AT_FDCWD, "/var/lib/firecracker/vm/0fd7d383-e547-4c59-918d-e238100f6ff6/metadata.json", O_RDONLY|O_CLOEXEC) = 7
[pid  3630] epoll_ctl(4, EPOLL_CTL_ADD, 7, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=465838016, u64=140145248706496}}) = -1 EPERM (Operation not permitted)
[pid  3630] epoll_ctl(4, EPOLL_CTL_DEL, 7, 0xc0001df61c) = -1 EPERM (Operation not permitted)
[pid  3630] fstat(7, {st_mode=S_IFREG|0644, st_size=917, ...}) = 0
[pid  3630] read(7, "{\n  \"kind\": \"VM\",\n  \"apiVersion\""..., 918) = 917
[pid  3630] read(7, "", 1)              = 0
[pid  3630] close(7)                    = 0
[pid  3630] mmap(NULL, 262144, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f761b8c3000
[pid  3630] newfstatat(AT_FDCWD, "/var/lib/firecracker/vm/0fd7d383-e547-4c59-918d-e238100f6ff6/metadata.json", {st_mode=S_IFREG|0644, st_size=917, ...}, 0) = 0
[pid  3630] newfstatat(AT_FDCWD, "/var/lib/firecracker/vm/0fd7d383-e547-4c59-918d-e238100f6ff6/metadata.json", {st_mode=S_IFREG|0644, st_size=917, ...}, 0) = 0
[pid  3630] ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
[pid  3630] write(1, "\33[31mFATA\33[0m[0000] unknown runt"..., 66FATA[0000] unknown runtime ""
woodliu commented 11 months ago

@limegreendender2 Hi, could you post the file content of /etc/firecracker/manifests/smoke-test.yml on the machine? And show the command result: ctr --namespace firecracker container ls ctr --namespace firecracker task ls You can use ctr --namespace firecracker task attach ignite-xxxx attach the vm as well.

There are some issues of ignite, this issue may cause by this one

limegreendender2 commented 11 months ago 
apiVersion: ignite.weave.works/v1alpha4
kind: VM
metadata:
  created: "2023-08-04T13:43:10Z"
  name: smoke-test
  uid: 0fd7d383-e547-4c59-918d-e238100f6ff6
spec:
  cpus: 2
  diskSize: 3GB
  image:
    oci: weaveworks/ignite-ubuntu:latest
  kernel:
    cmdLine: console=ttyS0 reboot=k panic=1 pci=off ip=dhcp
    oci: weaveworks/ignite-kernel:5.10.51
  memory: 800MB
  network: {}
  sandbox:
    oci: weaveworks/ignite:v0.10.0
  storage: {}
status:
  idPrefix: ""
  image:
    id: null
    size: 0B
  kernel:
    id: null
    size: 0B
  network:
    ipAddresses: null
    plugin: ""
  running: true
  runtime:
    id: ""
    name: ""

root@firecracker:~# ctr --namespace firecracker container ls
CONTAINER    IMAGE    RUNTIME  

root@firecracker:~# ctr --namespace firecracker task ls
TASK    PID    STATUS  

root@firecracker:~# ignite ps
VM ID                                   IMAGE                           KERNEL                                  SIZE    CPUS    MEMORY          CREATED         STATUS          IPS     PORTS   NAME
0fd7d383-e547-4c59-918d-e238100f6ff6    weaveworks/ignite-ubuntu:latest weaveworks/ignite-kernel:5.10.51        3.0 GB  2       800.0 MB        3d18h ago       Up <nil>                        smoke-test

root@firecracker:~# ps -aef
UID          PID    PPID  C STIME TTY          TIME CMD
root           1       0  0 Aug04 ?        00:00:01 /sbin/init
root          51       1  0 Aug04 ?        00:00:00 /lib/systemd/systemd-journald
systemd+      65       1  0 Aug04 ?        00:00:00 /lib/systemd/systemd-networkd
root          90       1  0 Aug04 ?        00:00:00 /sbin/dhclient -4 -v -i -pf /run/dhclient.eth0.pid -lf /var/lib/dhcp/dhclient.eth0.leases -I -df /var/lib/dhcp/dhclient6.eth0.leases eth0
root          91       1  0 Aug04 ?        00:00:00 /usr/sbin/cron -f
message+      92       1  0 Aug04 ?        00:00:00 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
root          93       1  0 Aug04 ?        00:00:00 /usr/sbin/rsyslogd -n -iNONE
root          94       1  0 Aug04 ?        00:00:00 /lib/systemd/systemd-logind
systemd+     102       1  0 Aug04 ?        00:00:00 /lib/systemd/systemd-resolved
root         147       1  0 Aug04 ?        00:04:46 /usr/bin/containerd
root         151       1  0 Aug04 pts/0    00:00:00 /sbin/agetty -o -p -- \u --noclear --keep-baud console 115200,38400,9600 linux
root         152       1  0 Aug04 pts/1    00:00:00 /bin/login -p --
root         153       1  0 Aug04 pts/2    00:00:00 /sbin/agetty -o -p -- \u --noclear --keep-baud tty2 115200,38400,9600 linux
root         315       1  0 Aug04 ?        00:00:00 /usr/lib/postfix/sbin/master -w
postfix      317     315  0 Aug04 ?        00:00:00 qmgr -l -t unix -u
root         445     152  0 Aug04 pts/1    00:00:00 -bash
root        1603     445  0 Aug04 pts/1    00:00:11 ignited daemon --log-level debug
postfix     3988     315  0 07:42 ?        00:00:00 pickup -l -t unix -u -c
root        4021     445  0 07:48 pts/1    00:00:00 ps -aef
root@firecracker:~# ^C
woodliu commented 11 months ago

@limegreendender2 Hi, there actually some bugs with ignite rm, i will fix that later, for now, you can clean the env and restart the vm:

  1. use pe -ef|grep ignited to find the PID of ignited, and use kill command to stop it
  2. use ignite vm stop ignite-xxx and ignite vm rm ignite-xxx the clean the ignite vm
  3. cd /etc/firecracker/manifests/ and rm smoke-test.yml
  4. runignited daemon --log-level debug & to restart the ignited daemon
  5. recreate vm manifest file, and then 
    VMFILE=/etc/firecracker/manifests/smoke-test.yml
tee "$VMFILE" > /dev/null <<EOF
apiVersion: ignite.weave.works/v1alpha4
kind: VM
metadata:
name: smoke-test
uid: $(uuidgen)
spec:
image:
oci: weaveworks/ignite-ubuntu
cpus: 2
diskSize: 3GB
memory: 800MB
ssh:
generate: true
status:
running: true
EOF

And then run ctr --namespace firecracker container ls and ctr --namespace firecracker task ls to check if the contianerd contaier been created.

limegreendender2 commented 11 months ago 
root@firecracker:/etc/firecracker/manifests#  ignite vm stop ignite-xxx a
FATA[0000] can't find VM: no ID/name matches for "ignite-xxx" 
root@firecracker:/etc/firecracker/manifests#  ignite vm stop smoke-test  
FATA[0000] unknown runtime ""                           
root@firecracker:/etc/firecracker/manifests#  ignite vm stop 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] unknown runtime ""
woodliu commented 11 months ago 
root@firecracker:/etc/firecracker/manifests#  ignite vm stop ignite-xxx a
FATA[0000] can't find VM: no ID/name matches for "ignite-xxx" 
root@firecracker:/etc/firecracker/manifests#  ignite vm stop smoke-test  
FATA[0000] unknown runtime ""                           
root@firecracker:/etc/firecracker/manifests#  ignite vm stop 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] unknown runtime ""

please exec ignite vm rm 0fd7d383-e547-4c59-918d-e238100f6ff6

limegreendender2 commented 11 months ago 
root@firecracker:/etc/firecracker/manifests# ignite vm rm 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] 0fd7d383-e547-4c59-918d-e238100f6ff6 is running 
root@firecracker:/etc/firecracker/manifests# ignite vm stop 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] unknown runtime ""                           
root@firecracker:/etc/firecracker/manifests# ignite vm kill 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] unknown runtime ""                           
root@firecracker:/etc/firecracker/manifests# ignite vm rm 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] 0fd7d383-e547-4c59-918d-e238100f6ff6 is running
woodliu commented 11 months ago 
root@firecracker:/etc/firecracker/manifests# ignite vm rm 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] 0fd7d383-e547-4c59-918d-e238100f6ff6 is running 
root@firecracker:/etc/firecracker/manifests# ignite vm stop 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] unknown runtime ""                           
root@firecracker:/etc/firecracker/manifests# ignite vm kill 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] unknown runtime ""                           
root@firecracker:/etc/firecracker/manifests# ignite vm rm 0fd7d383-e547-4c59-918d-e238100f6ff6
FATA[0000] 0fd7d383-e547-4c59-918d-e238100f6ff6 is running

ok... it seems stuck there because the missing runtime. You can manually remove the vm by run rm -rf /var/lib/firecracker/vm/* all vm metadata are stored there. And there are devicemapper need to remove: Run dmsetup status like:

# dmsetup status
ignite-981c73c5-ddaa-4bfd-b1e7-6645e91e2f75: 0 6291456 snapshot 149672/6291456 592
ignite-981c73c5-ddaa-4bfd-b1e7-6645e91e2f75-base: 0 577536 linear 
ignite-981c73c5-ddaa-4bfd-b1e7-6645e91e2f75-base: 577536 6291456 zero

and remove the dm prefixed with ignite-,like:

# dmsetup remove ignite-981c73c5-ddaa-4bfd-b1e7-6645e91e2f75
# dmsetup remove ignite-981c73c5-ddaa-4bfd-b1e7-6645e91e2f75-base
limegreendender2 commented 11 months ago 
root@firecracker:/etc/firecracker/manifests# ignited daemon --log-level debug &
[1] 4192
root@firecracker:/etc/firecracker/manifests# DEBU[0000] Using ignite default configurations          
DEBU[0000] FileWatcher: Monitoring thread started       
DEBU[0000] FileWatcher: Dispatch thread started         
DEBU[0000] GenericWatchStorage: Monitoring thread started 
DEBU[0000] SyncStorage: Monitoring thread started       
INFO[0000] Starting reconciliation loop...              

root@firecracker:/etc/firecracker/manifests# export VMFILE=/etc/firecracker/manifests/smoke-test.yml
tee "$VMFILE" > /dev/null <<EOF
apiVersion: ignite.weave.works/v1alpha4
kind: VM
metadata:
  name: smoke-test
  uid: $(uuidgen)
spec:
  image:
    oci: weaveworks/ignite-ubuntu
  cpus: 2
  diskSize: 3GB
  memory: 800MB
  ssh:
    generate: true
status:
  running: true
EOF
DEBU[0101] FileWatcher: Registered inotify events [notify.InCloseWrite: "/etc/firecracker/manifests/smoke-test.yml"] for path "/etc/firecracker/manifests/smoke-test.yml" 
root@firecracker:/etc/firecracker/manifests# DEBU[0102] FileWatcher: Sending update: MODIFY -> "/etc/firecracker/manifests/smoke-test.yml" 
DEBU[0102] FileWatcher: Dispatched events batch and reset the events cache 
DEBU[0102] GenericMappedRawStorage: AddMapping: "vm/faf845f9-8e49-4ce2-af94-bc7fa4500253" -> "/etc/firecracker/manifests/smoke-test.yml" 
DEBU[0102] SyncStorage: Received update {{CREATE &TypeMeta{Kind:VM,APIVersion:ignite.weave.works/v1alpha4,}} 0xc000478d80} true 
DEBU[0102] SyncStorage: Sent update: {CREATE &TypeMeta{Kind:VM,APIVersion:ignite.weave.works/v1alpha4,}} 
DEBU[0102] FileWatcher: Skipping suspended event MODIFY for path: "/etc/firecracker/manifests/smoke-test.yml" 
INFO[0102] Creating VM "faf845f9-8e49-4ce2-af94-bc7fa4500253" with name "smoke-test"... 
DEBU[0102] Ensuring image weaveworks/ignite-ubuntu:latest exists, or importing it... 
DEBU[0102] Importing image with ociRef "weaveworks/ignite-ubuntu:latest" 
DEBU[0102] containerd: Inspecting image "weaveworks/ignite-ubuntu:latest" 
INFO[0102] Starting image import...                     
DEBU[0102] Allocating image file and formatting it with ext4... 
DEBU[0102] Copying in files to the image file from a source... 
ERRO[0102] An error occurred when processing a VM update: failed to mount image "/var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4": command ["mount" "-o" "loop" "/var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4" "/tmp/028009075"] exited with "mount: /tmp/028009075: mount failed: Operation not permitted.\n": exit status 1 

root@firecracker:/etc/firecracker/manifests# ctr --namespace firecracker container ls
CONTAINER    IMAGE    RUNTIME    
root@firecracker:/etc/firecracker/manifests# ctr --namespace firecracker task
NAME:

root@firecracker:/etc/firecracker/manifests# ctr --namespace firecracker task ls
TASK    PID    STATUS 

root@firecracker:/etc/firecracker/manifests# cat smoke-test.yml 
apiVersion: ignite.weave.works/v1alpha4
kind: VM
metadata:
  created: "2023-08-08T10:25:49Z"
  name: smoke-test
  uid: faf845f9-8e49-4ce2-af94-bc7fa4500253
spec:
  cpus: 2
  diskSize: 3GB
  image:
    oci: weaveworks/ignite-ubuntu:latest
  kernel:
    cmdLine: console=ttyS0 reboot=k panic=1 pci=off ip=dhcp
    oci: weaveworks/ignite-kernel:5.10.51
  memory: 800MB
  network: {}
  sandbox:
    oci: weaveworks/ignite:v0.10.0
  ssh: {}
  storage: {}
status:
  idPrefix: ""
  image:
    id: null
    size: 0B
  kernel:
    id: null
    size: 0B
  network:
    ipAddresses: null
    plugin: ""
  running: true
  runtime:
    id: ""
    name: ""

    root@firecracker:/etc/firecracker/manifests# ignite ps
VM ID                                   IMAGE                           KERNEL                                  SIZE    CPUS    MEMORY          CREATED         STATUS          IPS     PORTS   NAME
faf845f9-8e49-4ce2-af94-bc7fa4500253    weaveworks/ignite-ubuntu:latest weaveworks/ignite-kernel:5.10.51        3.0 GB  2       800.0 MB        4m6s ago        Up <nil>                        smoke-test
woodliu commented 11 months ago

So, here is the root cause why not creat a vm succesfully, it is not permitted to execute the mount command. Please check the existed ofimage.ext4 and the mount point/tmp/028009075, you can manually run it and check why not permitted: mount -o loop /var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4 /tmp/028009075

limegreendender2 commented 11 months ago

Oops should have looked at the ignite log first The tmp file wasn't there - but same error after creating

It must be some perms on my proxmox VM. I will investigate later. Thanks for your time looking at this.

root@firecracker:~# ls -la /var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4 /tmp/028009075
ls: cannot access '/tmp/028009075': No such file or directory
-rw-r--r-- 1 root root 782585856 Aug  8 10:25 /var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4
root@firecracker:~# mkdir /tmp/028009075
root@firecracker:~# mount -o loop /var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4 /tmp/028009075
mount: /tmp/028009075: mount failed: Operation not permitted.
woodliu commented 11 months ago

Oops should have looked at the ignite log first The tmp file wasn't there - but same error after creating

It must be some perms on my proxmox VM. I will investigate later. Thanks for your time looking at this.

root@firecracker:~# ls -la /var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4 /tmp/028009075
ls: cannot access '/tmp/028009075': No such file or directory
-rw-r--r-- 1 root root 782585856 Aug  8 10:25 /var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4
root@firecracker:~# mkdir /tmp/028009075
root@firecracker:~# mount -o loop /var/lib/firecracker/image/44c83e4c8e17c8dd/image.ext4 /tmp/028009075
mount: /tmp/028009075: mount failed: Operation not permitted.

welcome, i will fix the issues here, because i am not the maintainer ,so i forked another project to fix them.

limegreendender2 commented 11 months ago

FYI I needed to set the Proxmox VM CPU Type to "Host" I can now attach/stop/rm Thanks