search

tkestack / galaxy

Providing high-performance network for Kubernetes
Other
109 stars 39 forks source link

galaxy-veth creates veth but not work in arm64 #98

Closed sstoner closed 3 years ago

sstoner commented 3 years ago

Expected Behavior

pod can communicate with the host it exists.

Current Behavior

pod can't communiate with the host it exists. 

# on the pod
[root@localhost data]# nsenter  -n -t 324701 ip nei
10.122.1.15 dev eth0 lladdr 26:b0:2e:9d:f5:bf STALE
169.254.1.1 dev eth0 lladdr 0e:ba:fd:2c:a4:4f PERMANENT
[root@localhost data]# nsenter  -n -t 324701 ip r
default via 169.254.1.1 dev eth0
169.254.1.1 dev eth0 scope link

# on the host
[root@localhost data]# ip r get 192.168.0.11
192.168.0.11 dev v-h6ea2d3176 src 10.122.1.15 uid 0
    cache

[root@localhost data]# ip a s v-h6ea2d3176
31: v-h6ea2d3176@if30: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
    link/ether 26:b0:2e:9d:f5:bf brd ff:ff:ff:ff:ff:ff link-netnsid 2
    inet6 fe80::24b0:2eff:fe9d:f5bf/64 scope link
       valid_lft forever preferred_lft foreve

the pod's default gateway and the gateway's mac are added by galaxy-veth, but the problem is the gateway's mac should be the veth mac of host netns(v-h6ea2d3176).

Possible Solution

  1. add delay(sleep) after https://github.com/tkestack/galaxy/blob/master/pkg/utils/utils.go#L243-L246
  2. TODO.

Steps to Reproduce

A:

  1. kernel: 4.19.90-17.ky10.aarch64, arch: arm64
  2. k8s 1.16.6, (tke install)
  3. network: galaxy-sdn + galaxy-veth (0.4.0)
  4. kubectl create pod

B: (iproute2 5.4.0)

  1. ip monitor (foregroud)

  2. ip l add veth-pba type veth peer name veth-pab output: 

     inet veth-pab forwarding on rp_filter strict mc_forwarding off proxy_neigh off ignore_routes_with_linkdown off
 inet6 veth-pab forwarding off mc_forwarding off proxy_neigh off ignore_routes_with_linkdown off
121: veth-pab@NONE: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default
  link/ether 62:37:44:12:b0:e5 brd ff:ff:ff:ff:ff:ff
 inet veth-pba forwarding on rp_filter strict mc_forwarding off proxy_neigh off ignore_routes_with_linkdown off
 inet6 veth-pba forwarding off mc_forwarding off proxy_neigh off ignore_routes_with_linkdown off
122: veth-pba@veth-pab: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN group default
   link/ether a2:ff:ed:39:22:4c brd ff:ff:ff:ff:ff:ff
121: veth-pab@veth-pba: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN group default
    link/ether 6a:2e:95:ff:9b:b0 brd ff:ff:ff:ff:ff:ff
122: veth-pba@veth-pab: <BROADCAST,MULTICAST,M-DOWN> mtu 1500 qdisc noop state DOWN group default
    link/ether e2:5c:00:33:cc:5e brd ff:ff:ff:ff:ff:ff

    mac address flapping in the creating process

Context (Environment)

strace -f -s 100 ip l add veth-pba type veth peer name veth-pab

execve("/usr/sbin/ip", ["ip", "l", "add", "veth-p130", "type", "veth", "peer", "name", "veth-p013"], 0xffffd45c7580 /* 30 vars */) = 0
brk(NULL)                               = 0xaaae33200000
faccessat(AT_FDCWD, "/etc/ld.so.preload", R_OK) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106368, ...}) = 0
mmap(NULL, 106368, PROT_READ, MAP_PRIVATE, 3, 0) = 0xfffc48570000
close(3)                                = 0
openat(AT_FDCWD, "/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0\260m\0\0\0\0\0\0@\0\0\0\0\0\0\0000\2\3\0\0\0\0\0\0\0\0\0@\08\0\10\0@\0\34\0\33\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20l\2\0\0\0\0\0\20l\2\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0(\366\2\0\0\0\0\0(\366\3\0\0\0\0\0(\366\3\0\0\0\0\08\n\0\0\0\0\0\0`/\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0\300\366\2\0\0\0\0\0\300\366\3\0\0\0\0\0\300\366\3\0\0\0\0\0000\2\0\0\0\0\0\0000\2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0\2\0\0\0\0\0\0\0\2\0\0\0\0\0\0\0\2\0\0\0\0\0\0$\0\0\0\0\0\0\0$\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0\7\0\0\0\4\0\0\0(\366\2\0\0\0\0\0(\366\3\0\0\0\0\0(\366\3\0\0\0\0\0(\0\0\0\0\0\0\0\341\0\0\0\0\0\0\0\10\0\0\0\0\0\0\0P\345td\4\0\0\0\350\t\2\0\0\0\0\0\350\t\2\0\0\0\0\0\350\t\2\0\0\0\0\0D\v\0\0\0\0\0\0D\v\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0(\366\2\0\0\0\0\0(\366\3\0\0\0\0\0(\366\3\0\0\0\0\0\330\t\0\0\0\0\0\0\330\t\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\352\365\353'\275\30\35\343\233\n\235\344c?\243\33\244J\313\313\0\0\0\0\305\0\0\0\222\0\0\0 \0\0\0\v\0\0\0\230 \4\201 L\0\3#\20\300\nT\210\357\322B\6h\10\24\26\0\6\20\4\4\0\0\311@\1P.\21\30\0&\th\30 \200*H\10\22\1\1,!\0\20\0\217\7\250\1@S\0\274\0\0\0R\20\236\4\1\5\24\33\21\33\214 \0\3C\10\1H\21\2\2\0\200j\17\304\1\2\210`\203\200\2*\200\202@\200\4\0\0C\30\4\0\20\200:\0!\246\10\5\0\206\0\4\0@\204\20\0@\n\0\200 $`0T\1@\304\0\20\0\240\24\20\234\10\200\0D\0\204\10@&\7\202\0\200\200\1\10\10\0\2\2\0\200\232\203\4\304\25\220(\2\200\0\10\0\201@\0\200\200\210@@\241\1\2\21\200@\24\200\4\221\313@ \10D\1\0\0\0\2F@\1\0A\200q\0@Eb\0\t\"\0\20@@Il\341\24\200<@!\0\0 \0\0\200\310\206\343@\200\23\0\3 @\2!\200T\2!\222\0\0\0\0\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=198960, ...}) = 0
mmap(NULL, 271752, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc48520000
mmap(0xfffc48550000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20000) = 0xfffc48550000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libelf.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0\0200\0\0\0\0\0\0@\0\0\0\0\0\0\0\340\1\2\0\0\0\0\0\0\0\0\0@\08\0\10\0@\0\34\0\33\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\240\202\1\0\0\0\0\0\240\202\1\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0\20\372\1\0\0\0\0\0\20\372\2\0\0\0\0\0\20\372\2\0\0\0\0\0\370\5\0\0\0\0\0\0\10\6\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0\320\373\1\0\0\0\0\0\320\373\2\0\0\0\0\0\320\373\2\0\0\0\0\0000\2\0\0\0\0\0\0000\2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0\2\0\0\0\0\0\0\0\2\0\0\0\0\0\0\0\2\0\0\0\0\0\0$\0\0\0\0\0\0\0$\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0\7\0\0\0\4\0\0\0\20\372\1\0\0\0\0\0\20\372\2\0\0\0\0\0\20\372\2\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0P\345td\4\0\0\0\20E\1\0\0\0\0\0\20E\1\0\0\0\0\0\20E\1\0\0\0\0\0\244\6\0\0\0\0\0\0\244\6\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0\20\372\1\0\0\0\0\0\20\372\2\0\0\0\0\0\20\372\2\0\0\0\0\0\360\5\0\0\0\0\0\0\360\5\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0:f[/\356\360\f\361C\327\257&\222D\361:\275\351\22M\0\0\0\0a\0\0\0003\0\0\0\20\0\0\0\n\0\0\0C\10\2\0\20\3\10\300\24\0\10\372\207\204\0\10\0\200\201\6\0\0\0\10\200\4\0\0000 \1\240`\20\264S\r\230\30\20\22\0`!\202\210D\n\200\240\300\316\300\201\22\3\10 E\1\310\300\2\2\2\v\21\200A\24\6\n\10\0\0\0\30\202\0 \0\202\200\200\261\1 \0(\1\1T\22H\3642B\20\22\241\10B\210A\0\200\0\200A\200\10\207\4\200\0A\200P\0\1A  \0\215UH\0243\0\0\0004\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0005\0\0\0007\0\0\08\0\0\0;\0\0\0\0\0\0\0>\0\0\0\0\0\0\0?\0\0\0\0\0\0\0\0\0\0\0A\0\0\0D\0\0\0F\0\0\0G\0\0\0\0\0\0\0H\0\0\0J\0\0\0L\0\0\0\0\0\0\0M\0\0\0O\0\0\0\0\0\0\0Q\0\0\0R\0\0\0T\0\0\0V\0\0\0W\0\0\0Y\0\0\0Z\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=133344, ...}) = 0
mmap(NULL, 196632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc484e0000
mmap(0xfffc48500000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x10000) = 0xfffc48500000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libmnl.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0\260\34\0\0\0\0\0\0@\0\0\0\0\0\0\0\330\1\1\0\0\0\0\0\0\0\0\0@\08\0\7\0@\0\33\0\32\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\204A\0\0\0\0\0\0\204A\0\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0\340\373\0\0\0\0\0\0\340\373\1\0\0\0\0\0\340\373\1\0\0\0\0\0(\4\0\0\0\0\0\0000\4\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0p\374\0\0\0\0\0\0p\374\1\0\0\0\0\0p\374\1\0\0\0\0\0 \2\0\0\0\0\0\0 \2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0$\0\0\0\0\0\0\0$\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0P\345td\4\0\0\0\0205\0\0\0\0\0\0\0205\0\0\0\0\0\0\0205\0\0\0\0\0\0t\2\0\0\0\0\0\0t\2\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0\340\373\0\0\0\0\0\0\340\373\1\0\0\0\0\0\340\373\1\0\0\0\0\0 \4\0\0\0\0\0\0 \4\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\205\257\t\351\211\274Mn\24P\261\324\327|q{\243\177\22\325\0\0\0\0C\0\0\0\33\0\0\0\10\0\0\0\t\0\0\0  fPH\360@@\20\200\16\2\0\300E\0001\220\245h\0F\300\20\30\0\200\0@\0B\30\220\20(@\220\0\1\201@\202@\f\30\0@\30\205\21P\0@b\20@\203\244G/\7*\315A\33\0\0\0\0\0\0\0\35\0\0\0\36\0\0\0 \0\0\0!\0\0\0\"\0\0\0\0\0\0\0&\0\0\0'\0\0\0)\0\0\0*\0\0\0\0\0\0\0\0\0\0\0,\0\0\0\0\0\0\0-\0\0\0/\0\0\0\0\0\0\0\0\0\0\0000\0\0\0001\0\0\0\0\0\0\0\0\0\0\0002\0\0\0\0\0\0\0005\0\0\08\0\0\09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0;\0\0\0<\0\0\0>\0\0\0\0\0\0\0?\0\0\0A\0\0\0\0\0\0\0\0\0\0\0B\0\0\0C\0\0\0D\0\0\0E\0\0\0H\0\0\0I\0\0\0J\0\0\0K\0\0\0\0\0\0\0M\0\0\0Q\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0R\0\0\0S\0\0\0T\0\0\0\0\0\0\0U\0\0\0\0\0\0\0W\0\0\0[\0\0\0]\0\0\0`\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=67736, ...}) = 0
mmap(NULL, 131088, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc484b0000
mmap(0xfffc484c0000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xfffc484c0000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0P\26\0\0\0\0\0\0@\0\0\0\0\0\0\0\360\3\1\0\0\0\0\0\0\0\0\0@\08\0\7\0@\0\32\0\31\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0PB\0\0\0\0\0\0PB\0\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0\300\375\0\0\0\0\0\0\300\375\1\0\0\0\0\0\300\375\1\0\0\0\0\0p\4\0\0\0\0\0\0x\4\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0\320\375\0\0\0\0\0\0\320\375\1\0\0\0\0\0\320\375\1\0\0\0\0\0\340\1\0\0\0\0\0\0\340\1\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0$\0\0\0\0\0\0\0$\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0P\345td\4\0\0\0\2748\0\0\0\0\0\0\2748\0\0\0\0\0\0\2748\0\0\0\0\0\0L\1\0\0\0\0\0\0L\1\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0\300\375\0\0\0\0\0\0\300\375\1\0\0\0\0\0\300\375\1\0\0\0\0\0@\2\0\0\0\0\0\0@\2\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\300\350\371oWH\245B\4\305Z\3169\257\356\372p@3\221\0\0\0\0\21\0\0\0 \0\0\0\4\0\0\0\10\0\0\0@\10$\201\30\20@\204\230\246\0(\212\0\30\244\20\202\220\4\0\210\233\0\n\0H\340\204P\2\220 \0\0\0\0\0\0\0\0\0\0\0\"\0\0\0&\0\0\0'\0\0\0(\0\0\0*\0\0\0,\0\0\0001\0\0\0006\0\0\08\0\0\0\0\0\0\0:\0\0\0;\0\0\0=\0\0\0@\0\0\0z\177\352\33E3vg\216\332\241]\nzqP\210q\356\262[\207i\224\301\277<\316\37\344l\224\30\277[\27\261D=\\t\343\324\225{a,\20\334\347\250\231\266\257\203\372\360\342kP\226t\207c\25\6\365\262`\n\352\262\302.\334:\226\327kPB\312*\362M\247:\352NI\2032\337\374~\25,\366-\20#\330\250\231}\335\201c\354\26\v\210#\322\201c\252T\275\331\322^3\20\377\356\311x\261s\16\210\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\t\0(\24\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\25\0\370\0\2\0\0\0\0\0\0\0\0\0\0\0\0\0r\0\0\0\22\0\0\0\0\0\0\0\0\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=68208, ...}) = 0
mmap(NULL, 131640, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc48480000
mmap(0xfffc48490000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xfffc48490000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0\0\21\0\0\0\0\0\0@\0\0\0\0\0\0\0\330\1\1\0\0\0\0\0\0\0\0\0@\08\0\7\0@\0\34\0\33\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0@%\0\0\0\0\0\0@%\0\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0\210\374\0\0\0\0\0\0\210\374\1\0\0\0\0\0\210\374\1\0\0\0\0\0\200\3\0\0\0\0\0\0\330\3\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0\250\374\0\0\0\0\0\0\250\374\1\0\0\0\0\0\250\374\1\0\0\0\0\0000\2\0\0\0\0\0\0000\2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0D\0\0\0\0\0\0\0D\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0P\345td\4\0\0\0\350\37\0\0\0\0\0\0\350\37\0\0\0\0\0\0\350\37\0\0\0\0\0\0\344\0\0\0\0\0\0\0\344\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0\210\374\0\0\0\0\0\0\210\374\1\0\0\0\0\0\210\374\1\0\0\0\0\0x\3\0\0\0\0\0\0x\3\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0]\375\236\"\352\256$\367\300\0\26+^\325\334\237\364P\3\213\4\0\0\0\20\0\0\0\1\0\0\0GNU\0\0\0\0\0\3\0\0\0\7\0\0\0\0\0\0\0\0\0\0\0?\0\0\0-\0\0\0\0\0\0\0\30\0\0\0\26\0\0\0%\0\0\0\0\0\0\0\0\0\0\0)\0\0\0\23\0\0\0\0\0\0\0!\0\0\0\"\0\0\0\0\0\0\0\10\0\0\0\37\0\0\0\27\0\0\0\36\0\0\0\t\0\0\0\0\0\0\0\7\0\0\0\0\0\0\0\22\0\0\0\0\0\0\0\35\0\0\0\32\0\0\0(\0\0\0\0\0\0\0$\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\31\0\0\0\0\0\0\0\24\0\0\0\0\0\0\0\f\0\0\0\0\0\0\0&\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\0\0\25\0\0\0'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0#\0\0\0\r\0\0\0\33\0\0\0\0\0\0\0\0\0\0\0,\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\n\0\0\0\0\0\0\0\v\0\0\0\17\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=67800, ...}) = 0
mmap(NULL, 131168, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc48450000
mmap(0xfffc48460000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xfffc48460000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0 A\2\0\0\0\0\0@\0\0\0\0\0\0\0\350L\27\0\0\0\0\0\0\0\0\0@\08\0\n\0@\0L\0K\0\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0000\2\0\0\0\0\0\0000\2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\3\0\0\0\4\0\0\0\210\237\23\0\0\0\0\0\210\237\23\0\0\0\0\0\210\237\23\0\0\0\0\0\33\0\0\0\0\0\0\0\33\0\0\0\0\0\0\0\10\0\0\0\0\0\0\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0`a\26\0\0\0\0\0`a\26\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0\10\327\26\0\0\0\0\0\10\327\27\0\0\0\0\0\10\327\27\0\0\0\0\08L\0\0\0\0\0\0\20\206\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0\250\372\26\0\0\0\0\0\250\372\27\0\0\0\0\0\250\372\27\0\0\0\0\0\360\1\0\0\0\0\0\0\360\1\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0p\2\0\0\0\0\0\0p\2\0\0\0\0\0\0p\2\0\0\0\0\0\0D\0\0\0\0\0\0\0D\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0\7\0\0\0\4\0\0\0\10\327\26\0\0\0\0\0\10\327\27\0\0\0\0\0\10\327\27\0\0\0\0\0\20\0\0\0\0\0\0\0\220\0\0\0\0\0\0\0\10\0\0\0\0\0\0\0P\345td\4\0\0\0\244\237\23\0\0\0\0\0\244\237\23\0\0\0\0\0\244\237\23\0\0\0\0\0\314X\0\0\0\0\0\0\314X\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0\10\327\26\0\0\0\0\0\10\327\27\0\0\0\0\0\10\327\27\0\0\0\0\0\370(\0\0\0\0\0\0\370(\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\334=\347%8\227WC\33\234\6}=\305-\250\344ct\222\4\0\0\0\20\0\0\0\1\0\0\0GNU\0\0\0\0\0\3\0\0\0\7\0\0\0\0\0\0\0\0\0\0\0\371\3\0\0\371\10\0\0\352\1\0\0(\3\0\0\273\10\0\0)\4\0\0\302\5\0\0\332\2\0\0\0\0\0\0\267\0\0\0\274\1\0\0\217\6\0\0\255\0\0\0@\4\0\0\"\1\0\0\204\4\0\0\235\1\0\0\6\7\0\0\346\1\0\0\352\3\0\0\246\7\0\0\22\6\0\0\214\7\0\0a\5\0\0\0\0\0\0h\5\0\0}\7\0\0\0\0\0\0\252\7\0\0\367\5\0\0\345\6\0\0\n\4\0\0\362\3\0\0\32\6\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1531880, ...}) = 0
mmap(NULL, 1596696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc482c0000
mmap(0xfffc48430000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x160000) = 0xfffc48430000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libsecurity.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0\240\7\0\0\0\0\0\0@\0\0\0\0\0\0\0\310\1\1\0\0\0\0\0\0\0\0\0@\08\0\7\0@\0\31\0\30\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0X\f\0\0\0\0\0\0X\f\0\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0X\375\0\0\0\0\0\0X\375\1\0\0\0\0\0X\375\1\0\0\0\0\0\260\2\0\0\0\0\0\0\270\2\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0h\375\0\0\0\0\0\0h\375\1\0\0\0\0\0h\375\1\0\0\0\0\0\0\2\0\0\0\0\0\0\0\2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0$\0\0\0\0\0\0\0$\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0P\345td\4\0\0\0\24\v\0\0\0\0\0\0\24\v\0\0\0\0\0\0\24\v\0\0\0\0\0\0<\0\0\0\0\0\0\0<\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0X\375\0\0\0\0\0\0X\375\1\0\0\0\0\0X\375\1\0\0\0\0\0\250\2\0\0\0\0\0\0\250\2\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\16\257\226]g\0345\6\322>\257\246\214\344G\253f\\Ve\0\0\0\0\2\0\0\0\20\0\0\0\1\0\0\0\6\0\0\0\0\0\0\0\0 \0\4\0\0\0\0\20\0\0\0\255^\333_\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\t\0\330\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\24\0\0\0\2\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0 \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0F\0\0\0\"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0t\0\0\0\21\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\232\0\0\0\22\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\243\0\0\0\22\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1\0\0\0 \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0U\0\0\0\21\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\314\0\0\0\22\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0{\0\0\0\22\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0m\0\0\0\22\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=67592, ...}) = 0
mmap(NULL, 131088, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc48290000
mmap(0xfffc482a0000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xfffc482a0000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0\200#\0\0\0\0\0\0@\0\0\0\0\0\0\0000\3\10\0\0\0\0\0\0\0\0\0@\08\0\7\0@\0\32\0\31\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0(\272\7\0\0\0\0\0(\272\7\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0\340\372\7\0\0\0\0\0\340\372\10\0\0\0\0\0\340\372\10\0\0\0\0\0\210\6\0\0\0\0\0\0\350\6\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0\270\373\7\0\0\0\0\0\270\373\10\0\0\0\0\0\270\373\10\0\0\0\0\0\20\2\0\0\0\0\0\0\20\2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0$\0\0\0\0\0\0\0$\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0P\345td\4\0\0\0Xh\7\0\0\0\0\0Xh\7\0\0\0\0\0Xh\7\0\0\0\0\0L\7\0\0\0\0\0\0L\7\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0\340\372\7\0\0\0\0\0\340\372\10\0\0\0\0\0\340\372\10\0\0\0\0\0 \5\0\0\0\0\0\0 \5\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\331\\\211\253\213R\370q\301\2320\220\"\210(\v\2517_\334\0\0\0\0C\0\0\0\34\0\0\0\10\0\0\0\t\0\0\0#&`tELfQ\3D\6\0\245\244\25\252\2\2\0\3@\0\0\0\30\22\201\210\0\200H\0\"\0020\20@\2\30\0t\20AA(SB\201\0 \31\244\222\2@T\18\314\0217\4\1&\34\0\0\0\35\0\0\0\37\0\0\0!\0\0\0#\0\0\0\0\0\0\0\0\0\0\0%\0\0\0\0\0\0\0'\0\0\0)\0\0\0\0\0\0\0*\0\0\0\0\0\0\0+\0\0\0\0\0\0\0,\0\0\0-\0\0\0.\0\0\0/\0\0\0000\0\0\0001\0\0\0\0\0\0\0003\0\0\0005\0\0\0\0\0\0\0\0\0\0\0006\0\0\0\0\0\0\0007\0\0\08\0\0\09\0\0\0:\0\0\0<\0\0\0>\0\0\0?\0\0\0@\0\0\0B\0\0\0C\0\0\0E\0\0\0F\0\0\0G\0\0\0\0\0\0\0J\0\0\0\0\0\0\0M\0\0\0N\0\0\0\0\0\0\0Q\0\0\0R\0\0\0\0\0\0\0S\0\0\0T\0\0\0V\0\0\0W\0\0\0X\0\0\0Y\0\0\0Z\0\0\0\\\0\0\0^\0\0\0`\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=526768, ...}) = 0
mmap(NULL, 590280, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc481f0000
mmap(0xfffc48270000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x70000) = 0xfffc48270000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\0\340$\0\0\0\0\0\0@\0\0\0\0\0\0\0\330\1\2\0\0\0\0\0\0\0\0\0@\08\0\7\0@\0\33\0\32\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0000\256\1\0\0\0\0\0000\256\1\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0\260\372\1\0\0\0\0\0\260\372\2\0\0\0\0\0\260\372\2\0\0\0\0\0X\5\0\0\0\0\0\0`\5\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0\20\374\1\0\0\0\0\0\20\374\2\0\0\0\0\0\20\374\2\0\0\0\0\0 \2\0\0\0\0\0\0 \2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0\310\1\0\0\0\0\0\0$\0\0\0\0\0\0\0$\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0P\345td\4\0\0\0\340\206\1\0\0\0\0\0\340\206\1\0\0\0\0\0\340\206\1\0\0\0\0\0\304\3\0\0\0\0\0\0\304\3\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0\260\372\1\0\0\0\0\0\260\372\2\0\0\0\0\0\260\372\2\0\0\0\0\0P\5\0\0\0\0\0\0P\5\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\271\263\36/[g\360'1\327<\234rDa\26b\0E\206\0\0\0\0a\0\0\0\31\0\0\0\20\0\0\0\n\0\0\0\0\"@b\f\3\2\0E\0\2A\10\4%\22\4\1\4\4\0\6\10\0\4\1\26\n\0\21\2\0\0A\0\224\204\204\10\10\0l\0\24 \260\204\200\0\2603@\t\360d\200\2B\206\0\0\0\4\20\0\1 \24\0\2\2\0\10\0#\n\2\10\7 \f\0\4\340\1\0 \0 \0H!\22\35\230\221\0/12\1\6\0\10\4\0\0\0\200!\2\n\200\2\0\10 G\0\1Q$\0\0\2\202\0\0\0\0\0\0\31\0\0\0\34\0\0\0\35\0\0\0\0\0\0\0\37\0\0\0\"\0\0\0$\0\0\0%\0\0\0)\0\0\0\0\0\0\0\0\0\0\0*\0\0\0,\0\0\0-\0\0\0\0\0\0\0\0\0\0\0.\0\0\0000\0\0\0004\0\0\0005\0\0\0\0\0\0\0007\0\0\0\0\0\0\0\0\0\0\0\0\0\0\08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0:\0\0\0<\0\0\0\0\0\0\0\0\0\0\0>\0\0\0?\0\0\0@\0\0\0\0\0\0\0\0\0\0\0A\0\0\0\0\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=133272, ...}) = 0
mmap(NULL, 196624, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc481b0000
mmap(0xfffc481d0000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x10000) = 0xfffc481d0000
close(3)                                = 0
openat(AT_FDCWD, "/dev/cur_gl", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
openat(AT_FDCWD, "/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0\267\0\1\0\0\08t\0\0\0\0\0\0@\0\0\0\0\0\0\0\340\f\2\0\0\0\0\0\0\0\0\0@\08\0\t\0@\0\"\0!\0\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0\370\1\0\0\0\0\0\0\370\1\0\0\0\0\0\0\10\0\0\0\0\0\0\0\3\0\0\0\4\0\0\0Pe\1\0\0\0\0\0Pe\1\0\0\0\0\0Pe\1\0\0\0\0\0\33\0\0\0\0\0\0\0\33\0\0\0\0\0\0\0\10\0\0\0\0\0\0\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\370\240\1\0\0\0\0\0\370\240\1\0\0\0\0\0\0\0\1\0\0\0\0\0\1\0\0\0\6\0\0\0\340\370\1\0\0\0\0\0\340\370\2\0\0\0\0\0\340\370\2\0\0\0\0\0X\7\0\0\0\0\0\0xH\0\0\0\0\0\0\0\0\1\0\0\0\0\0\2\0\0\0\6\0\0\0\220\372\1\0\0\0\0\0\220\372\2\0\0\0\0\0\220\372\2\0\0\0\0\0000\2\0\0\0\0\0\0000\2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\4\0\0\0\4\0\0\08\2\0\0\0\0\0\08\2\0\0\0\0\0\08\2\0\0\0\0\0\0D\0\0\0\0\0\0\0D\0\0\0\0\0\0\0\4\0\0\0\0\0\0\0P\345td\4\0\0\0le\1\0\0\0\0\0le\1\0\0\0\0\0le\1\0\0\0\0\0\324\10\0\0\0\0\0\0\324\10\0\0\0\0\0\0\4\0\0\0\0\0\0\0Q\345td\6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\20\0\0\0\0\0\0\0R\345td\4\0\0\0\340\370\1\0\0\0\0\0\340\370\2\0\0\0\0\0\340\370\2\0\0\0\0\0 \7\0\0\0\0\0\0 \7\0\0\0\0\0\0\1\0\0\0\0\0\0\0\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\3)\354\364\355\24c\244\324\375*\261\240%~\205\340\210\27G\4\0\0\0\20\0\0\0\1\0\0\0GNU\0\0\0\0\0\3\0\0\0\7\0\0\0\0\0\0\0\0\0\0\0\236\2\0\0d\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0z\0\0\0\220\0\0\0\230\0\0\0\0\0\0\0\0\0\0\0D\0\0\0\0\0\0\0O\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0[\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\333\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0P\1\0\0\0\0\0\0\0\0\0\0\22\0\0\0\0\0\0\0\0\0\0\0\365\0\0\0\0\0\0\0\326\0\0\0\0\0\0\0!\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0b\0\0\0/\1\0\0\0\0\0\0\22\1\0\0\316\0\0\0", 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=136544, ...}) = 0
mmap(NULL, 213336, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xfffc48170000
mmap(0xfffc48190000, 131072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x10000) = 0xfffc48190000
close(3)                                = 0
mprotect(0xfffc48430000, 65536, PROT_READ) = 0
mprotect(0xfffc48190000, 65536, PROT_READ) = 0
mprotect(0xfffc481d0000, 65536, PROT_READ) = 0
mprotect(0xfffc48270000, 65536, PROT_READ) = 0
mprotect(0xfffc482a0000, 65536, PROT_READ) = 0
mprotect(0xfffc48460000, 65536, PROT_READ) = 0
mprotect(0xfffc48490000, 65536, PROT_READ) = 0
mprotect(0xfffc484c0000, 65536, PROT_READ) = 0
mprotect(0xfffc48500000, 65536, PROT_READ) = 0
mprotect(0xfffc48550000, 65536, PROT_READ) = 0
mprotect(0xaaadfbff0000, 65536, PROT_READ) = 0
mprotect(0xfffc485e0000, 65536, PROT_READ) = 0
munmap(0xfffc48570000, 106368)          = 0
set_tid_address(0xfffc485f6060)         = 2614803
set_robust_list(0xfffc485f6070, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0xfffc48176e98, sa_mask=[], sa_flags=SA_SIGINFO}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0xfffc48176f68, sa_mask=[], sa_flags=SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0xffffcc63fe40) = -1 ENOENT (没有那个文件或目录)
statfs("/selinux", 0xffffcc63fe40)      = -1 ENOENT (没有那个文件或目录)
brk(NULL)                               = 0xaaae33200000
brk(0xaaae33230000)                     = 0xaaae33230000
openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tramfs\nnodev\tbdev\nnodev\tproc\nnodev\tcpuset\nnodev\tcgroup\nnodev\tcgroup2\nnodev\ttmpfs\nnodev\tdevtmpfs\nnodev\tconfigfs\nnodev\tdebugfs\nnodev\tsecurityfs\nnodev\tsockfs\nnodev\tdax\nnodev\tbpf\nnodev\tpipefs\nnodev\thugetlbfs\nnodev\tdevpts\n\text3\n\text2\n\text4\nnodev\tecryptfs\nnodev\tautofs\n\txfs\nnodev\tefivarfs\nnodev\tmqueue\nnodev\tpstore\n\tvfat\nnodev\trpc_pipefs\nnodev\toverlay\nnodev\tbinfmt_misc\n", 1024) = 394
read(3, "", 1024)                       = 0
close(3)                                = 0
faccessat(AT_FDCWD, "/etc/selinux/config", F_OK) = 0
getuid()                                = 0
socket(AF_NETLINK, SOCK_RAW|SOCK_CLOEXEC, NETLINK_ROUTE) = 3
setsockopt(3, SOL_SOCKET, SO_SNDBUF, [32768], 4) = 0
setsockopt(3, SOL_SOCKET, SO_RCVBUF, [1048576], 4) = 0
setsockopt(3, SOL_NETLINK, NETLINK_EXT_ACK, [1], 4) = 0
bind(3, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, 12) = 0
getsockname(3, {sa_family=AF_NETLINK, nl_pid=2614803, nl_groups=00000000}, [12]) = 0
setsockopt(3, SOL_NETLINK, NETLINK_DUMP_STRICT_CHK, [1], 4) = -1 ENOPROTOOPT (不可用的协议)
sendto(3, {{len=32, type=RTM_NEWLINK, flags=NLM_F_REQUEST|NLM_F_ACK, seq=0, pid=0}, {ifi_family=AF_UNSPEC, ifi_type=ARPHRD_NETROM, ifi_index=0, ifi_flags=0, ifi_change=0}}, 32, 0, NULL, 0) = 32
recvmsg(3, {msg_name={sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, msg_namelen=12, msg_iov=[{iov_base={{len=52, type=NLMSG_ERROR, flags=0, seq=0, pid=2614803}, {error=-ENODEV, msg={{len=32, type=RTM_NEWLINK, flags=NLM_F_REQUEST|NLM_F_ACK, seq=0, pid=0}, {ifi_family=AF_UNSPEC, ifi_type=ARPHRD_NETROM, ifi_index=0, ifi_flags=0, ifi_change=0}}}}, iov_len=16384}], msg_iovlen=1, msg_controllen=0, msg_flags=0}, 0) = 52
futex(0xfffc48470048, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/usr/lib/ip/link_veth.so", O_RDONLY|O_CLOEXEC) = -1 ENOENT (没有那个文件或目录)
sendmsg(3, {msg_name={sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, msg_namelen=12, msg_iov=[{iov_base={{len=100, type=RTM_NEWLINK, flags=NLM_F_REQUEST|NLM_F_ACK|NLM_F_EXCL|NLM_F_CREATE, seq=1604388683, pid=0}, {ifi_family=AF_UNSPEC, ifi_type=ARPHRD_NETROM, ifi_index=0, ifi_flags=0, ifi_change=0}, [{{nla_len=14, nla_type=IFLA_IFNAME}, "veth-p130"}, {{nla_len=52, nla_type=IFLA_LINKINFO}, [{{nla_len=8, nla_type=IFLA_INFO_KIND}, "veth"...}, {{nla_len=40, nla_type=IFLA_INFO_DATA}, "\x24\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0e\x00\x03\x00\x76\x65\x74\x68\x2d\x70\x30\x31\x33\x00\x00\x00"}]}]}, iov_len=100}], msg_iovlen=1, msg_controllen=0, msg_flags=0}, 0) = 100
recvmsg(3, {msg_name={sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, msg_namelen=12, msg_iov=[{iov_base=NULL, iov_len=0}], msg_iovlen=1, msg_controllen=0, msg_flags=MSG_TRUNC}, MSG_PEEK|MSG_TRUNC) = 36
recvmsg(3, {msg_name={sa_family=AF_NETLINK, nl_pid=0, nl_groups=00000000}, msg_namelen=12, msg_iov=[{iov_base={{len=36, type=NLMSG_ERROR, flags=NLM_F_CAPPED, seq=1604388683, pid=2614803}, {error=0, msg={len=100, type=RTM_NEWLINK, flags=NLM_F_REQUEST|NLM_F_ACK|NLM_F_EXCL|NLM_F_CREATE, seq=1604388683, pid=0}}}, iov_len=32768}], msg_iovlen=1, msg_controllen=0, msg_flags=0}, 0) = 36
exit_group(0)                           = ?
+++ exited with 0 +++

Detailed Description

Possible Implementation

chenchun commented 3 years ago

Thanks for reporting. I'm quite confused of your suggestion

add delay(sleep) after https://github.com/tkestack/galaxy/blob/master/pkg/utils/utils.go#L243-L246

Do you mean after creating veth device, we should wait for its mac address to be present?

chenchun commented 3 years ago

Also I have tested on kernel 4.14.105 and I can't reproduce the issue you reported. Sorry, I can't get a machine of kernel: 4.19.90-17.ky10.aarch64, arch: arm64 to try to reproduce your issue.

sstoner commented 3 years ago

Thanks for reporting. I'm quite confused of your suggestion

add delay(sleep) after https://github.com/tkestack/galaxy/blob/master/pkg/utils/utils.go#L243-L246

Do you mean after creating veth device, we should wait for its mac address to be present?

yeah, because the mac address will be changed once after we created veth devices(see "Steps to Reproduce" B ), we need to aoivd to get the mac address passed away.

I'm suspicious of this fault maybe caused by kernel. I will close this issue and thanks for your reply.

yaoice commented 3 years ago

I have the same issue, do you use the kylin v10 system with arm? @sstoner

sstoner commented 3 years ago

I have the same issue, do you use the kylin v10 system with arm? @sstoner

@yaoice yeah...

do you find suitable method to resolve it?