rapiz1 / rathole

A lightweight and high-performance reverse proxy for NAT traversal, written in Rust. An alternative to frp and ngrok.
Apache License 2.0
9.8k stars 488 forks source link

Powerpc Failed to run the control channel: Device or resource busy (os error 16) #173

Open shyn opened 2 years ago

shyn commented 2 years ago

Describe the bug

To Reproduce Steps to reproduce the behavior:

  1. cross compile for powerpc with static linking
  2. run with client mode

Configuration Configuration used to reproduce the behavior:

Logs

root@OpenWrt:/data# strace ./rathole config.toml
execve("./rathole", ["./rathole", "config.toml"], 0xbf933394 /* 14 vars */) = 0
brk(NULL)                               = 0x10613000
brk(0x10613e68)                         = 0x10613e68
uname({sysname="Linux", nodename="OpenWrt", ...}) = 0
set_tid_address(0x10613068)             = 28322
set_robust_list(0x10613070, 12)         = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x101c0420, sa_mask=[], sa_flags=SA_SIGINFO}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x101c0510, sa_mask=[], sa_flags=SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
ugetrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
readlink("/proc/self/exe", "/data/rathole", 4096) = 13
brk(0x10634e68)                         = 0x10634e68
brk(0x10635000)                         = 0x10635000
mprotect(0x10324000, 180224, PROT_READ) = 0
poll([{fd=0, events=0}, {fd=1, events=0}, {fd=2, events=0}], 3, 0) = 0 (Timeout)
rt_sigaction(SIGPIPE, {sa_handler=SIG_IGN, sa_mask=[PIPE], sa_flags=SA_RESTART}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGSEGV, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGSEGV, {sa_handler=0x10148470, sa_mask=[], sa_flags=SA_ONSTACK|SA_SIGINFO}, NULL, 8) = 0
rt_sigaction(SIGBUS, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
rt_sigaction(SIGBUS, {sa_handler=0x10148470, sa_mask=[], sa_flags=SA_ONSTACK|SA_SIGINFO}, NULL, 8) = 0
sigaltstack(NULL, {ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=0}) = 0
mmap2(NULL, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb7d5f000
mprotect(0xb7d5f000, 4096, PROT_NONE)   = 0
sigaltstack({ss_sp=0xb7d60000, ss_flags=0, ss_size=16384}, NULL) = 0
openat(AT_FDCWD, "/proc/self/maps", O_RDONLY|O_CLOEXEC) = 3
ugetrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "00100000-00103000 r-xp 00000000 "..., 1024) = 481
close(3)                                = 0
sched_getaffinity(28322, 32, [0])       = 4
openat(AT_FDCWD, "/proc/self/cgroup", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3
read(3, "0::/services/dropbear/instance1\n", 8192) = 32
read(3, "", 8192)                       = 0
close(3)                                = 0
sched_getaffinity(0, 128, [0])          = 4
epoll_create1(EPOLL_CLOEXEC)            = 3
eventfd2(0, EFD_CLOEXEC|EFD_NONBLOCK)   = 4
epoll_ctl(3, EPOLL_CTL_ADD, 4, {EPOLLIN|EPOLLRDHUP|EPOLLET, {u32=0, u64=2147483648}}) = 0
fcntl64(3, F_DUPFD_CLOEXEC, 3)          = 5
socketpair(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0, [6, 7]) = 0
fcntl64(6, F_DUPFD_CLOEXEC, 0)          = 8
epoll_ctl(5, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=0, u64=0}}) = 0
clock_gettime64(CLOCK_MONOTONIC, {tv_sec=1642298, tv_nsec=147363218}) = 0
getrandom("\xec\xf2\x95\x3e\xb4\xac\x80\x30\x79\x0e\x56\x49\x80\xf5\x00\xbd", 16, GRND_NONBLOCK) = 16
futex(0x10353c5c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
mmap2(NULL, 2101248, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb7b5e000
mprotect(0xb7b5f000, 2097152, PROT_READ|PROT_WRITE) = 0
clone(child_stack=0xb7d5dd00, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[28323], tls=0xb7d65670, child_tidptr=0xb7d5e208) = 28323
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
brk(0x1065a000)                         = 0x1065a000
brk(0x10657000)                         = 0x10657000
brk(0x1067d000)                         = 0x1067d000
brk(0x106ae000)                         = 0x106ae000
mmap2(NULL, 167936, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7b30000
munmap(0xb7b30000, 167936)              = 0
ugetrlimit(RLIMIT_NOFILE, {rlim_cur=1024, rlim_max=4*1024}) = 0
prlimit64(0, RLIMIT_NOFILE, {rlim_cur=4*1024, rlim_max=4*1024}, NULL) = 0
mmap2(NULL, 2101248, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb77ff000
mprotect(0xb7800000, 2097152, PROT_READ|PROT_WRITE) = 0
clone(child_stack=0xb79fed00, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[28324], tls=0xb7a06670, child_tidptr=0xb79ff208) = 28324
clock_gettime64(CLOCK_MONOTONIC, {tv_sec=1642298, tv_nsec=174408757}) = 0
futex(0x106134d0, FUTEX_WAIT_PRIVATE, 1, NULL) = 0
write(4, "\0\0\0\0\0\0\0\1", 8)         = 8
write(4, "\0\0\0\0\0\0\0\1", 8Aug 07 12:54:38.296  INFO handle{service=my_nas_ssh}: rathole::client: Starting e20655976b0b281da642b3acc136a6ede1ab2c7428bf8b3f3b3e99364c327594
)         = 8
futex(0x106134d0, FUTEX_WAIT_PRIVATE, 1, NULLAug 07 12:54:38.298 ERROR handle{service=my_nas_ssh}: rathole::client: Failed to run the control channel: Device or resource busy (os error 16). Retry in 473.275921ms...

Environment:


Cross compiled for my powerpc NAS: RUSTFLAGS='-C target-feature=+crt-static' cross build --target powerpc-unknown-linux-gnu --release --no-default-features --features client

rapiz1 commented 2 years ago

This looks more like a permission/resource issue rather than an architecture one. You can try increasing the nofile limit on your system to a large enough value e.g. 32768 .

Nproc limit is also worth checking

---Original--- From: @.> Date: Sun, Aug 7, 2022 21:06 PM To: @.>; Cc: @.***>; Subject: [rapiz1/rathole] Powerpc Failed to run the control channel: Device orresource busy (os error 16) (Issue #173)

Describe the bug

To Reproduce Steps to reproduce the behavior:

cross compile for powerpc with static linking

run with client mode

Configuration Configuration used to reproduce the behavior:

Logs @.**:/data# strace ./rathole config.toml execve("./rathole", ["./rathole", "config.toml"], 0xbf933394 / 14 vars /) = 0 brk(NULL) = 0x10613000 brk(0x10613e68) = 0x10613e68 uname({sysname="Linux", nodename="OpenWrt", ...}) = 0 set_tid_address(0x10613068) = 28322 set_robust_list(0x10613070, 12) = 0 rt_sigaction(SIGRTMIN, {sa_handler=0x101c0420, sa_mask=[], sa_flags=SA_SIGINFO}, NULL, 8) = 0 rt_sigaction(SIGRT_1, {sa_handler=0x101c0510, sa_mask=[], sa_flags=SA_RESTART|SA_SIGINFO}, NULL, 8) = 0 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 ugetrlimit(RLIMIT_STACK, {rlim_cur=81921024, rlim_max=RLIM_INFINITY}) = 0 readlink("/proc/self/exe", "/data/rathole", 4096) = 13 brk(0x10634e68) = 0x10634e68 brk(0x10635000) = 0x10635000 mprotect(0x10324000, 180224, PROT_READ) = 0 poll([{fd=0, events=0}, {fd=1, events=0}, {fd=2, events=0}], 3, 0) = 0 (Timeout) rt_sigaction(SIGPIPE, {sa_handler=SIG_IGN, sa_mask=[PIPE], sa_flags=SA_RESTART}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 rt_sigaction(SIGSEGV, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 rt_sigaction(SIGSEGV, {sa_handler=0x10148470, sa_mask=[], sa_flags=SA_ONSTACK|SA_SIGINFO}, NULL, 8) = 0 rt_sigaction(SIGBUS, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0 rt_sigaction(SIGBUS, {sa_handler=0x10148470, sa_mask=[], sa_flags=SA_ONSTACK|SA_SIGINFO}, NULL, 8) = 0 sigaltstack(NULL, {ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=0}) = 0 mmap2(NULL, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb7d5f000 mprotect(0xb7d5f000, 4096, PROT_NONE) = 0 sigaltstack({ss_sp=0xb7d60000, ss_flags=0, ss_size=16384}, NULL) = 0 openat(AT_FDCWD, "/proc/self/maps", O_RDONLY|O_CLOEXEC) = 3 ugetrlimit(RLIMIT_STACK, {rlim_cur=81921024, rlim_max=RLIM_INFINITY}) = 0 fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0 read(3, "00100000-00103000 r-xp 00000000 "..., 1024) = 481 close(3) = 0 sched_getaffinity(28322, 32, [0]) = 4 openat(AT_FDCWD, "/proc/self/cgroup", O_RDONLY|O_LARGEFILE|O_CLOEXEC) = 3 read(3, "0::/services/dropbear/instance1\n", 8192) = 32 read(3, "", 8192) = 0 close(3) = 0 sched_getaffinity(0, 128, [0]) = 4 epoll_create1(EPOLL_CLOEXEC) = 3 eventfd2(0, EFD_CLOEXEC|EFD_NONBLOCK) = 4 epoll_ctl(3, EPOLL_CTL_ADD, 4, {EPOLLIN|EPOLLRDHUP|EPOLLET, {u32=0, u64=2147483648}}) = 0 fcntl64(3, F_DUPFD_CLOEXEC, 3) = 5 socketpair(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0, [6, 7]) = 0 fcntl64(6, F_DUPFD_CLOEXEC, 0) = 8 epoll_ctl(5, EPOLL_CTL_ADD, 8, {EPOLLIN|EPOLLOUT|EPOLLRDHUP|EPOLLET, {u32=0, u64=0}}) = 0 clock_gettime64(CLOCK_MONOTONIC, {tv_sec=1642298, tv_nsec=147363218}) = 0 getrandom("\xec\xf2\x95\x3e\xb4\xac\x80\x30\x79\x0e\x56\x49\x80\xf5\x00\xbd", 16, GRND_NONBLOCK) = 16 futex(0x10353c5c, FUTEX_WAKE_PRIVATE, 2147483647) = 0 mmap2(NULL, 2101248, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb7b5e000 mprotect(0xb7b5f000, 2097152, PROT_READ|PROT_WRITE) = 0 clone(child_stack=0xb7d5dd00, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[28323], tls=0xb7d65670, child_tidptr=0xb7d5e208) = 28323 ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0 brk(0x1065a000) = 0x1065a000 brk(0x10657000) = 0x10657000 brk(0x1067d000) = 0x1067d000 brk(0x106ae000) = 0x106ae000 mmap2(NULL, 167936, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7b30000 munmap(0xb7b30000, 167936) = 0 ugetrlimit(RLIMIT_NOFILE, {rlim_cur=1024, rlim_max=41024}) = 0 prlimit64(0, RLIMIT_NOFILE, {rlim_cur=41024, rlim_max=41024}, NULL) = 0 mmap2(NULL, 2101248, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb77ff000 mprotect(0xb7800000, 2097152, PROT_READ|PROT_WRITE) = 0 clone(child_stack=0xb79fed00, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[28324], tls=0xb7a06670, child_tidptr=0xb79ff208) = 28324 clock_gettime64(CLOCK_MONOTONIC, {tv_sec=1642298, tv_nsec=174408757}) = 0 futex(0x106134d0, FUTEX_WAIT_PRIVATE, 1, NULL) = 0 write(4, "\0\0\0\0\0\0\0\1", 8) = 8 write(4, "\0\0\0\0\0\0\0\1", 8Aug 07 12:54:38.296 INFO handle{service=my_nas_ssh}: rathole::client: Starting e20655976b0b281da642b3acc136a6ede1ab2c7428bf8b3f3b3e99364c327594 ) = 8 futex(0x106134d0, FUTEX_WAIT_PRIVATE, 1, NULLAug 07 12:54:38.298 ERROR handle{service=my_nas_ssh}: rathole::client: Failed to run the control channel: Device or resource busy (os error 16). Retry in 473.275921ms...

Environment:

OS: Linux OpenWrt 5.4.154 #0 Sun Oct 24 09:01:35 2021 ppc GNU/Linux

rathole --version output: Build Timestamp: 2022-08-07T12:31:46.346166431+00:00 Build Version: 0.4.2 Commit SHA: Some("1f2fc5b28f0c5034da6ca684d6a7b8f144e00224") Commit Date: Some("2022-06-11T03:39:11+00:00") Commit Branch: Some("main") cargo Target Triple: powerpc-unknown-linux-gnu cargo Profile: release cargo Features: client

CPU architecture: APM821XX (PowerPC 44x Platform)

Cross compiled for my powerpc NAS: RUSTFLAGS='-C target-feature=+crt-static' cross build --target powerpc-unknown-linux-gnu --release --no-default-features --features client

— Reply to this email directly, view it on GitHub, or unsubscribe. You are receiving this because you are subscribed to this thread.Message ID: @.***>

shyn commented 2 years ago

@rapiz1 thx. I tried changing nofile limit. Not work.

root@OpenWrt:/data# ulimit -a
core file size (blocks)         (-c) 0
data seg size (kb)              (-d) unlimited
scheduling priority             (-e) 0
file size (blocks)              (-f) unlimited
pending signals                 (-i) 1966
max locked memory (kb)          (-l) 64
max memory size (kb)            (-m) unlimited
open files                      (-n) 32768
POSIX message queues (bytes)    (-q) 819200
real-time priority              (-r) 0
stack size (kb)                 (-s) 8192
cpu time (seconds)              (-t) unlimited
max user processes              (-u) 32768
virtual memory (kb)             (-v) unlimited
file locks                      (-x) unlimited
> ps | grep rathole
29032 root     12368 S    ./rathole config.toml
> cat /proc/29032/limits
Limit                     Soft Limit           Hard Limit           Units
Max cpu time              unlimited            unlimited            seconds
Max file size             unlimited            unlimited            bytes
Max data size             unlimited            unlimited            bytes
Max stack size            8388608              unlimited            bytes
Max core file size        0                    0                    bytes
Max resident set          unlimited            unlimited            bytes
Max processes             32768                32768                processes
Max open files            32768                32768                files
Max locked memory         65536                65536                bytes
Max address space         unlimited            unlimited            bytes
Max file locks            unlimited            unlimited            locks
Max pending signals       1966                 1966                 signals
Max msgqueue size         819200               819200               bytes
Max nice priority         0                    0
Max realtime priority     0                    0
Max realtime timeout      unlimited            unlimited            us