Illegal instruction with prebuilt binary

[roryk]

Hi Rob,

The prebuilt binary doesn't work on one system I am on:

Linux loge 2.6.32-504.12.2.el6.x86_64 #1 SMP Wed Mar 11 22:03:14 UTC 2015 x86_64 x86_64 x86_64 GNU/Linux

stracing it:

execve("/home/rdk4/miniconda/bin/rapmap", ["rapmap"], [/* 65 vars */]) = 0
brk(0)                                  = 0x283c000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f01681000
readlink("/proc/self/exe", "/home/rdk4/miniconda/bin/rapmap", 4096) = 31
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/home/rdk4/miniconda/bin/../lib/tls/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/rdk4/miniconda/bin/../lib/tls/x86_64", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/home/rdk4/miniconda/bin/../lib/tls/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/rdk4/miniconda/bin/../lib/tls", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/home/rdk4/miniconda/bin/../lib/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/rdk4/miniconda/bin/../lib/x86_64", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/home/rdk4/miniconda/bin/../lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/home/rdk4/miniconda/bin/../lib", {st_mode=S_IFDIR|S_ISGID|0775, st_size=9075, ...}) = 0
open("/opt/gcc-5.2.0/lib64/tls/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/gcc-5.2.0/lib64/tls/x86_64", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/tls/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/gcc-5.2.0/lib64/tls", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/gcc-5.2.0/lib64/x86_64", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/gcc-5.2.0/lib64", {st_mode=S_IFDIR|S_ISGID|0775, st_size=3463, ...}) = 0
open("/opt/bcbio/centos/lib/tls/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/bcbio/centos/lib/tls/x86_64", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/bcbio/centos/lib/tls/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/bcbio/centos/lib/tls", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/bcbio/centos/lib/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/bcbio/centos/lib/x86_64", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/bcbio/centos/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/bcbio/centos/lib", {st_mode=S_IFDIR|0775, st_size=402, ...}) = 0
open("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/tls/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/tls/x86_64", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/tls/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/tls", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/x86_64/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/x86_64", 0x7fff767d89a0) = -1 ENOENT (No such file or directory)
open("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/libpthread.so.0", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib", {st_mode=S_IFDIR|0755, st_size=3072, ...}) = 0
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=53831, ...}) = 0
mmap(NULL, 53831, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0f01673000
close(3)                                = 0
open("/lib64/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000^\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=142688, ...}) = 0
mmap(NULL, 2212848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0f01246000
mprotect(0x7f0f0125d000, 2097152, PROT_NONE) = 0
mmap(0x7f0f0145d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f0f0145d000
mmap(0x7f0f0145f000, 13296, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0f0145f000
close(3)                                = 0
open("/home/rdk4/miniconda/bin/../lib/libgomp.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/libgomp.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340a\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=711939, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f01672000
mmap(NULL, 2230416, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0f01025000
mprotect(0x7f0f01045000, 2097152, PROT_NONE) = 0
mmap(0x7f0f01245000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20000) = 0x7f0f01245000
close(3)                                = 0
open("/home/rdk4/miniconda/bin/../lib/librt.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/librt.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/bcbio/centos/lib/librt.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/librt.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/librt.so.1", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=43944, ...}) = 0
mmap(NULL, 2128816, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0f00e1d000
mprotect(0x7f0f00e24000, 2093056, PROT_NONE) = 0
mmap(0x7f0f01023000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f0f01023000
close(3)                                = 0
open("/home/rdk4/miniconda/bin/../lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/bcbio/centos/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/libm.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libm.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p>\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=596272, ...}) = 0
mmap(NULL, 2633912, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0f00b99000
mprotect(0x7f0f00c1c000, 2093056, PROT_NONE) = 0
mmap(0x7f0f00e1b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x82000) = 0x7f0f00e1b000
close(3)                                = 0
open("/home/rdk4/miniconda/bin/../lib/libgcc_s.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/libgcc_s.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260)\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=532820, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f01671000
mmap(NULL, 2185248, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0f00983000
mprotect(0x7f0f00999000, 2093056, PROT_NONE) = 0
mmap(0x7f0f00b98000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f0f00b98000
close(3)                                = 0
open("/home/rdk4/miniconda/bin/../lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/bcbio/centos/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\356\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1920936, ...}) = 0
mmap(NULL, 3750152, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0f005ef000
mprotect(0x7f0f00779000, 2097152, PROT_NONE) = 0
mmap(0x7f0f00979000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18a000) = 0x7f0f00979000
mmap(0x7f0f0097e000, 18696, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0f0097e000
close(3)                                = 0
open("/home/rdk4/miniconda/bin/../lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/gcc-5.2.0/lib64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/bcbio/centos/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/opt/lsf/7.0/linux2.6-glibc2.3-x86_64/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libdl.so.2", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19536, ...}) = 0
mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0f003eb000
mprotect(0x7f0f003ed000, 2097152, PROT_NONE) = 0
mmap(0x7f0f005ed000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f0f005ed000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f01670000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f0166f000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f0166d000
arch_prctl(ARCH_SET_FS, 0x7f0f0166d7e0) = 0
mprotect(0x7f0f005ed000, 4096, PROT_READ) = 0
mprotect(0x7f0f00979000, 16384, PROT_READ) = 0
mprotect(0x7f0f00e1b000, 4096, PROT_READ) = 0
mprotect(0x7f0f01023000, 4096, PROT_READ) = 0
mprotect(0x7f0f0145d000, 4096, PROT_READ) = 0
mprotect(0x7f0f01682000, 4096, PROT_READ) = 0
munmap(0x7f0f01673000, 53831)           = 0
set_tid_address(0x7f0f0166dab0)         = 16752
set_robust_list(0x7f0f0166dac0, 0x18)   = 0
futex(0x7fff767d929c, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0x7fff767d929c, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, 7f0f0166d7e0) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7f0f0124bcb0, [], SA_RESTORER|SA_SIGINFO, 0x7f0f012557e0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f0f0124bd40, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f0f012557e0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=10240*1024, rlim_max=RLIM_INFINITY}) = 0
open("/sys/devices/system/cpu", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
fcntl(3, F_GETFD)                       = 0x1 (flags FD_CLOEXEC)
readlink("/etc/malloc.conf", 0x7fff767d8060, 4096) = -1 ENOENT (No such file or directory)
mmap(NULL, 4194304, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0efffeb000
munmap(0x7f0efffeb000, 4194304)         = 0
mmap(NULL, 8384512, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0effbec000
munmap(0x7f0effbec000, 81920)           = 0
munmap(0x7f0f00000000, 4108288)         = 0
open("/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC) = 4
read(4, "0-23\n", 8192)                 = 5
close(4)                                = 0
mmap(NULL, 4194304, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0eff800000
getdents(3, /* 35 entries */, 32768)    = 1056
getdents(3, /* 0 entries */, 32768)     = 0
madvise(0x7f0eff806000, 36864, MADV_DONTNEED) = 0
close(3)                                = 0
sched_getaffinity(16752, 8,  { ffffff }) = 8
futex(0x86200c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x862018, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/dev/urandom", O_RDONLY)          = 3
fstat(3, {st_mode=S_IFCHR|0666, st_rdev=makedev(1, 9), ...}) = 0
ioctl(3, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff767d8ff0) = -1 EINVAL (Invalid argument)
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0f01680000
read(3, "\200\241\342VA\374F%\244\304:T\251\317\24\273O(\205\301\364\2053\221P\20m\227 i\221K"..., 4096) = 4096
--- SIGILL (Illegal instruction) @ 0 (0) ---
+++ killed by SIGILL +++
Illegal instruction

If I build it myself on the same system it works fine. The binary built as part of bioconda also has the same issue. Do you know what might be causing the difference?

[rob-p]

Hi Rory,

The binary was made with Holy Build Box, so I'm guessing / hoping that it's not a library-level issue (i.e. libc or some such). It is possible that there is an actual instruction used in the binary that is not supported on the target machine. This happens if e.g. the code is compiled on a machine with access to an instruction set not available on the machine where it is run. Unfortunately, I don't have a lot of experience with this. I wonder if there is some way to "check" a binary for the instruction sets it uses. In that case, the solution would be to explicitly disable those instructions during build (or provide multiple binaries where one uses the instructions and one does not).

[roryk]

Hi Rob,

I did an objdump of the broken binary and the binary that works, and then cut out the opcodes to see which ones are different and ran comm on the two lists. I attached the comm results, the first column are opcodes only in the broken rapmap binary, the second column is only in the one that works on this system and the third are common to both. No idea if that is helpful at all.

broken-vs-works-opcodes.txt

[roryk]

Here is the output from backtrace and disassemble on the broken binary:

Reading symbols from /home/rdk4/tmp/rapmap...done.
(gdb) run
Starting program: /home/rdk4/tmp/rapmap
[Thread debugging using libthread_db enabled]

Program received signal SIGILL, Illegal instruction.
0x000000000047eabe in _GLOBAL__sub_I_intersection.cpp ()
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.166.el6_7.7.x86_64
(gdb) backtrace
#0  0x000000000047eabe in _GLOBAL__sub_I_intersection.cpp ()
#1  0x00000000005c74bf in __libc_csu_init ()
#2  0x00007ffff6f87cf0 in __libc_start_main () from /lib64/libc.so.6
#3  0x000000000047ef19 in _start ()
(gdb) disassemble
Dump of assembler code for function _GLOBAL__sub_I_intersection.cpp:
   0x000000000047ea90 <+0>: push   %rbx
   0x000000000047ea91 <+1>: lea    0x3c0ca8(%rip),%rdi        # 0x83f740 <_ZStL8__ioinit>
   0x000000000047ea98 <+8>: callq  0x57b040 <_ZNSt8ios_base4InitC2Ev>
   0x000000000047ea9d <+13>:    lea    0xfce6c(%rip),%rdi        # 0x57b910 <_ZNSt8ios_base4InitD2Ev>
   0x000000000047eaa4 <+20>:    lea    0x148a9d(%rip),%rdx        # 0x5c7548 <__dso_handle>
   0x000000000047eaab <+27>:    lea    0x3c0c8e(%rip),%rsi        # 0x83f740 <_ZStL8__ioinit>
   0x000000000047eab2 <+34>:    callq  0x47a730 <__cxa_atexit@plt>
   0x000000000047eab7 <+39>:    lea    0x3c0b42(%rip),%rbx        # 0x83f600 <_ZN18SIMDCompressionLib19IntersectionFactory20intersection_schemesE>
=> 0x000000000047eabe <+46>:    vmovdqa 0x16671a(%rip),%xmm0        # 0x5e51e0
   0x000000000047eac6 <+54>:    vmovdqa 0x166722(%rip),%xmm1        # 0x5e51f0
   0x000000000047eace <+62>:    vmovdqa 0x16672a(%rip),%xmm2        # 0x5e5200
   0x000000000047ead6 <+70>:    vmovdqa %xmm0,0x3c0b62(%rip)        # 0x83f640 <_ZN18SIMDCompressionLibL12shuffle_maskE>
   0x000000000047eade <+78>:    vmovdqa 0x16672a(%rip),%xmm3        # 0x5e5210
   0x000000000047eae6 <+86>:    mov    %rbx,%rdi
   0x000000000047eae9 <+89>:    vmovdqa %xmm0,0x3c0b5f(%rip)        # 0x83f650 <_ZN18SIMDCompressionLibL12shuffle_maskE+16>
   0x000000000047eaf1 <+97>:    vmovdqa 0x166727(%rip),%xmm4        # 0x5e5220
   0x000000000047eaf9 <+105>:   vmovdqa %xmm1,0x3c0b5f(%rip)        # 0x83f660 <_ZN18SIMDCompressionLibL12shuffle_maskE+32>
   0x000000000047eb01 <+113>:   vmovdqa 0x166727(%rip),%xmm5        # 0x5e5230
   0x000000000047eb09 <+121>:   vmovdqa %xmm0,0x3c0b5f(%rip)        # 0x83f670 <_ZN18SIMDCompressionLibL12shuffle_maskE+48>
   0x000000000047eb11 <+129>:   vmovdqa 0x166727(%rip),%xmm6        # 0x5e5240
   0x000000000047eb19 <+137>:   vmovdqa %xmm2,0x3c0b5f(%rip)        # 0x83f680 <_ZN18SIMDCompressionLibL12shuffle_maskE+64>
   0x000000000047eb21 <+145>:   vmovdqa 0x166727(%rip),%xmm7        # 0x5e5250
   0x000000000047eb29 <+153>:   vmovdqa %xmm3,0x3c0b5f(%rip)        # 0x83f690 <_ZN18SIMDCompressionLibL12shuffle_maskE+80>
   0x000000000047eb31 <+161>:   vmovdqa 0x166727(%rip),%xmm8        # 0x5e5260
   0x000000000047eb39 <+169>:   vmovdqa %xmm4,0x3c0b5f(%rip)        # 0x83f6a0 <_ZN18SIMDCompressionLibL12shuffle_maskE+96>
   0x000000000047eb41 <+177>:   vmovdqa 0x166727(%rip),%xmm9        # 0x5e5270
   0x000000000047eb49 <+185>:   vmovdqa %xmm0,0x3c0b5f(%rip)        # 0x83f6b0 <_ZN18SIMDCompressionLibL12shuffle_maskE+112>
   0x000000000047eb51 <+193>:   vmovdqa 0x166727(%rip),%xmm10        # 0x5e5280
   0x000000000047eb59 <+201>:   vmovdqa %xmm5,0x3c0b5f(%rip)        # 0x83f6c0 <_ZN18SIMDCompressionLibL12shuffle_maskE+128>
   0x000000000047eb61 <+209>:   vmovdqa 0x166727(%rip),%xmm11        # 0x5e5290
   0x000000000047eb69 <+217>:   vmovdqa %xmm6,0x3c0b5f(%rip)        # 0x83f6d0 <_ZN18SIMDCompressionLibL12shuffle_maskE+144>
   0x000000000047eb71 <+225>:   vmovdqa %xmm7,0x3c0b67(%rip)        # 0x83f6e0 <_ZN18SIMDCompressionLibL12shuffle_maskE+160>
   0x000000000047eb79 <+233>:   vmovdqa %xmm8,0x3c0b6f(%rip)        # 0x83f6f0 <_ZN18SIMDCompressionLibL12shuffle_maskE+176>
   0x000000000047eb81 <+241>:   vmovdqa %xmm9,0x3c0b77(%rip)        # 0x83f700 <_ZN18SIMDCompressionLibL12shuffle_maskE+192>
   0x000000000047eb89 <+249>:   vmovdqa %xmm10,0x3c0b7f(%rip)        # 0x83f710 <_ZN18SIMDCompressionLibL12shuffle_maskE+208>
   0x000000000047eb91 <+257>:   vmovdqa %xmm11,0x3c0b87(%rip)        # 0x83f720 <_ZN18SIMDCompressionLibL12shuffle_maskE+224>
   0x000000000047eb99 <+265>:   vmovdqa %xmm0,0x3c0b8f(%rip)        # 0x83f730 <_ZN18SIMDCompressionLibL12shuffle_maskE+240>
   0x000000000047eba1 <+273>:   callq  0x524980 <_ZN18SIMDCompressionLib29initializeintersectionfactoryEv>
   0x000000000047eba6 <+278>:   mov    %rbx,%rsi
   0x000000000047eba9 <+281>:   lea    0xa5920(%rip),%rdi        # 0x5244d0 <_ZNSt3mapISsPFmPKjmS1_mPjESt4lessISsESaISt4pairIKSsS4_EEED2Ev>
   0x000000000047ebb0 <+288>:   pop    %rbx
   0x000000000047ebb1 <+289>:   lea    0x148990(%rip),%rdx        # 0x5c7548 <__dso_handle>
   0x000000000047ebb8 <+296>:   jmpq   0x47a730 <__cxa_atexit@plt>
End of assembler dump.
(gdb)

vmovdqa doesn't appear as an instruction in the binary that works, so maybe that is a good clue.

[rob-p]

Hrmm, OK, that does seem like a vectorized instruction (perhaps not available on all processors). Interestingly, it seems to be originating from the SIMDCompressionLib (whose code I thought was unused in the current codebase). Perhaps it's just loading the code that is causing the problem. Does the illegal instruction happen as soon as you run RapMap, or only during execution?

[roryk]

Hi Rob,

Right away, so just ./rapmap does it.

[roryk]

Processor info:

processor   : 0
vendor_id   : GenuineIntel
cpu family  : 6
model       : 44
model name  : Intel(R) Xeon(R) CPU           L5640  @ 2.27GHz
stepping    : 2
microcode   : 19
cpu MHz     : 2266.904
cache size  : 12288 KB
physical id : 0
siblings    : 12
core id     : 0
cpu cores   : 6
apicid      : 0
initial apicid  : 0
fpu     : yes
fpu_exception   : yes
cpuid level : 11
wp      : yes
flags       : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_ts
c arch_perfmon pebs bts rep_good xtopology nonstop_tsc aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 popcnt aes lahf_
lm arat epb dts tpr_shadow vnmi flexpriority ept vpid
bogomips    : 4533.80
clflush size    : 64
cache_alignment : 64
address sizes   : 40 bits physical, 48 bits virtual
power management:

[rob-p]

Does this work any better? RapMap-0.1.0_CentOS5.tar.gz

[roryk]

Same issue:

-bash-4.1$ gdb ./rapmap
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-75.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /groups/bcbio/singlecell-test/allon/rerun/foo/RapMap-0.1.0_CentOS5/bin/rapmap...done.
(gdb) run
Starting program: /groups/bcbio/singlecell-test/allon/rerun/foo/RapMap-0.1.0_CentOS5/bin/rapmap
[Thread debugging using libthread_db enabled]

Program received signal SIGILL, Illegal instruction.
__static_initialization_and_destruction_0 () at /RapMap/src/intersection.cpp:635
635 /RapMap/src/intersection.cpp: No such file or directory.
    in /RapMap/src/intersection.cpp
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.166.el6_7.7.x86_64 libgcc-4.4.7-11.el6.x86_64 libgomp-4.4.7-11.el6.x86_64
(gdb) disassemble
Dump of assembler code for function _GLOBAL__sub_I_intersection.cpp(void):
   0x0000000000472d20 <+0>: push   %rbx
   0x0000000000472d21 <+1>: lea    0x42b478(%rip),%rdi        # 0x89e1a0 <_ZStL8__ioinit>
   0x0000000000472d28 <+8>: callq  0x5bd7e0 <_ZNSt8ios_base4InitC2Ev>
   0x0000000000472d2d <+13>:    lea    0x14b37c(%rip),%rdi        # 0x5be0b0 <_ZNSt8ios_base4InitD2Ev>
   0x0000000000472d34 <+20>:    lea    0x196f8d(%rip),%rdx        # 0x609cc8 <__dso_handle>
   0x0000000000472d3b <+27>:    lea    0x42b45e(%rip),%rsi        # 0x89e1a0 <_ZStL8__ioinit>
   0x0000000000472d42 <+34>:    lea    0x42b317(%rip),%rbx        # 0x89e060 <_ZN18SIMDCompressionLib19IntersectionFactory20intersection_schemesE>
   0x0000000000472d49 <+41>:    callq  0x46e090 <__cxa_atexit@plt>
=> 0x0000000000472d4e <+46>:    vmovdqa 0x1c8b0a(%rip),%xmm0        # 0x63b860
   0x0000000000472d56 <+54>:    mov    %rbx,%rdi
   0x0000000000472d59 <+57>:    vmovdqa 0x1c8b0f(%rip),%xmm1        # 0x63b870
   0x0000000000472d61 <+65>:    vmovdqa 0x1c8b17(%rip),%xmm2        # 0x63b880
   0x0000000000472d69 <+73>:    vmovdqa %xmm0,0x42b32f(%rip)        # 0x89e0a0 <_ZN18SIMDCompressionLibL12shuffle_maskE>
   0x0000000000472d71 <+81>:    vmovdqa 0x1c8b17(%rip),%xmm3        # 0x63b890
   0x0000000000472d79 <+89>:    vmovdqa %xmm0,0x42b32f(%rip)        # 0x89e0b0 <_ZN18SIMDCompressionLibL12shuffle_maskE+16>
   0x0000000000472d81 <+97>:    vmovdqa 0x1c8b17(%rip),%xmm4        # 0x63b8a0
   0x0000000000472d89 <+105>:   vmovdqa %xmm1,0x42b32f(%rip)        # 0x89e0c0 <_ZN18SIMDCompressionLibL12shuffle_maskE+32>
   0x0000000000472d91 <+113>:   vmovdqa 0x1c8b17(%rip),%xmm5        # 0x63b8b0
   0x0000000000472d99 <+121>:   vmovdqa %xmm0,0x42b32f(%rip)        # 0x89e0d0 <_ZN18SIMDCompressionLibL12shuffle_maskE+48>
   0x0000000000472da1 <+129>:   vmovdqa 0x1c8b17(%rip),%xmm6        # 0x63b8c0
   0x0000000000472da9 <+137>:   vmovdqa %xmm2,0x42b32f(%rip)        # 0x89e0e0 <_ZN18SIMDCompressionLibL12shuffle_maskE+64>
   0x0000000000472db1 <+145>:   vmovdqa 0x1c8b17(%rip),%xmm7        # 0x63b8d0
   0x0000000000472db9 <+153>:   vmovdqa %xmm3,0x42b32f(%rip)        # 0x89e0f0 <_ZN18SIMDCompressionLibL12shuffle_maskE+80>
   0x0000000000472dc1 <+161>:   vmovdqa 0x1c8b17(%rip),%xmm8        # 0x63b8e0
   0x0000000000472dc9 <+169>:   vmovdqa %xmm4,0x42b32f(%rip)        # 0x89e100 <_ZN18SIMDCompressionLibL12shuffle_maskE+96>
   0x0000000000472dd1 <+177>:   vmovdqa 0x1c8b17(%rip),%xmm9        # 0x63b8f0
   0x0000000000472dd9 <+185>:   vmovdqa %xmm0,0x42b32f(%rip)        # 0x89e110 <_ZN18SIMDCompressionLibL12shuffle_maskE+112>
   0x0000000000472de1 <+193>:   vmovdqa 0x1c8b17(%rip),%xmm10        # 0x63b900
   0x0000000000472de9 <+201>:   vmovdqa %xmm5,0x42b32f(%rip)        # 0x89e120 <_ZN18SIMDCompressionLibL12shuffle_maskE+128>
   0x0000000000472df1 <+209>:   vmovdqa 0x1c8b17(%rip),%xmm11        # 0x63b910
   0x0000000000472df9 <+217>:   vmovdqa %xmm6,0x42b32f(%rip)        # 0x89e130 <_ZN18SIMDCompressionLibL12shuffle_maskE+144>
   0x0000000000472e01 <+225>:   vmovdqa %xmm7,0x42b337(%rip)        # 0x89e140 <_ZN18SIMDCompressionLibL12shuffle_maskE+160>
   0x0000000000472e09 <+233>:   vmovdqa %xmm8,0x42b33f(%rip)        # 0x89e150 <_ZN18SIMDCompressionLibL12shuffle_maskE+176>
   0x0000000000472e11 <+241>:   vmovdqa %xmm9,0x42b347(%rip)        # 0x89e160 <_ZN18SIMDCompressionLibL12shuffle_maskE+192>
   0x0000000000472e19 <+249>:   vmovdqa %xmm10,0x42b34f(%rip)        # 0x89e170 <_ZN18SIMDCompressionLibL12shuffle_maskE+208>
   0x0000000000472e21 <+257>:   vmovdqa %xmm11,0x42b357(%rip)        # 0x89e180 <_ZN18SIMDCompressionLibL12shuffle_maskE+224>
   0x0000000000472e29 <+265>:   vmovdqa %xmm0,0x42b35f(%rip)        # 0x89e190 <_ZN18SIMDCompressionLibL12shuffle_maskE+240>
   0x0000000000472e31 <+273>:   callq  0x5669b0 <SIMDCompressionLib::initializeintersectionfactory()>
   0x0000000000472e36 <+278>:   mov    %rbx,%rsi
   0x0000000000472e39 <+281>:   lea    0xf36c0(%rip),%rdi        # 0x566500 <std::map<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long (*)(unsigned int const*, unsigned long, unsigned int const*, unsigned long, unsigned int*), std::less<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, unsigned long (*)(unsigned int const*, unsigned long, unsigned int const*, unsigned long, unsigned int*)> > >::~map()>
   0x0000000000472e40 <+288>:   pop    %rbx
   0x0000000000472e41 <+289>:   lea    0x196e80(%rip),%rdx        # 0x609cc8 <__dso_handle>
   0x0000000000472e48 <+296>:   jmpq   0x46e090 <__cxa_atexit@plt>
End of assembler dump.

[rob-p]

hrmm; what the heck. I thought I removed all of the SIMDCompressionLib code from this one. This is very strange. I'll look harder.

[rob-p]

Ahh, I removed the headers but not the implementation file --- another binary is coming right up.

[rob-p]

Ok, how about this one? RapMap-0.1.0_CentOS5.tar.gz

[roryk]

Awesome, that totally fixed it.

[rob-p]

Great --- it was a SIMD instruction being compiled in from a library that I'm not actually using ;P.

[roryk]

Ah, spoke too soon. I ran it on the test fastq file from the other issue to try to close out that one too and it has another issue:

(gdb) bt
#0  rapidjson::GenericReader<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> >::ParseNumber<0u, rapidjson::GenericReadStream, rapidjson::GenericDocument<rapidjson::UTF8<> > >(rapidjson::GenericReadStream &, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> > &) (
    this=0x7fffffff93a0, stream=..., handler=...) at /RapMap/external/install/include/cereal/external/rapidjson/reader.h:608
#1  0x0000000000501843 in rapidjson::GenericReader<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> >::ParseObject<0u, rapidjson::GenericReadStream, rapidjson::GenericDocument<rapidjson::UTF8<> > >(rapidjson::GenericReadStream &, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> > &) (this=0x7fffffff93a0, stream=..., handler=...) at /RapMap/external/install/include/cereal/external/rapidjson/reader.h:295
#2  0x0000000000501843 in rapidjson::GenericReader<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> >::ParseObject<0u, rapidjson::GenericReadStream, rapidjson::GenericDocument<rapidjson::UTF8<> > >(rapidjson::GenericReadStream &, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> > &) (this=0x7fffffff93a0, stream=..., handler=...) at /RapMap/external/install/include/cereal/external/rapidjson/reader.h:295
#3  0x00000000005022f4 in rapidjson::GenericReader<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> >::Parse<0u, rapidjson::GenericReadStream, rapidjson::GenericDocument<rapidjson::UTF8<> > >(rapidjson::GenericReadStream &, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> > &) (this=0x7fffffff93a0, stream=..., handler=...) at /RapMap/external/install/include/cereal/external/rapidjson/reader.h:248
#4  0x00000000005024e6 in rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> >::ParseStream<0u, rapidjson::GenericReadStream> (this=0x7fffffffd2b8, stream=...) at /RapMap/external/install/include/cereal/external/rapidjson/document.h:712
#5  0x00000000004f08ae in JSONInputArchive (argc=Unhandled dwarf expression opcode 0xf3
) at /RapMap/external/install/include/cereal/archives/json.hpp:412
#6  rapMapMap (argc=Unhandled dwarf expression opcode 0xf3
) at /RapMap/src/RapMapMapper.cpp:984
#7  0x000000000046f737 in main (argc=6, argv=0x7fffffffdd28) at /RapMap/src/RapMap.cpp:65

[rob-p]

Hrmm, this unfortunately seems like a different issue (related to the code to read and write the JSON file?). I doubt that's using any advanced instructions. I wonder if this is an issue with the compiler / linker / libc used to create the binary.

[roryk]

For what it's worth, here is the instruction it fails on:

son::GenericReadStream, rapidjson::GenericDocument<rapidjson::UTF8<> > >(rapidjson::GenericReadStream &, rapidjson::GenericDocument<rapidjson::UTF8<char>, rapidjson::MemoryPoolAllocator<rapidjson::CrtAllocator> > &)+2703>
   0x00000000004feae8 <+232>:   mov    0x20(%rsp),%rdx
   0x00000000004feaed <+237>:   lea    0x12be6c(%rip),%r14        # 0x62a960
   0x00000000004feaf4 <+244>:   mov    0x18(%rsp),%r15
   0x00000000004feaf9 <+249>:   mov    (%rdx),%rdi
   0x00000000004feafc <+252>:   mov    %r14,0xf8(%r15)
   0x00000000004feb03 <+259>:   callq  0x5c50d0 <_ZNSi5tellgEv>
   0x00000000004feb08 <+264>:   lea    0x30(%r15),%rdi
   0x00000000004feb0c <+268>:   mov    $0x1,%esi
   0x00000000004feb11 <+273>:   cltq
   0x00000000004feb13 <+275>:   mov    %rax,0x100(%r15)
   0x00000000004feb1a <+282>:   callq  0x46e260 <longjmp@plt>
   0x00000000004feb1f <+287>:   nop
=> 0x00000000004feb20 <+288>:   vxorpd %xmm14,%xmm14,%xmm14

[rob-p]

hrmm, looks like it might be an instruction not available on all CPUs (http://stackoverflow.com/questions/26942952/difference-between-the-avx-instructions-vxorpd-and-vpxor)?

[roryk]

Maybe passing along -mno-avx would do the trick?

[rob-p]

I'll try that and provide a binary when I get home.

[roryk]

Thanks Rob, I can paste in the CMakeCache that has the options that get set for building on the picky machine if that would help.

[rob-p]

Ok, I built this guy w/o -march=native and using the suggested -mno-avx. If this works, I have to figure a good way to keep these flags in the automated builds. RapMap-0.1.0_CentOS5.tar.gz

[roryk]

-bash-4.1$ ./RapMap-0.1.0_CentOS5/bin/rapmap pseudomap -r rapmap-test.fq -i allon/rerun/transcriptome/hg19_ERCC92
RapMap Mapper
terminate called after throwing an instance of 'cereal::Exception'
  what():  JSON Parsing failed - provided NVP not found
Aborted

[rob-p]

Hrmm, but that's not a binary related error. Are you doing the mapping on an index built with an older version of rapmap? The new one (this binary) requires a name-value-pair (NVP) saying if the hash is a dense hash or a perfect hash.

[rob-p]

Also, I'd tend to recommend quasimap and quasiindex over pseudomap and pseudoindex --- it tends to be more accurate :).

[roryk]

Hi Rob,

Awesome, swapped over to using quasi*, more accurate sounds good to me. The index built, mapping worked and the truncating file at empty reads issue was fixed.

[rob-p]

Hi Rory,

Glad to hear we finally got this issue resolved! The pseudo commands should still work, so hopefully the issue you were seeing before was just a version mis-match issue. I'll take a look at how I can automate the proper CXXFLAGS for when I cut binary releases. Thanks for all of your help reporting this issue, and helping me track it down and fix it!

[roryk]

Thanks. Sorry to ask such a noob question but for fixing up the bioconda formula, is there some way I can add the flags myself? Does cmake respect environment variables? If I do export CXXFLAGS=-mno-avx before running cmake does that get pulled in or no.

[rob-p]

No need to apologize! If you want to add CXX flags you can pass them to cmake during the cmake phase of the build. That is:

$ cmake -DCMAKE_CXX_FLAGS=-mno-avx <OTHER OPTIONS> ..
$ make
$ make install

Of course, this solves the issue of adding the -mno-avx flags. However, it doesn't remove -march=native from the set of CXXFLAGS. I'm not sure if that is necessary to avoid the binary error or not. One option is to add a custom flag to the CMakeLists.txt file to disable this flag when passed in. I'm looking into the best way to do that.

[rob-p]

Alright, I just pushed a change to the quasi-mph branch. If you run CMake with the following command:

$ cmake -DNO_NATIVE_ARCH=TRUE -DCMAKE_CXX_FLAGS=-mno-avx ..

This should both disable -march=native and pass along the -mno-avx switch.

[roryk]

Thanks, I added those flags to the bioconda formula, built it in the docker image and moved it over to the picky machine and it works fine. Any chance at a new release so I can fix the formula proper instead of pulling from a branch?

[rob-p]

Sure; I can probably get a tagged release out for you tomorrow. Thanks for all the help (and for making RapMap available via bioconda!).

[roryk]

Sweet, sounds great. Thanks a lot for taking the time!

[rob-p]

Hi @roryk --- I've cut a v0.2.0 release. You should be able to pull a source tarball from there directly.

[roryk]

Thanks Rob, bumped bioconda to use that version.