Release Mode: illegal hardware instruction (core dumped)

[ZuseZ4]

Even simple examples like sin(*x) segfault in release mode. Investigating.

[compiler/rustc_codegen_llvm/src/llvm/ffi.rs:936:5] &fnc = (ptr:; Function Attrs: mustprogress nofree noinline nosync nounwind nonlazybind sanitize_hwaddress willreturn memory(argmem: read) uwtable
define internal noundef double @_ZN2ad3foo17h962107b3e3405598E(ptr noalias nocapture noundef readonly align 8 dereferenceable(8) %0) unnamed_addr #5 {
  %2 = load double, ptr %0, align 8, !noundef !6
  %3 = tail call double @llvm.sin.f64(double %2)
  ret double %3
}
)
[compiler/rustc_codegen_llvm/src/llvm/ffi.rs:958:5] &primary_ret = true
[compiler/rustc_codegen_llvm/src/llvm/ffi.rs:959:5] &ret_activity = DFT_OUT_DIFF
[compiler/rustc_codegen_llvm/src/llvm/ffi.rs:960:5] &input_activity = [
    DFT_DUP_ARG,
]
[compiler/rustc_codegen_llvm/src/llvm/ffi.rs:983:5] &res = (ptr:; Function Attrs: mustprogress nofree noinline nosync nounwind nonlazybind sanitize_hwaddress willreturn memory(argmem: readwrite) uwtable
define internal { double } @diffe_ZN2ad3foo17h962107b3e3405598E(ptr noalias nocapture noundef readonly align 8 dereferenceable(8) %0, ptr nocapture align 8 %1, double %2) unnamed_addr #111 {
  %4 = load double, ptr %0, align 8, !alias.scope !41888, !noalias !41891, !noundef !6
  %5 = tail call double @llvm.sin.f64(double %4) #112
  %6 = call fast double @llvm.cos.f64(double %4)
  %7 = fmul fast double %2, %6
  %8 = load double, ptr %1, align 8, !alias.scope !41891, !noalias !41888
  %9 = fadd fast double %8, %7
  store double %9, ptr %1, align 8, !alias.scope !41891, !noalias !41888
  %10 = insertvalue { double } undef, double %5, 0
  ret { double } %10
}
)
[compiler/rustc_codegen_llvm/src/back/write.rs:696:5] &outer_fnc = (ptr:define double @_ZN2ad3bar17h041e2a630fee488fE.852(ptr %0, ptr %1, double %2) {
  %4 = call { double } @inner__ZN2ad3bar17h041e2a630fee488fE(ptr %0, ptr %1, double %2)
  %5 = extractvalue { double } %4, 0
  ret double %5
}
)

Running it results in crashing when calling the outer fnc.

warning: `ad` (bin "ad") generated 3 warnings (run `cargo fix --bin "ad"` to apply 1 suggestion)
    Finished release [optimized] target(s) in 0.01s
     Running `target/release/ad`
primal result: 5569.144408652917
[1]    316614 illegal hardware instruction (core dumped)  cargo +enzyme run --release

[ZuseZ4]

cc @wsmoses

[jedbrown]

$ RUSTFLAGS='-C opt-level=1' cargo +enzyme b
$ gdb -ex r target/debug/enzyme-sandbox
No symbol table is loaded.  Use the "file" command.
Breakpoint 1 (PetscError) pending.
No symbol table is loaded.  Use the "file" command.
Breakpoint 2 (CeedError) pending.
No symbol table is loaded.  Use the "file" command.
Breakpoint 3 (rust_panic) pending.
Reading symbols from target/debug/enzyme-sandbox...
warning: Missing auto-load script at offset 0 in section .debug_gdb_scripts
of file /home/jed/lang/rust/enzyme-sandbox/target/debug/enzyme-sandbox.
Use `info auto-load python-scripts [REGEXP]' to list them.
Starting program: /home/jed/lang/rust/enzyme-sandbox/target/debug/enzyme-sandbox
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/usr/lib/libthread_db.so.1".
9

Program received signal SIGILL, Illegal instruction.
enzyme_sandbox::main () at src/main.rs:51
51          println!("df_dx: {:?}", df_dx);
(gdb) bt full
#0  enzyme_sandbox::main () at src/main.rs:51
        df_dx = 6
        output = 9
        x = 3
(gdb) disas
Dump of assembler code for function _ZN14enzyme_sandbox4main17h0c99a76a33bbd9c0E:
   0x0000555555561640 <+0>:     push   rbx
   0x0000555555561641 <+1>:     sub    rsp,0x50
   0x0000555555561645 <+5>:     movabs rax,0x4008000000000000
   0x000055555556164f <+15>:    mov    QWORD PTR [rsp],rax
   0x0000555555561653 <+19>:    mov    rbx,rsp
   0x0000555555561656 <+22>:    mov    rdi,rbx
   0x0000555555561659 <+25>:    call   0x555555561610 <_ZN14enzyme_sandbox6square17h04bbfe0b65328bd3E>
   0x000055555556165e <+30>:    movsd  QWORD PTR [rsp+0x8],xmm0
   0x0000555555561664 <+36>:    lea    rax,[rsp+0x8]
   0x0000555555561669 <+41>:    mov    QWORD PTR [rsp+0x10],rax
   0x000055555556166e <+46>:    lea    rax,[rip+0x86bb]        # 0x555555569d30 <_ZN4core3fmt5float52_$LT$impl$u20$core..fmt..Display$u20$for$u20$f64$GT$3fmt17h07ca31a04ff0b175E>
   0x0000555555561675 <+53>:    mov    QWORD PTR [rsp+0x18],rax
   0x000055555556167a <+58>:    lea    rax,[rip+0x6a39f]        # 0x5555555cba20
   0x0000555555561681 <+65>:    mov    QWORD PTR [rsp+0x20],rax
   0x0000555555561686 <+70>:    mov    QWORD PTR [rsp+0x28],0x2
   0x000055555556168f <+79>:    mov    QWORD PTR [rsp+0x40],0x0
   0x0000555555561698 <+88>:    lea    rax,[rsp+0x10]
   0x000055555556169d <+93>:    mov    QWORD PTR [rsp+0x30],rax
   0x00005555555616a2 <+98>:    mov    QWORD PTR [rsp+0x38],0x1
   0x00005555555616ab <+107>:   lea    rdi,[rsp+0x20]
   0x00005555555616b0 <+112>:   call   0x55555559e0b0 <_ZN3std2io5stdio6_print17h8b36bf14c1d7d396E>
   0x00005555555616b5 <+117>:   mov    QWORD PTR [rsp+0x20],0x0
   0x00005555555616be <+126>:   lea    rsi,[rsp+0x20]
   0x00005555555616c3 <+131>:   movsd  xmm0,QWORD PTR [rip+0x4d935]        # 0x5555555af000
   0x00005555555616cb <+139>:   mov    rdi,rbx
   0x00005555555616ce <+142>:   call   0x5555555aeea0 <_ZN14enzyme_sandbox8d_square17h9d5b8e883243df76E.1248>
=> 0x00005555555616d3 <+147>:   ud2
(gdb) l
46          let output = square(&x);
47          println!("{output}");
48
49          let mut df_dx = 0.0;
50          d_square(&x, &mut df_dx, 1.0);
51          println!("df_dx: {:?}", df_dx);
52      }

[ZuseZ4]

@wsmoses Does any of the llvm attributes look suspicious to you? Remember that we strip the HWSanitizier later. Other than that we re-run llvm opts. I don't assign attributes to wrappers yet.

[wsmoses]

Nothing jumps out, but this looks like an issue in the shim Rust creates. Maybe looking at LLVM IR may help?

[ZuseZ4]

Yep. mod.ll.txt modDbg.ll.txt

In Release mod it cut's out everything after the call to my diffe wrapper and replaces it by unreachable, which is obviously wrong. The wrapper I have had a noreturn attr, probably because the original placeholder (before I replaced it's body) ended with a infinite loop {} statement to match arbitrary Rust return types.