`compile_shlib` fails with `_gpu_gc_pool_alloc` linker error

[YingboMa]

using RecursiveFactorization, StrideArraysCore, StaticCompiler, Static, LinearAlgebra
function _linsolve!(A::Ptr{T}, b::Ptr{T}, ::Val{N}) where {N, T}
    piv = Ref{NTuple{N, Int}}()
    D = static(N)
    GC.@preserve piv begin
        F = RecursiveFactorization.lu!(PtrArray(A, (D, D)),
            PtrArray(Base.unsafe_convert(Ptr{Int}, piv), (D,)))
        ldiv!(F, PtrArray(b, (D,)))
    end
    return b
end
linsolve!(n::Val{N}) where N = (A, b) -> _linsolve!(A, b, n)

N = 10
path = compile_shlib(linsolve!(Val(N)), (Ptr{Float64}, Ptr{Float64}), "./", filename="linsolve$N")

julia> path = compile_shlib(linsolve!(Val(N)), (Ptr{Float64}, Ptr{Float64}), "./", filename="linsolve$N")
ld: Undefined symbols:
  _ijl_throw, referenced from:
      _julia_generic_trimatdiv__1404 in linsolve10.o
      _julia_generic_trimatdiv__1404 in linsolve10.o
      _julia_generic_trimatdiv__1404 in linsolve10.o
      _julia_generic_trimatdiv__1404 in linsolve10.o
      _julia__linsolve__1399 in linsolve10.o
      _gpu_gc_pool_alloc in linsolve10.o
clang: error: linker command failed with exit code 1 (use -v to see invocation)
ERROR: failed process: Process(`cc -shared ./linsolve10.o -o ./linsolve10.dylib`, ProcessExited(1)) [1]

Stacktrace:
 [1] pipeline_error
   @ Base ./process.jl:565 [inlined]
 [2] run(::Cmd; wait::Bool)
   @ Base ./process.jl:480
 [3] run
   @ ./process.jl:477 [inlined]
 [4] generate_shlib(funcs::Tuple{Tuple{…}}, external::Bool, path::String, filename::String; demangle::Bool, cflags::Cmd, kwargs::@Kwargs{})
   @ StaticCompiler ~/.julia/packages/StaticCompiler/LMT2M/src/StaticCompiler.jl:563
 [5] generate_shlib
   @ ~/.julia/packages/StaticCompiler/LMT2M/src/StaticCompiler.jl:551 [inlined]
 [6] compile_shlib(funcs::Tuple{Tuple{var"#1#2"{…}, Tuple{…}}}, path::String; filename::String, demangle::Bool, cflags::Cmd, kwargs::@Kwargs{})
   @ StaticCompiler ~/.julia/packages/StaticCompiler/LMT2M/src/StaticCompiler.jl:352
 [7] compile_shlib(f::Function, types::Tuple{DataType, DataType}, path::String, name::String; filename::String, kwargs::@Kwargs{})
   @ StaticCompiler ~/.julia/packages/StaticCompiler/LMT2M/src/StaticCompiler.jl:332
 [8] top-level scope
   @ REPL[5]:1
Some type information was truncated. Use `show(err)` to see complete types.

julia> versioninfo()
Julia Version 1.10.0-rc1
Commit 5aaa9485436 (2023-11-03 07:44 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: macOS (arm64-apple-darwin22.4.0)
  CPU: 12 × Apple M2 Max
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-15.0.7 (ORCJIT, apple-m1)
  Threads: 1 on 8 virtual cores

[chriselrod]

It works for me on Intel Linux

julia> path = compile_shlib(linsolve!(Val(N)), (Ptr{Float64}, Ptr{Float64}), "./", filename="linsolve$N")
"/home/chriselrod/Documents/languages/juliaold/linsolve10.so"

julia> versioninfo()
Julia Version 1.10.0-rc1
Commit 5aaa948543 (2023-11-03 07:44 UTC)
Build Info:

    Note: This is an unofficial build, please report bugs to the project
    responsible for this build and not to the Julia project unless you can
    reproduce the issue using official builds available at https://julialang.org/downloads

Platform Info:
  OS: Linux (x86_64-generic-linux)
  CPU: 28 × Intel(R) Core(TM) i9-9940X CPU @ 3.30GHz

I tried on both 1.9.3 and 1.10.0-rc1.

[MasonProtter]

Works for me too.

julia> versioninfo()
Julia Version 1.10.0-rc1
Commit 5aaa948543 (2023-11-03 07:44 UTC)
Build Info:

    Note: This is an unofficial build, please report bugs to the project
    responsible for this build and not to the Julia project unless you can
    reproduce the issue using official builds available at https://julialang.org/downloads

Platform Info:
  OS: Linux (x86_64-pc-linux-gnu)
  CPU: 12 × AMD Ryzen 5 5600X 6-Core Processor
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-15.0.7 (ORCJIT, znver3)
  Threads: 8 on 12 virtual cores
Environment:
  JULIA_NUM_THREADS = 6

[chriselrod]

julia> path = compile_shlib(linsolve!(Val(N)), (Ptr{Float64}, Ptr{Float64}), "./", filename="linsolve$N")
"/home/chriselrod/Documents/languages/juliadev/linsolve10.so"

julia> versioninfo()
Julia Version 1.10.0-rc1
Commit 5aaa948543* (2023-11-03 07:44 UTC)
Build Info:

    Note: This is an unofficial build, please report bugs to the project
    responsible for this build and not to the Julia project unless you can
    reproduce the issue using official builds available at https://julialang.org/downloads

Platform Info:
  OS: Linux (aarch64-unknown-linux-gnu)
  CPU: 8 × unknown
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-15.0.7 (ORCJIT, apple-m1)

Also works for me on AArch64-Linux.

[gbaraldi]

So the difference here is that the linker is a lot more picky in macos than in linux, but if you try to link this .so to a c file it complains. It probably doesn't complain if you load it from julia though.

[YingboMa]

@gbaraldi pointed to me that singular exception is the culprit. For posterity, the following works.

using RecursiveFactorization, StrideArraysCore, Static, LinearAlgebra
using StaticCompiler
function myldiv!(A::LU, B::AbstractVecOrMat)
    LinearAlgebra._apply_ipiv_rows!(A, B)
    myldiv!(UpperTriangular(A.factors), myldiv!(UnitLowerTriangular(A.factors), B))
end
function myldiv!(A::UpperTriangular, b::AbstractVector, x::AbstractVector = b)
    n = size(A, 2)
    @inbounds for j in n:-1:1
        xj = x[j] = A.data[j,j] \ b[j]
        for i in j-1:-1:1
            b[i] -= A.data[i,j] * xj
        end
    end
    x
end
function myldiv!(A::UnitLowerTriangular, b::AbstractVector, x::AbstractVector = b)
    n = size(A, 2)
    @inbounds for j in 1:n
        xj = x[j] = b[j]
        for i in j+1:n
            b[i] -= A.data[i,j] * xj
        end
    end
    x
end
function _linsolve!(A::Ptr{T}, b::Ptr{T}, ::Val{N}) where {N, T}
    piv = Ref{NTuple{N, Int}}()
    D = static(N)
    GC.@preserve piv begin
        F = RecursiveFactorization.lu!(PtrArray(A, (D, D)),
            PtrArray(Base.unsafe_convert(Ptr{Int}, piv), (D,)),
            check = false)
        myldiv!(F, PtrArray(b, (D,)))
    end
    return b
end
function linsolve(n::Val{N}) where N
    function linsolve!(A, b)
        _linsolve!(A, b, n)
    end
end

N = 10
f = linsolve(Val(N))
A = rand(N, N); b = rand(N)
x = A\b; f(pointer(A), pointer(b))
path = compile_shlib(f, (Ptr{Float64}, Ptr{Float64}), "./", filename="linsolve$N")

[MasonProtter]

Does the old version work for you if you do

@device_override @noinline LinearAlgebra.SingularException(x) =
    @print_and_throw c"SingularException"

?

[gbaraldi]

So the throw override isn't quite correct in inference

[gbaraldi]

And we need to eliminate the call to throw as well. It's annoying that this code doesn't have the error outlined

[MasonProtter]

yeah I had hoped that if you stuck the override in, then the call to throw could get eliminated, but I guess that wasn't realistic.

[gbaraldi]

I'll try something that only works in master, but maybe it can work. But basically we need to teach inference that exit returns an Union{}

[MasonProtter]

Oh is that all that's missing? It should then work to just do

@device_override @noinline LinearAlgebra.SingularException(x) =
    @print_and_throw(c"SingularException")::Union{}

right?

[MasonProtter]

julia> g(x) = throw(f(1))
g (generic function with 1 method)

julia> @noinline f(x) = @print_and_throw(c"hi")::Union{}
f (generic function with 1 method)

julia> code_typed(g, Tuple{Int})
1-element Vector{Any}:
 CodeInfo(
1 ─     invoke Main.f(1::Int64)::Union{}
└──     unreachable
) => Union{}