Open avik-pal opened 1 month ago
Seems like it could stem from
function sum_loss(x)
return sum(abs2, x)
end
begin
x = cu(rand(Float32, 10, 10))
dx = zero(x)
Enzyme.autodiff(Reverse, sum_loss, Active, Duplicated(x, dx))
end
which is documented as non-functional, but how can we bypass it? We could write in the pullback form which prevents the reduction and that might work.
Apparently the differentiation of CUmemPoolProps_st
@avik-pal with all of these issues can you upload the full error log?
Added both the stacktraces with CUDA, cuDNN and Enzyme master
Oh cool so now with master your last one (https://github.com/EnzymeAD/Enzyme.jl/issues/1442#issuecomment-2108083820) now successfully differentiates and hits a GPUCompiler / LLVM.jl / LLJIT related question that I may need some assist/explanation from @vchuravy / @maleadt
So quick tldr. The original code we AD has say a cublasgemm inside it. We take that code via gpucompiler and also inject a cudamemset call to it. then send it to our custom jit infra to deal compile/run (in 1.10+ thats now the lljit stuff @gbaraldi did)
Ignore the nullptr stuff for now.
The JIT now complains it can't find cuMemsetD8
(we specificallt don't call that we call cudaMemset, but presumably somewhere something says that should be called instead maybe or maybe cuda.jl itself calls it). Either way somehow we're not loading the symbols cuda needs (either from the ones we added and/or the ones from the original cuda program).
Any thoughts?
Basically as soon as we fix this we seem to successfully differentiate through cublasgemm (since at least generating the llvm module of the derivative has now succeeded!)
In these cases Enzyme clearly is not the originator of all these symbols:
ERROR: LoadError: LLVM error: Failed to materialize symbols: { (JuliaExternal, { libname_cuCtxSetCurrent_40759, libname_cuMemHostAlloc_38650, libname_cuDeviceCanAccessPeer_41778, libname_cuCtxSetCurrent_40231, libname_cuStreamSynchronize_40575, libname_cuCtxSetCurrent_41988, libname_cuStreamGetCaptureInfo_41984, libname_cuCtxGetId_39658, libname_cublasGetProperty_39361, libname_cuMemPoolCreate_40279, libname_cuCtxPopCurrent_v2_40732, libname_cuCtxSetCurrent_41463, libname_cuMemAllocFromPoolAsync_38536, libname_cuMemPoolCreate_41596, libname_cuStreamQuery_41973, libname_cuCtxGetId_37318, libname_cuCtxPushCurrent_v2_40729, libname_cuMemPoolTrimTo_39311, libname_cuMemPoolSetAttribute_40353, libname_cublasSsymm_v2_64_39307, libname_cuDeviceGetAttribute_40201, libname_cuMemPoolCreate_40364, libname_cudaRuntimeGetVersion_39775, libname_cuCtxGetCurrent_40830, libname_cuMemAlloc_v2_38538, libname_cuMemsetD32Async_39068, libname_cuMemsetD32Async_39556, libname_cuCtxSetCurrent_40756, libname_cuMemPoolTrimTo_38488, libname_cuDeviceGet_41449, libname_cuStreamQuery_40740, libname_cuStreamGetCaptureInfo_40752, libname_cuStreamSynchronize_37225, libname_cuDeviceGetName_41015, libname_cuCtxSynchronize_38500, libname_cuCtxGetDevice_41967, libname_cuStreamCreate_41741, libname_cuMemGetInfo_v2_38102, libname_cuMemPoolGetAttribute_39590, libname_cuCtxGetCurrent_37272, libname_cuCtxGetApiVersion_41478, libname_cuCtxGetId_41476, libname_cuDeviceSetMemPool_38018, libname_cuLaunchKernel_38608, libname_cuOccupancyMaxPotentialBlockSize_38792, libname_cublasSgemm_v2_64_39435, libname_cuStreamSynchronize_37220, libname_cuMemGetInfo_v2_40442, libname_cuStreamSynchronize_41807, libname_cuDeviceGet_37877, libname_cuMemAlloc_v2_38530, libname_cuMemcpyDtoHAsync_v2_38437, libname_cuDevicePrimaryCtxRetain_37310, libname_cuDeviceCanAccessPeer_38375, libname_cuCtxSynchronize_38494, libname_cublasGetProperty_39352, libname_cuCtxGetDevice_38330, libname_cuPointerGetAttribute_38405, libname_cuStreamCreate_38169, libname_cuStreamCreate_40509, libname_cuStreamSynchronize_40580, libname_cuCtxSetCurrent_41991, libname_cuMemPoolTrimTo_39526, libname_cuMemPoolCreate_38024, libname_cuDeviceGet_40217, libname_cuDeviceGetAttribute_41433, libname_cuCtxSetCurrent_38409, libname_cuCtxGetCurrent_40285, libname_cuMemPoolTrimTo_39511, libname_cuMemPoolTrimTo_39395, libname_cuStreamQuery_41972, libname_cuCtxGetCurrent_37945, libname_cublasSscal_v2_39494, libname_cuMemPoolTrimTo_39462, libname_cuDeviceGetName_37443, libname_cuDeviceGetCount_39613, libname_cublasSgemm_v2_39442, libname_cuCtxSetCurrent_37891, libname_cuMemAllocAsync_38528, libname_cuMemPoolSetAttribute_38013, libname_cuCtxPushCurrent_v2_41961, libname_cuMemcpyDtoDAsync_v2_38454, libname_cuMemcpyHtoDAsync_v2_38348, libname_cuDeviceGetCount_37287, libname_cublasSetStream_v2_39406, libname_cuMemsetD32Async_40788, libname_cuStreamQuery_38336, libname_cuMemGetInfo_v2_41674, libname_cuMemPoolSetAccess_41767, libname_cuStreamSynchronize_41812, libname_cuMemPoolTrimTo_39365, libname_cuCtxGetDevice_40735, libname_cuCtxSetCurrent_38412, libname_cuStreamGetCaptureInfo_38400, libname_cuDeviceGetAttribute_37861, libname_cuStreamQuery_38335, libname_cuDeviceSetMemPool_40358, libname_cuMemPoolTrimTo_39480, libname_cuMemPoolSetAccess_38364, libname_cuCtxGetId_40890, libname_cublasSscal_v2_64_39523, libname_cudaRuntimeGetVersion_41007, libname_cuCtxGetId_37904, libname_cuMemPoolGetAttribute_37264, libname_cuDeviceGetCount_40845, libname_cublasSetMathMode_39329, libname_cuCtxGetCurrent_41517, libname_cuDeviceGetName_39783, libname_cuDevicePrimaryCtxRetain_40882, libname_cublasSsymm_v2_39283, libname_cublasSgemm_v2_39457, libname_cublasSscal_v2_39507, diffejulia_loss_function_37136wrap, libname_cublasSsymm_v2_64_39261, libname_cuMemPoolCreate_37939, libname_cuCtxPopCurrent_v2_38327, libname_cuMemPoolSetAccess_40535, libname_cuMemPoolCreate_41511, libname_cudaRuntimeGetVersion_37435, libname_cublasSscal_v2_64_39490, libname_cuMemPoolGetAttribute_40822, libname_cuMemPoolSetAttribute_41585, libname_cuMemPoolTrimTo_39288, libname_cuStreamQuery_40741, libname_cublasSetStream_v2_39392, libname_cuCtxGetApiVersion_40246, libname_cublasSetStream_v2_39382, libname_cuCtxSetCurrent_40832, libname_cuCtxGetId_40244, libname_cuCtxSetCurrent_39600, libname_cuDeviceCanAccessPeer_40546, libname_cuMemcpyPeerAsync_38458, libname_cuCtxPopCurrent_v2_41964, libname_cuCtxGetApiVersion_37906, libname_cublasSetMathMode_39340, libname_cuCtxGetCurrent_39598, libname_cublasSsymm_v2_39268, libname_cublasSgemm_v2_64_39476, libname_cuCtxSetCurrent_37274, libname_cuLaunchCooperativeKernel_38605, libname_cuDevicePrimaryCtxRetain_39650, libname_cuDeviceSetMemPool_41590, libname_cuMemPoolTrimTo_39343, libname_cuCtxPushCurrent_v2_38324 }) }
Fails with
and a very long LLVM dump.
Details
JIT session error: Symbols not found: [ cuMemsetD8 ] JIT session error: Symbols not found: [ cuMemsetD8 ] JIT session error: Symbols not found: [ cuMemsetD8 ] JIT session error: Symbols not found: [ cuMemsetD8 ] JIT session error: Symbols not found: [ cuMemsetD8 ] JIT session error: Symbols not found: [ cuMemsetD8 ] ┌ Warning: The Pkg REPL mode is intended for interactive use only, and should not be used from scripts. It is recommended to use the functional API instead. └ @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.3+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:382 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ┌ Error: Found null pointer at │ │ Stacktrace: │ [1] get_repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:63 │ [2] repl_frontend_task │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:56 │ [3] default_scalar_indexing │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:75 │ [4] assertscalar │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:105 │ [5] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:48 │ [6] scalar_getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:34 │ [7] _getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:17 │ [8] getindex │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/indexing.jl:15 │ [9] macro expansion │ @ ~/.julia/packages/GPUArraysCore/GMsgk/src/GPUArraysCore.jl:210 │ [10] #_mapreduce#42 │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:71 │ [11] _mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:33 │ [12] mapreduce │ @ ~/.julia/packages/GPUArrays/OKkAu/src/host/mapreduce.jl:28 │ [13] _sum │ @ ./reducedim.jl:1015 │ [14] sum │ @ ./reducedim.jl:1011 │ [15] loss_function │ @ /mnt/research/ongoing/lux/enzyme_cuda.jl:9 │ arg = %active_repl.checked = load atomic {} addrspace(10)*, {} addrspace(10)** inttoptr (i64 138527901362528 to {} addrspace(10)**) unordered, align 8, !dbg !439, !tbaa !440, !alias.scope !431, !noalias !434 └ @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/absint.jl:121 ERROR: LoadError: LLVM error: Failed to materialize symbols: { (JuliaExternal, { libname_cuCtxSetCurrent_40759, libname_cuMemHostAlloc_38650, libname_cuDeviceCanAccessPeer_41778, libname_cuCtxSetCurrent_40231, libname_cuStreamSynchronize_40575, libname_cuCtxSetCurrent_41988, libname_cuStreamGetCaptureInfo_41984, libname_cuCtxGetId_39658, libname_cublasGetProperty_39361, libname_cuMemPoolCreate_40279, libname_cuCtxPopCurrent_v2_40732, libname_cuCtxSetCurrent_41463, libname_cuMemAllocFromPoolAsync_38536, libname_cuMemPoolCreate_41596, libname_cuStreamQuery_41973, libname_cuCtxGetId_37318, libname_cuCtxPushCurrent_v2_40729, libname_cuMemPoolTrimTo_39311, libname_cuMemPoolSetAttribute_40353, libname_cublasSsymm_v2_64_39307, libname_cuDeviceGetAttribute_40201, libname_cuMemPoolCreate_40364, libname_cudaRuntimeGetVersion_39775, libname_cuCtxGetCurrent_40830, libname_cuMemAlloc_v2_38538, libname_cuMemsetD32Async_39068, libname_cuMemsetD32Async_39556, libname_cuCtxSetCurrent_40756, libname_cuMemPoolTrimTo_38488, libname_cuDeviceGet_41449, libname_cuStreamQuery_40740, libname_cuStreamGetCaptureInfo_40752, libname_cuStreamSynchronize_37225, libname_cuDeviceGetName_41015, libname_cuCtxSynchronize_38500, libname_cuCtxGetDevice_41967, libname_cuStreamCreate_41741, libname_cuMemGetInfo_v2_38102, libname_cuMemPoolGetAttribute_39590, libname_cuCtxGetCurrent_37272, libname_cuCtxGetApiVersion_41478, libname_cuCtxGetId_41476, libname_cuDeviceSetMemPool_38018, libname_cuLaunchKernel_38608, libname_cuOccupancyMaxPotentialBlockSize_38792, libname_cublasSgemm_v2_64_39435, libname_cuStreamSynchronize_37220, libname_cuMemGetInfo_v2_40442, libname_cuStreamSynchronize_41807, libname_cuDeviceGet_37877, libname_cuMemAlloc_v2_38530, libname_cuMemcpyDtoHAsync_v2_38437, libname_cuDevicePrimaryCtxRetain_37310, libname_cuDeviceCanAccessPeer_38375, libname_cuCtxSynchronize_38494, libname_cublasGetProperty_39352, libname_cuCtxGetDevice_38330, libname_cuPointerGetAttribute_38405, libname_cuStreamCreate_38169, libname_cuStreamCreate_40509, libname_cuStreamSynchronize_40580, libname_cuCtxSetCurrent_41991, libname_cuMemPoolTrimTo_39526, libname_cuMemPoolCreate_38024, libname_cuDeviceGet_40217, libname_cuDeviceGetAttribute_41433, libname_cuCtxSetCurrent_38409, libname_cuCtxGetCurrent_40285, libname_cuMemPoolTrimTo_39511, libname_cuMemPoolTrimTo_39395, libname_cuStreamQuery_41972, libname_cuCtxGetCurrent_37945, libname_cublasSscal_v2_39494, libname_cuMemPoolTrimTo_39462, libname_cuDeviceGetName_37443, libname_cuDeviceGetCount_39613, libname_cublasSgemm_v2_39442, libname_cuCtxSetCurrent_37891, libname_cuMemAllocAsync_38528, libname_cuMemPoolSetAttribute_38013, libname_cuCtxPushCurrent_v2_41961, libname_cuMemcpyDtoDAsync_v2_38454, libname_cuMemcpyHtoDAsync_v2_38348, libname_cuDeviceGetCount_37287, libname_cublasSetStream_v2_39406, libname_cuMemsetD32Async_40788, libname_cuStreamQuery_38336, libname_cuMemGetInfo_v2_41674, libname_cuMemPoolSetAccess_41767, libname_cuStreamSynchronize_41812, libname_cuMemPoolTrimTo_39365, libname_cuCtxGetDevice_40735, libname_cuCtxSetCurrent_38412, libname_cuStreamGetCaptureInfo_38400, libname_cuDeviceGetAttribute_37861, libname_cuStreamQuery_38335, libname_cuDeviceSetMemPool_40358, libname_cuMemPoolTrimTo_39480, libname_cuMemPoolSetAccess_38364, libname_cuCtxGetId_40890, libname_cublasSscal_v2_64_39523, libname_cudaRuntimeGetVersion_41007, libname_cuCtxGetId_37904, libname_cuMemPoolGetAttribute_37264, libname_cuDeviceGetCount_40845, libname_cublasSetMathMode_39329, libname_cuCtxGetCurrent_41517, libname_cuDeviceGetName_39783, libname_cuDevicePrimaryCtxRetain_40882, libname_cublasSsymm_v2_39283, libname_cublasSgemm_v2_39457, libname_cublasSscal_v2_39507, diffejulia_loss_function_37136wrap, libname_cublasSsymm_v2_64_39261, libname_cuMemPoolCreate_37939, libname_cuCtxPopCurrent_v2_38327, libname_cuMemPoolSetAccess_40535, libname_cuMemPoolCreate_41511, libname_cudaRuntimeGetVersion_37435, libname_cublasSscal_v2_64_39490, libname_cuMemPoolGetAttribute_40822, libname_cuMemPoolSetAttribute_41585, libname_cuMemPoolTrimTo_39288, libname_cuStreamQuery_40741, libname_cublasSetStream_v2_39392, libname_cuCtxGetApiVersion_40246, libname_cublasSetStream_v2_39382, libname_cuCtxSetCurrent_40832, libname_cuCtxGetId_40244, libname_cuCtxSetCurrent_39600, libname_cuDeviceCanAccessPeer_40546, libname_cuMemcpyPeerAsync_38458, libname_cuCtxPopCurrent_v2_41964, libname_cuCtxGetApiVersion_37906, libname_cublasSetMathMode_39340, libname_cuCtxGetCurrent_39598, libname_cublasSsymm_v2_39268, libname_cublasSgemm_v2_64_39476, libname_cuCtxSetCurrent_37274, libname_cuLaunchCooperativeKernel_38605, libname_cuDevicePrimaryCtxRetain_39650, libname_cuDeviceSetMemPool_41590, libname_cuMemPoolTrimTo_39343, libname_cuCtxPushCurrent_v2_38324 }) } Stacktrace: [1] macro expansion @ ~/.julia/packages/LLVM/bzSzE/src/executionengine/utils.jl:32 [inlined] [2] lookup @ ~/.julia/packages/LLVM/bzSzE/src/orc.jl:434 [inlined] [3] lookup @ ~/.julia/packages/LLVM/bzSzE/src/orc.jl:433 [inlined] [4] lookup @ ~/.julia/packages/Enzyme/NVk8T/src/compiler/orcv2.jl:255 [inlined] [5] _link(job::GPUCompiler.CompilerJob{…}, ::Tuple{…}) @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/compiler.jl:5751 [6] cached_compilation @ ~/.julia/packages/Enzyme/NVk8T/src/compiler.jl:5811 [inlined] [7] (::Enzyme.Compiler.var"#562#563"{…})(ctx::LLVM.Context) @ Enzyme.Compiler ~/.julia/packages/Enzyme/NVk8T/src/compiler.jl:5876 [8] JuliaContext(f::Enzyme.Compiler.var"#562#563"{…}; kwargs::@Kwargs{}) @ GPUCompiler ~/.julia/packages/GPUCompiler/kqxyC/src/driver.jl:52 [9] JuliaContext(f::Function) @ GPUCompiler ~/.julia/packages/GPUCompiler/kqxyC/src/driver.jl:42 [10] #s2025#561 @ ~/.julia/packages/Enzyme/NVk8T/src/compiler.jl:5828 [inlined] [11] @ Enzyme.Compiler ./none:0 [12] (::Core.GeneratedFunctionStub)(::UInt64, ::LineNumberNode, ::Any, ::Vararg{Any}) @ Core ./boot.jl:602 [13] autodiff @ ~/.julia/packages/Enzyme/NVk8T/src/Enzyme.jl:286 [inlined] [14] autodiff(::ReverseMode{…}, ::typeof(loss_function), ::Type{…}, ::Const{…}, ::Duplicated{…}, ::Duplicated{…}, ::Duplicated{…}) @ Enzyme ~/.julia/packages/Enzyme/NVk8T/src/Enzyme.jl:303 [15] top-level scope @ /mnt/research/ongoing/lux/enzyme_cuda.jl:17 Some type information was truncated. Use `show(err)` to see complete types. in expression starting at /mnt/research/ongoing/lux/enzyme_cuda.jl:12