search

JuliaGPU / CUDA.jl

CUDA programming in Julia.
https://juliagpu.org/cuda/
Other
1.22k stars 225 forks source link

`add CUDA` leads to a segmentation fault #2163

Open Vilin97 opened 1 year ago

Vilin97 commented 1 year ago

Describe the bug

I try to add CUDA and get a segfault.

To reproduce

The Minimal Working Example (MWE) for this bug:


(@v1.9) pkg> activate new
  Activating new project at `~/new`

(new) pkg> add CUDA
   Resolving package versions...
    Updating `~/new/Project.toml`
  [052768ef] + CUDA v5.1.0
    Updating `~/new/Manifest.toml`

[77441] signal (11.1): Segmentation fault
in expression starting at /home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/platform_augmentation.jl:102
Allocations: 2997 (Pool: 2985; Big: 12); GC: 0
ERROR: failed process: Process(`/home/vilin/julia-1.9.0/bin/julia -Cnative -J/home/vilin/julia-1.9.0/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --startup-file=no --compiled-modules=yes --project=/home/vilin/new/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/julia-1.9.0/local/share/julia", "/home/vilin/julia-1.9.0/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.9.0`, ProcessSignaled(11)) [0]

Stacktrace:
  [1] pipeline_error
    @ ./process.jl:565 [inlined]
  [2] read(cmd::Cmd)
    @ Base ./process.jl:449
  [3] collect_artifacts(pkg_root::String; platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:721
  [4] collect_artifacts
    @ ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:707 [inlined]
  [5] check_artifacts_downloaded(pkg_root::String; platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:765
  [6] check_artifacts_downloaded
    @ ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:764 [inlined]
  [7] is_package_downloaded(project_file::String, pkg::Pkg.Types.PackageSpec; platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:2188
  [8] is_package_downloaded
    @ ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:2183 [inlined]
  [9] print_status(env::Pkg.Types.EnvCache, old_env::Pkg.Types.EnvCache, registries::Vector{Pkg.Registry.RegistryInstance}, header::Symbol, uuids::Vector{Base.UUID}, names::Vector{String}; manifest::Bool, diff::Bool, ignore_indent::Bool, outdated::Bool, extensions::Bool, io::Base.TTY, mode::Pkg.Types.PackageMode, hidden_upgrades_info::Bool, show_usagetips::Bool)
    @ Pkg.Operations ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:2301
 [10] print_status
    @ ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:2233 [inlined]
 [11] status(env::Pkg.Types.EnvCache, registries::Vector{Pkg.Registry.RegistryInstance}, pkgs::Vector{Pkg.Types.PackageSpec}; header::Symbol, mode::Pkg.Types.PackageMode, git_diff::Bool, env_diff::Pkg.Types.EnvCache, ignore_indent::Bool, io::Base.TTY, outdated::Bool, extensions::Bool, hidden_upgrades_info::Bool, show_usagetips::Bool)
    @ Pkg.Operations ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:2467
 [12] status (repeats 2 times)
    @ ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:2435 [inlined]
 [13] #show_update#167
    @ ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:2431 [inlined]
 [14] add(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}, new_git::Set{Base.UUID}; preserve::Pkg.Types.PreserveLevel, platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1393
 [15] add
    @ ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/Operations.jl:1372 [inlined]
 [16] add(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; preserve::Pkg.Types.PreserveLevel, platform::Base.BinaryPlatforms.Platform, kwargs::Base.Pairs{Symbol, Base.TTY, Tuple{Symbol}, NamedTuple{(:io,), Tuple{Base.TTY}}})
    @ Pkg.API ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/API.jl:275
 [17] add(pkgs::Vector{Pkg.Types.PackageSpec}; io::Base.TTY, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Pkg.API ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/API.jl:156
 [18] add(pkgs::Vector{Pkg.Types.PackageSpec})
    @ Pkg.API ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/API.jl:145
 [19] do_cmd!(command::Pkg.REPLMode.Command, repl::REPL.LineEditREPL)
    @ Pkg.REPLMode ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/REPLMode/REPLMode.jl:412
 [20] do_cmd(repl::REPL.LineEditREPL, input::String; do_rethrow::Bool)
    @ Pkg.REPLMode ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/REPLMode/REPLMode.jl:390
 [21] do_cmd
    @ ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/REPLMode/REPLMode.jl:380 [inlined]
 [22] (::Pkg.REPLMode.var"#24#27"{REPL.LineEditREPL, REPL.LineEdit.Prompt})(s::REPL.LineEdit.MIState, buf::IOBuffer, ok::Bool)
    @ Pkg.REPLMode ~/julia-1.9.0/share/julia/stdlib/v1.9/Pkg/src/REPLMode/REPLMode.jl:557
 [23] #invokelatest#2
    @ ./essentials.jl:816 [inlined]
 [24] invokelatest
    @ ./essentials.jl:813 [inlined]
 [25] run_interface(terminal::REPL.Terminals.TextTerminal, m::REPL.LineEdit.ModalInterface, s::REPL.LineEdit.MIState)
    @ REPL.LineEdit ~/julia-1.9.0/share/julia/stdlib/v1.9/REPL/src/LineEdit.jl:2647
 [26] run_frontend(repl::REPL.LineEditREPL, backend::REPL.REPLBackendRef)
    @ REPL ~/julia-1.9.0/share/julia/stdlib/v1.9/REPL/src/REPL.jl:1300
 [27] (::REPL.var"#62#68"{REPL.LineEditREPL, REPL.REPLBackendRef})()
    @ REPL ./task.jl:514
Manifest.toml

``` It's a new environment so I guess no Manifest. ```

Expected behavior

No segfault.

Version info

Details on Julia:

Julia Version 1.9.0
Commit 8e630552924 (2023-05-07 11:25 UTC)
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 80 × Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-14.0.6 (ORCJIT, broadwell)
  Threads: 1 on 80 virtual cores

Details on CUDA:

Not applicable

Additional context

I think something went wrong when I tried to add CUDAdrv. Before that CUDA was working fine. When I tried to add CUDAdrv, I got a segfault. Then I removed both CUDA and CUDAdrv with ] rm CUDA CUDAdrv and tried to just add CUDA but got a segfault again. At the same time, CUDA is working fine in another REPL window on the same machine.

vchuravy commented 1 year ago

How did you install Julia? Can you post versioninfo(verbose=true) and try if Julia 1.9.3/1.9.4 work?

Vilin97 commented 1 year ago 
julia> versioninfo(verbose=true)
Julia Version 1.9.0
Commit 8e630552924 (2023-05-07 11:25 UTC)
Platform Info:
  OS: Linux (x86_64-linux-gnu)
      Ubuntu 18.04.6 LTS
  uname: Linux 4.15.0-213-generic #224-Ubuntu SMP Mon Jun 19 13:30:12 UTC 2023 x86_64 x86_64
  CPU: Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz: 
                 speed         user         nice          sys         idle          irq
       #1-80  2618 MHz    4079822 s   21339817 s     225150 s    3588343 s          0 s
  Memory: 503.59081649780273 GB (428895.515625 MB free)
  Uptime: 36638.07 sec
  Load Avg:  125.28  128.74  129.86
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-14.0.6 (ORCJIT, broadwell)
  Threads: 1 on 80 virtual cores
Environment:
  HOME = /home/vilin
  TERM = xterm-256color
  PATH = /home/vilin/julia-1.9.0/bin:/home/vilin/.vscode-server/bin/2b35e1e6d88f1ce073683991d1eff5284a32690f/bin/remote-cli:/home/vilin/.local/bin:/home/vilin/julia-1.9.0/bin:/home/vilin/anaconda3/bin:/home/vilin/anaconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin

It usually takes me a day to install a new version because I usually run into issues, so I won't try other versions for now. I installed from the main website.

maleadt commented 1 year ago

This looks a bit like https://github.com/JuliaLang/julia/issues/48360, which also happens during artifact selection.

It usually takes me a day to install a new version because I usually run into issues, so I won't try other versions for now.

You don't need to install anything, just download the tarball and run a single binary. I'm not sure how that takes a day? There's been many bugfixes between Julia 1.9.0 and the latest version, so we generally don't support those older versions.

Vilin97 commented 1 year ago

I get the same error on 1.10

(@v1.10) pkg> add CUDA
   Resolving package versions...
    Updating `~/.julia/environments/v1.10/Project.toml`
  [052768ef] + CUDA v5.1.0
    Updating `~/.julia/environments/v1.10/Manifest.toml`

[41772] signal (11.1): Segmentation fault
in expression starting at /home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/platform_augmentation.jl:95
Allocations: 2905 (Pool: 2896; Big: 9); GC: 0
ERROR: failed process: Process(`/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0`, ProcessSignaled(11)) [0]

Stacktrace:
  [1] pipeline_error
    @ Base ./process.jl:565 [inlined]
  [2] read(cmd::Cmd)
    @ Base ./process.jl:449
  [3] collect_artifacts(pkg_root::String; platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:720
  [4] collect_artifacts
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:706 [inlined]
  [5] check_artifacts_downloaded(pkg_root::String; platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:764
  [6] check_artifacts_downloaded
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:763 [inlined]
  [7] is_package_downloaded(manifest_file::String, pkg::Pkg.Types.PackageSpec; platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:2191
  [8] is_package_downloaded
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:2186 [inlined]
  [9] print_status(env::Pkg.Types.EnvCache, old_env::Pkg.Types.EnvCache, registries::Vector{Pkg.Registry.RegistryInstance}, header::Symbol, uuids::Vector{Base.UUID}, names::Vector{String}; manifest::Bool, diff::Bool, ignore_indent::Bool, outdated::Bool, extensions::Bool, io::Base.TTY, mode::Pkg.Types.PackageMode, hidden_upgrades_info::Bool, show_usagetips::Bool)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:2304
 [10] print_status
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:2236 [inlined]
 [11] status(env::Pkg.Types.EnvCache, registries::Vector{Pkg.Registry.RegistryInstance}, pkgs::Vector{Pkg.Types.PackageSpec}; header::Symbol, mode::Pkg.Types.PackageMode, git_diff::Bool, env_diff::Pkg.Types.EnvCache, ignore_indent::Bool, io::Base.TTY, outdated::Bool, extensions::Bool, hidden_upgrades_info::Bool, show_usagetips::Bool)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:2470
 [12] status (repeats 2 times)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:2438 [inlined]
 [13] #show_update#167
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:2434 [inlined]
 [14] add(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}, new_git::Set{Base.UUID}; preserve::Pkg.Types.PreserveLevel, platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:1393
 [15] add
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:1372 [inlined]
 [16] add(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; preserve::Pkg.Types.PreserveLevel, platform::Base.BinaryPlatforms.Platform, kwargs::@Kwargs{io::Base.TTY})
    @ Pkg.API ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/API.jl:278
 [17] add(pkgs::Vector{Pkg.Types.PackageSpec}; io::Base.TTY, kwargs::@Kwargs{})
    @ Pkg.API ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/API.jl:159
 [18] add(pkgs::Vector{Pkg.Types.PackageSpec})
    @ Pkg.API ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/API.jl:148
 [19] do_cmd!(command::Pkg.REPLMode.Command, repl::REPL.LineEditREPL)
    @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:412
 [20] do_cmd(repl::REPL.LineEditREPL, input::String; do_rethrow::Bool)
    @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:390
 [21] do_cmd
    @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:380 [inlined]
 [22] (::Pkg.REPLMode.var"#24#27"{REPL.LineEditREPL, REPL.LineEdit.Prompt})(s::REPL.LineEdit.MIState, buf::IOBuffer, ok::Bool)
    @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:557
 [23] #invokelatest#2
    @ Base ./essentials.jl:887 [inlined]
 [24] invokelatest
    @ Base ./essentials.jl:884 [inlined]
 [25] run_interface(terminal::REPL.Terminals.TextTerminal, m::REPL.LineEdit.ModalInterface, s::REPL.LineEdit.MIState)
    @ REPL.LineEdit ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/REPL/src/LineEdit.jl:2656
 [26] run_frontend(repl::REPL.LineEditREPL, backend::REPL.REPLBackendRef)
    @ REPL ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/REPL/src/REPL.jl:1312
 [27] (::REPL.var"#62#68"{REPL.LineEditREPL, REPL.REPLBackendRef})()
    @ REPL ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/REPL/src/REPL.jl:386

julia> versioninfo(verbose=true)
Julia Version 1.10.0-rc1
Commit 5aaa9485436 (2023-11-03 07:44 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: Linux (x86_64-linux-gnu)
      Ubuntu 18.04.6 LTS
  uname: Linux 4.15.0-213-generic #224-Ubuntu SMP Mon Jun 19 13:30:12 UTC 2023 x86_64 x86_64
  CPU: Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz: 
                 speed         user         nice          sys         idle          irq
       #1-80  2573 MHz   19542850 s   63496025 s     790037 s    3847657 s          0 s
  Memory: 503.59081649780273 GB (420917.34375 MB free)
  Uptime: 109803.07 sec
  Load Avg:  108.46  104.5  106.45
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-15.0.7 (ORCJIT, broadwell)
  Threads: 1 on 80 virtual cores
Environment:
  HOME = /home/vilin
  TERM = xterm-256color
  PATH = /home/vilin/.juliaup/bin:/home/vilin/julia-1.9.0/bin:/home/vilin/.vscode-server/bin/1a5daa3a0231a0fbba4f14db7ec463cf99d7768e/bin/remote-cli:/home/vilin/.local/bin:/home/vilin/julia-1.9.0/bin:/home/vilin/anaconda3/bin:/home/vilin/anaconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
Vilin97 commented 1 year ago

I deleted the .julia folder thinking that maybe something got installed wrong. But that did not solve the problem:

(@v1.10) pkg> add CUDA
   Resolving package versions...
    Updating `~/.julia/environments/v1.10/Project.toml`
  [052768ef] + CUDA v5.1.0
    Updating `~/.julia/environments/v1.10/Manifest.toml`
  [621f4979] + AbstractFFTs v1.5.0
  [79e6a3ab] + Adapt v3.7.1
  [a9b6321e] + Atomix v0.1.0
  [ab4f0b2a] + BFloat16s v0.4.2
⌅ [fa961155] + CEnum v0.4.2
  [052768ef] + CUDA v5.1.0
  [1af6417a] + CUDA_Runtime_Discovery v0.2.2
  [3da002f7] + ColorTypes v0.11.4
  [5ae59095] + Colors v0.12.10
  [34da2185] + Compat v4.10.0
  [a8cc5b0e] + Crayons v4.1.1
  [9a962f9c] + DataAPI v1.15.0
  [a93c6f00] + DataFrames v1.6.1
  [864edb3b] + DataStructures v0.18.15
  [e2d170a0] + DataValueInterfaces v1.0.0
  [e2ba6199] + ExprTools v0.1.10
  [53c48c17] + FixedPointNumbers v0.8.4
  [0c68f7d7] + GPUArrays v9.1.0
  [46192b85] + GPUArraysCore v0.1.5
  [61eb1bfa] + GPUCompiler v0.25.0
  [842dd82b] + InlineStrings v1.4.0
  [41ab1584] + InvertedIndices v1.3.0
  [82899510] + IteratorInterfaceExtensions v1.0.0
  [692b3bcd] + JLLWrappers v1.5.0
  [63c18a36] + KernelAbstractions v0.9.12
  [929cbde3] + LLVM v6.4.0
  [8b046642] + LLVMLoopInfo v1.0.0
  [b964fa9f] + LaTeXStrings v1.3.1
  [1914dd2f] + MacroTools v0.5.11
  [e1d29d7a] + Missings v1.1.0
  [5da4648a] + NVTX v0.3.3
  [bac558e1] + OrderedCollections v1.6.2
  [69de0a69] + Parsers v2.8.0
  [2dfb63ee] + PooledArrays v1.4.3
  [aea7be01] + PrecompileTools v1.2.0
  [21216c6a] + Preferences v1.4.1
  [08abe8d2] + PrettyTables v2.2.8
  [74087812] + Random123 v1.6.1
  [e6cf234a] + RandomNumbers v1.5.3
  [189a3867] + Reexport v1.2.2
  [ae029012] + Requires v1.3.0
  [6c6a2e73] + Scratch v1.2.1
  [91c51154] + SentinelArrays v1.4.1
  [a2af1166] + SortingAlgorithms v1.2.0
  [90137ffa] + StaticArrays v1.6.5
  [1e83bf80] + StaticArraysCore v1.4.2
  [892a3eda] + StringManipulation v0.3.4
  [3783bdb8] + TableTraits v1.0.1
  [bd369af6] + Tables v1.11.1
  [a759f4b9] + TimerOutputs v0.5.23
  [013be700] + UnsafeAtomics v0.2.1
  [d80eeb9a] + UnsafeAtomicsLLVM v0.1.3
  [4ee394cb] + CUDA_Driver_jll v0.7.0+0
→ [76a88914] + CUDA_Runtime_jll v0.10.0+1
  [9c1d0b0a] + JuliaNVTXCallbacks_jll v0.2.1+0
  [dad2f222] + LLVMExtra_jll v0.0.27+0
  [e98f9f5b] + NVTX_jll v3.1.0+2
  [0dad84c5] + ArgTools v1.1.1
  [56f22d72] + Artifacts
  [2a0f44e3] + Base64
  [ade2ca70] + Dates
  [f43a241f] + Downloads v1.6.0
  [7b1f6079] + FileWatching
  [9fa8497b] + Future
  [b77e0a4c] + InteractiveUtils
  [4af54fe1] + LazyArtifacts
  [b27032c2] + LibCURL v0.6.4
  [76f85450] + LibGit2
  [8f399da3] + Libdl
  [37e2e46d] + LinearAlgebra
  [56ddb016] + Logging
  [d6f4376e] + Markdown
  [ca575930] + NetworkOptions v1.2.0
  [44cfe95a] + Pkg v1.10.0
  [de0858da] + Printf
  [3fa0cd96] + REPL
  [9a3f8284] + Random
  [ea8e919c] + SHA v0.7.0
  [9e88b42a] + Serialization
  [6462fe0b] + Sockets
  [2f01184e] + SparseArrays v1.10.0
  [10745b16] + Statistics v1.10.0
  [fa267f1f] + TOML v1.0.3
  [a4e569a6] + Tar v1.10.0
  [8dfed614] + Test
  [cf7118a7] + UUIDs
  [4ec0a83e] + Unicode
  [e66e0078] + CompilerSupportLibraries_jll v1.0.5+1
  [deac9b47] + LibCURL_jll v8.4.0+0
  [e37daf67] + LibGit2_jll v1.6.4+0
  [29816b5a] + LibSSH2_jll v1.11.0+1
  [c8ffd9c3] + MbedTLS_jll v2.28.2+1
  [14a3606d] + MozillaCACerts_jll v2023.1.10
  [4536629a] + OpenBLAS_jll v0.3.23+2
  [bea87d4a] + SuiteSparse_jll v7.2.1+1
  [83775a58] + Zlib_jll v1.2.13+1
  [8e850b90] + libblastrampoline_jll v5.8.0+1
  [8e850ede] + nghttp2_jll v1.52.0+1
  [3f19e933] + p7zip_jll v17.4.0+2
        Info Packages marked with → are not downloaded, use `instantiate` to download
        Info Packages marked with ⌅ have new versions available but compatibility constraints restrict them from upgrading. To see why use `status --outdated -m`

[58197] signal (11.1): Segmentation fault
in expression starting at /home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/platform_augmentation.jl:95
Allocations: 2905 (Pool: 2896; Big: 9); GC: 0
ERROR: failed process: Process(`/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0`, ProcessSignaled(11)) [0]

Stacktrace:
  [1] pipeline_error
    @ Base ./process.jl:565 [inlined]
  [2] read(cmd::Cmd)
    @ Base ./process.jl:449
  [3] collect_artifacts(pkg_root::String; platform::Base.BinaryPlatforms.Platform)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:720
  [4] collect_artifacts
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:706 [inlined]
  [5] download_artifacts(env::Pkg.Types.EnvCache; platform::Base.BinaryPlatforms.Platform, julia_version::VersionNumber, verbose::Bool, io::Base.TTY)
    @ Pkg.Operations ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Operations.jl:752
  [6] instantiate(ctx::Pkg.Types.Context; manifest::Nothing, update_registry::Bool, verbose::Bool, platform::Base.BinaryPlatforms.Platform, allow_build::Bool, allow_autoprecomp::Bool, kwargs::@Kwargs{})
    @ Pkg.API ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/API.jl:1868
  [7] precompile(ctx::Pkg.Types.Context, pkgs::Vector{Pkg.Types.PackageSpec}; internal_call::Bool, strict::Bool, warn_loaded::Bool, already_instantiated::Bool, timing::Bool, _from_loading::Bool, kwargs::@Kwargs{})
    @ Pkg.API ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/API.jl:1082
  [8] precompile
    @ Pkg.API ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/API.jl:1078 [inlined]
  [9] #_auto_precompile#6
    @ Pkg ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Pkg.jl:805 [inlined]
 [10] _auto_precompile (repeats 2 times)
    @ Pkg ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/Pkg.jl:803 [inlined]
 [11] add(pkgs::Vector{Pkg.Types.PackageSpec}; io::Base.TTY, kwargs::@Kwargs{})
    @ Pkg.API ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/API.jl:160
 [12] add(pkgs::Vector{Pkg.Types.PackageSpec})
    @ Pkg.API ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/API.jl:148
 [13] do_cmd!(command::Pkg.REPLMode.Command, repl::REPL.LineEditREPL)
    @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:412
 [14] do_cmd(repl::REPL.LineEditREPL, input::String; do_rethrow::Bool)
    @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:390
 [15] do_cmd
    @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:380 [inlined]
 [16] (::Pkg.REPLMode.var"#24#27"{REPL.LineEditREPL, REPL.LineEdit.Prompt})(s::REPL.LineEdit.MIState, buf::IOBuffer, ok::Bool)
    @ Pkg.REPLMode ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/Pkg/src/REPLMode/REPLMode.jl:557
 [17] #invokelatest#2
    @ Base ./essentials.jl:887 [inlined]
 [18] invokelatest
    @ Base ./essentials.jl:884 [inlined]
 [19] run_interface(terminal::REPL.Terminals.TextTerminal, m::REPL.LineEdit.ModalInterface, s::REPL.LineEdit.MIState)
    @ REPL.LineEdit ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/REPL/src/LineEdit.jl:2656
 [20] run_frontend(repl::REPL.LineEditREPL, backend::REPL.REPLBackendRef)
    @ REPL ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/REPL/src/REPL.jl:1312
 [21] (::REPL.var"#62#68"{REPL.LineEditREPL, REPL.REPLBackendRef})()
    @ REPL ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/REPL/src/REPL.jl:386

(@v1.10) pkg> st
Status `~/.julia/environments/v1.10/Project.toml`
  [052768ef] CUDA v5.1.0
maleadt commented 1 year ago

Does it happen if you run the offending command in isolation?

/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0

If so, can you run with --bug-report=rr and upload a trace?

Vilin97 commented 1 year ago

Sorry, what is the command to run? My general computer maturity is not enough to understand what to run from your previous message.

maleadt commented 1 year ago 
/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia --bug-report=rr -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0
Vilin97 commented 1 year ago

That's what I get.

vilin@doppio:~$ /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia --bug-report=rr -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0
[ Info: Loading BugReporting package...
[ Info: Package `BugReporting` not found - attempting temporary installation
   Resolving package versions...
    Updating `/tmp/jl_jja03B/Project.toml`
  [bcf9a6e7] + BugReporting v0.3.3
    Updating `/tmp/jl_jja03B/Manifest.toml`
  [d1d4a3ce] + BitFlags v0.1.8
  [bcf9a6e7] + BugReporting v0.3.3
  [944b1d66] + CodecZlib v0.7.3
  [f0e56b4a] + ConcurrentUtilities v2.3.0
  [460bff9d] + ExceptionUnwrapping v0.1.9
  [d7ba0133] + Git v1.3.0
  [cd3eb016] + HTTP v1.10.0
  [692b3bcd] + JLLWrappers v1.5.0
  [682c06a0] + JSON v0.21.4
  [e6f89c97] + LoggingExtras v1.0.3
  [739be429] + MbedTLS v1.1.8
  [4d8831e6] + OpenSSL v1.4.1
  [69de0a69] + Parsers v2.8.0
  [aea7be01] + PrecompileTools v1.2.0
  [21216c6a] + Preferences v1.4.1
  [92933f4c] + ProgressMeter v1.9.0
  [6c6a2e73] + Scratch v1.2.1
  [777ac1f9] + SimpleBufferStream v1.1.0
  [3bb67fe8] + TranscodingStreams v0.10.2
  [5c2747f8] + URIs v1.5.1
  [6e34b625] + Bzip2_jll v1.0.8+0
⌅ [ab5a07f8] + Elfutils_jll v0.182.0+1
  [2e619515] + Expat_jll v2.5.0+0
  [a8b33d9f] + GDB_jll v12.1.0+1
⌃ [f8c6e375] + Git_jll v2.36.1+2
⌅ [e9f186c6] + Libffi_jll v3.2.2+1
  [94ce4f54] + Libiconv_jll v1.17.0+0
⌅ [458c3c95] + OpenSSL_jll v1.1.23+0
⌅ [93d3a430] + Python_jll v3.8.8+3
  [ffd25f8a] + XZ_jll v5.4.5+0
  [3161d3a3] + Zstd_jll v1.5.5+0
  [c53206cc] + argp_standalone_jll v1.3.1+0
  [d65627f6] + fts_jll v1.2.8+0
  [c88a4935] + obstack_jll v1.2.3+0
  [e86bdf43] + rr_jll v5.6.0+2
  [9c7924da] + s5cmd_jll v2.0.0+0
  [0dad84c5] + ArgTools v1.1.1
  [56f22d72] + Artifacts
  [2a0f44e3] + Base64
  [ade2ca70] + Dates
  [8ba89e20] + Distributed
  [f43a241f] + Downloads v1.6.0
  [7b1f6079] + FileWatching
  [b77e0a4c] + InteractiveUtils
  [b27032c2] + LibCURL v0.6.4
  [76f85450] + LibGit2
  [8f399da3] + Libdl
  [56ddb016] + Logging
  [d6f4376e] + Markdown
  [a63ad114] + Mmap
  [ca575930] + NetworkOptions v1.2.0
  [44cfe95a] + Pkg v1.10.0
  [de0858da] + Printf
  [3fa0cd96] + REPL
  [9a3f8284] + Random
  [ea8e919c] + SHA v0.7.0
  [9e88b42a] + Serialization
  [6462fe0b] + Sockets
  [fa267f1f] + TOML v1.0.3
  [a4e569a6] + Tar v1.10.0
  [8dfed614] + Test
  [cf7118a7] + UUIDs
  [4ec0a83e] + Unicode
  [e66e0078] + CompilerSupportLibraries_jll v1.0.5+1
  [781609d7] + GMP_jll v6.2.1+6
  [deac9b47] + LibCURL_jll v8.4.0+0
  [e37daf67] + LibGit2_jll v1.6.4+0
  [29816b5a] + LibSSH2_jll v1.11.0+1
  [c8ffd9c3] + MbedTLS_jll v2.28.2+1
  [14a3606d] + MozillaCACerts_jll v2023.1.10
  [efcefdf7] + PCRE2_jll v10.42.0+1
  [83775a58] + Zlib_jll v1.2.13+1
  [8e850ede] + nghttp2_jll v1.52.0+1
  [3f19e933] + p7zip_jll v17.4.0+2
        Info Packages marked with ⌃ and ⌅ have new versions available. Those with ⌃ may be upgradable, but those with ⌅ are restricted by compatibility constraints from upgrading. To see why use `status --outdated -m`
rr: Saving execution to trace directory `/tmp/jl_olbumT/julia-0'.
Preparing trace for upload (if your trace is large this may take a few minutes)...

### IMPORTANT =============================================================
You are about to upload a trace directory to a publicly accessible location.
Such traces contain any information that was accessed by the traced
executable during its execution. This includes any code loaded, any
secrets entered, the contents of any configuration files, etc.

DO NOT proceed, if you do not wish to make this information publicly available.
By proceeding you explicitly agree to waive any privacy interest in the
uploaded information.
### =======================================================================

To upload a trace, please authenticate by visiting:

        https://github.com/login/oauth/authorize?client_id=Iv1.c29a629771fe63c4&state=OZrKUcVFoAMAbNg%3D

You can cancel upload by `Ctrl-C`.
ERROR: InvalidStateException: Channel is closed.
Stacktrace:
  [1] try_yieldto(undo::typeof(Base.ensure_rescheduled))
    @ Base ./task.jl:931
  [2] wait()
    @ Base ./task.jl:995
  [3] wait(c::Base.GenericCondition{ReentrantLock}; first::Bool)
    @ Base ./condition.jl:130
  [4] wait
    @ Base ./condition.jl:125 [inlined]
  [5] take_unbuffered(c::Channel{Any})
    @ Base ./channels.jl:489
  [6] take!
    @ ./channels.jl:466 [inlined]
  [7] get_upload_params()
    @ BugReporting ~/.julia/packages/BugReporting/ukm66/src/BugReporting.jl:629
  [8] (::BugReporting.var"#20#22"{Vector{String}})(trace_dir::String)
    @ BugReporting ~/.julia/packages/BugReporting/ukm66/src/BugReporting.jl:551
  [9] mktempdir(fn::BugReporting.var"#20#22"{Vector{String}}, parent::String; prefix::String)
    @ Base.Filesystem ./file.jl:766
 [10] mktempdir (repeats 2 times)
    @ ./file.jl:762 [inlined]
 [11] make_interactive_report(report_arg::String, ARGS::Vector{String})
    @ BugReporting ~/.julia/packages/BugReporting/ukm66/src/BugReporting.jl:547
 [12] #invokelatest#2
    @ ./essentials.jl:887 [inlined]
 [13] invokelatest
    @ ./essentials.jl:884 [inlined]
 [14] report_bug(kind::String)
    @ InteractiveUtils ~/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia/stdlib/v1.10/InteractiveUtils/src/InteractiveUtils.jl:365
 [15] exec_options(opts::Base.JLOptions)
    @ Base ./client.jl:253
 [16] _start()
    @ Base ./client.jl:552
maleadt commented 1 year ago

To upload a trace, please authenticate by visiting:

    https://github.com/login/oauth/authorize?client_id=Iv1.c29a629771fe63c4&state=OZrKUcVFoAMAbNg%3D

You can cancel upload by Ctrl-C.

Did you do this?

Also, please try again until it segfaults and upload that trace. Here, it looks like everything just worked.

Vilin97 commented 1 year ago

Yes, I followed that link and authorized. Here is what I see there now, after authorizing.

image

What do you mean by trying until it segfaults? I ran this command twice and both times got the same output.

maleadt commented 1 year ago

there now, after authorizing.

image

That's an internal server error, so authentication did not work. Are you sure you're logged in with GitHub and you allowed the authentication to happen? If so, we'll have to look at the server logs. I'm also adding a feature to BugReporting.jl to allow a manual upload if the automatic one fails, so please hold until that's been merged before retrying.

What do you mean by trying until it segfaults?

The command you're executing is recording a trace of what happens. Since you're not reproducing the segfault you filed this bug for, the trace will not be interesting. Does the segfault still occur when running that command without --bug-report=rr? If it only happens intermittently, you'll have to run the version of the command with --bug-report=rr multiple times too until it reproduces the segfault again.

Vilin97 commented 1 year ago

I ran the command from inside the project instead of the home directory. This reproduced the segfault.

Trace

vilin@doppio:~/GradientFlows.jl$ /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia -Cnative -J/home/vilin/.
julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/envi
ronments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-r
c1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0

[26162] signal (11.1): Segmentation fault
in expression starting at /home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/platform_augmentation.jl:95
Allocations: 2905 (Pool: 2896; Big: 9); GC: 0
maleadt commented 1 year ago

Thanks for the trace. Which CPU did you record this on? The server I normally use to replay traces from a different CPU sadly isn't available right now, so I'd need to match that up.

maleadt commented 1 year ago

Wait, this trace just exits without an abort? That's not useful... Again, if the trace doesn't contain the segfault, it doesn't help.

maleadt commented 1 year ago

Also, if the error never reproduces under rr (which is possible), can you try running under gdb instead and doing a bt when you hit the segmentation fault? i.e.:

$ cd ~/GradientFlows.jl
$ gdb --args /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia -Cnative -J/home/vilin/.
julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/envi
ronments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-r
c1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0

(gdb) run
# it should run until the segfault here
(gdb) bt
# report the backtrace shown here
Vilin97 commented 1 year ago

My CPU is 40-core (x2 20-core) Intel Xeon E5-2698 v4 @2.2GHz

It looks like the command did not run. The error is Undefined command: "julia". But I can start julia by typing julia just fine.

vilin@doppio:~/GradientFlows.jl$ gdb --args /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia -Cnative -J/home/vilin/.
nu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0
GNU gdb (Ubuntu 8.1.1-0ubuntu1) 8.1.1
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia...done.
(gdb) julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
Undefined command: "julia".  Try "help".
(gdb) append!(empty!(Base.DL_LOAD_PATH), String[])
Undefined append command: "!(empty!(Base.DL_LOAD_PATH), String[])".  Try "help append".
(gdb) 
Undefined append command: "!(empty!(Base.DL_LOAD_PATH), String[])".  Try "help append".
(gdb) cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg"): No such file or directory.
(gdb) include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
Undefined command: "include".  Try "help".
(gdb) ' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0
Undefined command: "".  Try "help".
(gdb) run
Starting program: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia -Cnative -J/home/vilin/.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libgcc_s.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libgcc_s.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libstdc++.so.6: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libstdc++.so.6: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libatomic.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libatomic.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
[New Thread 0x7ffff0b20700 (LWP 39247)]
ERROR: could not load library "/home/vilin"
/home/vilin: cannot read file data: Is a directory
[Thread 0x7ffff0b20700 (LWP 39247) exited]
[Inferior 1 (process 39142) exited with code 01]
(gdb) bt
No stack.
(gdb) q
(base) vilin@doppio:~/GradientFlows.jl$ julia
               _
   _       _ _(_)_     |  Documentation: https://docs.julialang.org
  (_)     | (_) (_)    |
   _ _   _| |_  __ _   |  Type "?" for help, "]?" for Pkg help.
  | | | | | | |/ _` |  |
  | | |_| | | | (_| |  |  Version 1.10.0-rc1 (2023-11-03)
 _/ |\__'_|_|_|\__'_|  |  Official https://julialang.org/ release
|__/                   |

In case it helps, I uploaded another trace by running the same command again.

maleadt commented 1 year ago

Looks like GDB is confused by how arguments are specified (or you accidentally pasted an additional newline). Try doing just gdb /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia and then from the GDB prompt

run -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/envi
ronments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-r
c1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0
Vilin97 commented 1 year ago

Here is what I get now. Thank you for not giving up on this! I appreciate your time.

vilin@doppio:~/GradientFlows.jl$ gdb /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia
GNU gdb (Ubuntu 8.1.1-0ubuntu1) 8.1.1
Copyright (C) 2018 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia...done.
(gdb) run -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empt(gdb) run -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-r
c1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")ulia_version+1.10.0
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0
Starting program: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/julia -Cnative -J/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/lib/julia/sys.so -g1 -O0 --color=no --history-file=no --project=/home/vilin/.julia/environments/v1.10/Project.toml --eval 'append!(empty!(Base.DEPOT_PATH), ["/home/vilin/.julia", "/home/vilin/.julia/juliaup/julia-1.10.0-r
c1+0.x64.linux.gnu/local/share/julia", "/home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/share/julia"])
append!(empty!(Base.DL_LOAD_PATH), String[])

cd("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg")
include("/home/vilin/.julia/packages/CUDA_Runtime_jll/G8f3Q/.pkg/select_artifacts.jl")
' -t1 --startup-file=no x86_64-linux-gnu-libgfortran5-cxx11-julia_version+1.10.0

[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libgcc_s.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libgcc_s.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libstdc++.so.6: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libstdc++.so.6: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libatomic.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libatomic.so.1: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
[New Thread 0x7ffff0b20700 (LWP 74513)]
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libgfortran.so.5: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libgfortran.so.5: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libquadmath.so.0: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010001
BFD: warning: /home/vilin/.julia/juliaup/julia-1.10.0-rc1+0.x64.linux.gnu/bin/../lib/julia/libquadmath.so.0: unsupported GNU_PROPERTY_TYPE (5) type: 0xc0010002
[New Thread 0x7fffdca15700 (LWP 74539)]
[New Thread 0x7fffdc214700 (LWP 74540)]
[New Thread 0x7fffcba13700 (LWP 74541)]
[New Thread 0x7fffcb212700 (LWP 74542)]
[New Thread 0x7fffc2a11700 (LWP 74543)]
[New Thread 0x7fffba210700 (LWP 74544)]
[New Thread 0x7fffa9a0f700 (LWP 74545)]
[New Thread 0x7fffa920e700 (LWP 74546)]
[New Thread 0x7fff98a0d700 (LWP 74547)]
[New Thread 0x7fff9020c700 (LWP 74548)]
[New Thread 0x7fff87a0b700 (LWP 74549)]
[New Thread 0x7fff7f20a700 (LWP 74550)]
[New Thread 0x7fff76a09700 (LWP 74551)]
[New Thread 0x7fff6e208700 (LWP 74552)]
[New Thread 0x7fff65a07700 (LWP 74553)]
[New Thread 0x7fff5d206700 (LWP 74554)]
[New Thread 0x7fff54a05700 (LWP 74555)]
[New Thread 0x7fff4c204700 (LWP 74556)]
[New Thread 0x7fff43a03700 (LWP 74557)]
[New Thread 0x7fff3b202700 (LWP 74558)]
[New Thread 0x7fff32a01700 (LWP 74559)]
[New Thread 0x7fff2a200700 (LWP 74560)]
[New Thread 0x7fff219ff700 (LWP 74561)]
[New Thread 0x7fff191fe700 (LWP 74562)]
[New Thread 0x7fff109fd700 (LWP 74563)]
[New Thread 0x7fff081fc700 (LWP 74564)]
[New Thread 0x7ffeff9fb700 (LWP 74565)]
[New Thread 0x7ffef71fa700 (LWP 74566)]
[New Thread 0x7ffeee9f9700 (LWP 74567)]
[New Thread 0x7ffee61f8700 (LWP 74568)]
[New Thread 0x7ffedd9f7700 (LWP 74569)]
[New Thread 0x7ffed51f6700 (LWP 74570)]
[New Thread 0x7ffecc9f5700 (LWP 74571)]
[New Thread 0x7ffec41f4700 (LWP 74572)]
[New Thread 0x7ffebb9f3700 (LWP 74573)]
[New Thread 0x7ffeb31f2700 (LWP 74574)]
[New Thread 0x7ffeaa9f1700 (LWP 74575)]
[New Thread 0x7ffea21f0700 (LWP 74576)]
[New Thread 0x7ffe999ef700 (LWP 74577)]
[New Thread 0x7ffe8e0f3700 (LWP 74583)]
[CUDA_Runtime]
arch = "x86_64"
git-tree-sha1 = "48c5718070c4561a5f63d987b9fec6beefba7ad3"
lazy = true
libc = "glibc"
cuda = "12.3"
os = "linux"

    [[CUDA_Runtime.download]]
    sha256 = "bc72b967450a1792060a54e6e967546b9052fdcd537db6f47beabd3033ea7e54"
    url = "https://github.com/JuliaBinaryWrappers/CUDA_Runtime_jll.jl/releases/download/CUDA_Runtime-v0.10.0+0/CUDA_Runtime.v0.10.0.x86_64-linux-gnu-cuda+12.3.tar.gz"
[Thread 0x7ffe8e0f3700 (LWP 74583) exited]
[Thread 0x7ffe999ef700 (LWP 74577) exited]
[Thread 0x7ffea21f0700 (LWP 74576) exited]
[Thread 0x7ffeaa9f1700 (LWP 74575) exited]
[Thread 0x7ffeb31f2700 (LWP 74574) exited]
[Thread 0x7ffebb9f3700 (LWP 74573) exited]
[Thread 0x7ffecc9f5700 (LWP 74571) exited]
[Thread 0x7ffed51f6700 (LWP 74570) exited]
[Thread 0x7ffedd9f7700 (LWP 74569) exited]
[Thread 0x7ffee61f8700 (LWP 74568) exited]
[Thread 0x7ffeee9f9700 (LWP 74567) exited]
[Thread 0x7ffef71fa700 (LWP 74566) exited]
[Thread 0x7ffeff9fb700 (LWP 74565) exited]
[Thread 0x7fff081fc700 (LWP 74564) exited]
[Thread 0x7fff109fd700 (LWP 74563) exited]
[Thread 0x7fff191fe700 (LWP 74562) exited]
[Thread 0x7fff219ff700 (LWP 74561) exited]
[Thread 0x7fff2a200700 (LWP 74560) exited]
[Thread 0x7fff32a01700 (LWP 74559) exited]
[Thread 0x7fff3b202700 (LWP 74558) exited]
[Thread 0x7fff43a03700 (LWP 74557) exited]
[Thread 0x7fff4c204700 (LWP 74556) exited]
[Thread 0x7fff54a05700 (LWP 74555) exited]
[Thread 0x7fff5d206700 (LWP 74554) exited]
[Thread 0x7fff65a07700 (LWP 74553) exited]
[Thread 0x7fff6e208700 (LWP 74552) exited]
[Thread 0x7fff76a09700 (LWP 74551) exited]
[Thread 0x7fff7f20a700 (LWP 74550) exited]
[Thread 0x7fff87a0b700 (LWP 74549) exited]
[Thread 0x7fff9020c700 (LWP 74548) exited]
[Thread 0x7fff98a0d700 (LWP 74547) exited]
[Thread 0x7fffa920e700 (LWP 74546) exited]
[Thread 0x7fffa9a0f700 (LWP 74545) exited]
[Thread 0x7fffba210700 (LWP 74544) exited]
[Thread 0x7fffc2a11700 (LWP 74543) exited]
[Thread 0x7fffcb212700 (LWP 74542) exited]
[Thread 0x7fffcba13700 (LWP 74541) exited]
[Thread 0x7fffdc214700 (LWP 74540) exited]
[Thread 0x7fffdca15700 (LWP 74539) exited]
[Thread 0x7ffff0b20700 (LWP 74513) exited]
[Thread 0x7ffff7f97b80 (LWP 74443) exited]
[Inferior 1 (process 74443) exited normally]
(gdb)
Vilin97 commented 1 year ago

Just to be clear, I still get a segfault when I add CUDA.

maleadt commented 1 year ago

Here is what I get now.

Again, no segfault, so sadly not useful. I realize this may be out of your area of experience, but both rr traces and gdb logs of executions without a segfault cannot help to debug this. Maybe try re-running multiple times to catch an instance where it does segfault, and post the backtrace from there?

Also make sure you're running the exact invocation that segfaults under gdb. Minor differences, such as executing from a different directory, may results in it not reproducing.

maleadt commented 1 year ago

Also try running (outside of gdb/rr) with JULIA_CUDA_USE_COMPAT=false in your environment.

christiangnrd commented 11 months ago

Also try running (outside of gdb/rr) with JULIA_CUDA_USE_COMPAT=false in your environment.

I am also have this issue on one of my devices and running with JULIA_CUDA_USE_COMPAT=false seems to resolve it.

maleadt commented 11 months ago

Once the package has been successfully precompiled and installed, can you then import CUDA.jl without that environment variable set?

maleadt commented 11 months ago

It would also be useful if you could provide the output of using CUDA (and the subsequent segfault) with LD_DEBUG=libs set in the environment.

christiangnrd commented 11 months ago

Once the package has been successfully precompiled and installed, can you then import CUDA.jl without that environment variable set?

Yes

It would also be useful if you could provide the output of using CUDA (and the subsequent segfault) with LD_DEBUG=libs set in the environment.

I only get the error when running a fresh precompile. Then I can call using CUDA without problem, but it then breaks when trying to run my code. Calling using cuDNN however breaks.

Output: 

(MST-NN) pkg> precompile
Precompiling project...
  ✗ CUDA_Runtime_jll
  418 dependencies successfully precompiled in 153 seconds. 9 already precompiled.
  1 dependency had output during precompilation:
┌ CUDNN_jll
│  [99167] signal (11.1): Segmentation fault
│  in expression starting at /home/christian/.julia/packages/CUDA_Runtime_jll/bmudn/.pkg/platform_augmentation.jl:102
│  Allocations: 2906 (Pool: 2897; Big: 9); GC: 0
└

Also notice how CUDA_runtime_jll has an x, but the result doesn't mention any failed precompilations, and the segfault comes from CUDNN_jll which had apparently succeeded precompilation.

Output of using cuDNN after unsuccessful precompilation and with LD_DEBUG=libs:

``` julia> using cuDNN 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/CEnum/0gyUJ_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Preferences/pWSk8_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/share/julia/compiled/v1.10/LazyArtifacts/MRP8l_SBKrC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/JLLWrappers/7Zgw7_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/LLVMExtra_jll/R9OeX_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/LLVM/e8NBy_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/ExprTools/eM8wu_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/TimerOutputs/hd2yD_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Scratch/ICI1U_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/GPUCompiler/yPwef_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Adapt/rUIgN_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Reexport/bTpYr_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/GPUArraysCore/qiYUe_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/share/julia/compiled/v1.10/SuiteSparse_jll/ME9At_SBKrC.so 104074: 104074: find library=libamd.so.3 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libamd.so.3 104074: 104074: find library=libsuitesparseconfig.so.7 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libsuitesparseconfig.so.7 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libsuitesparseconfig.so.7 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libamd.so.3 104074: 104074: find library=libbtf.so.2 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libbtf.so.2 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libbtf.so.2 104074: 104074: find library=libcamd.so.3 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcamd.so.3 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcamd.so.3 104074: 104074: find library=libccolamd.so.3 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libccolamd.so.3 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libccolamd.so.3 104074: 104074: find library=libcholmod.so.4 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcholmod.so.4 104074: 104074: find library=libcolamd.so.3 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcolamd.so.3 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcolamd.so.3 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcholmod.so.4 104074: 104074: find library=libklu.so.2 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libklu.so.2 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libklu.so.2 104074: 104074: find library=libldl.so.3 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libldl.so.3 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libldl.so.3 104074: 104074: find library=librbio.so.4 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/librbio.so.4 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/librbio.so.4 104074: 104074: find library=libspqr.so.4 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libspqr.so.4 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libspqr.so.4 104074: 104074: find library=libumfpack.so.6 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libumfpack.so.6 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libumfpack.so.6 104074: 104074: 104074: calling init: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/share/julia/compiled/v1.10/SparseArrays/P9ieR_SBKrC.so 104074: 104074: find library=libcholmod [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcholmod 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/../libcholmod 104074: search cache=/etc/ld.so.cache 104074: search path=/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/lib:/usr/lib (system search path) 104074: trying file=/lib/x86_64-linux-gnu/libcholmod 104074: trying file=/usr/lib/x86_64-linux-gnu/libcholmod 104074: trying file=/lib/libcholmod 104074: trying file=/usr/lib/libcholmod 104074: 104074: find library=libcholmod.so [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcholmod.so 104074: 104074: find library=libsuitesparseconfig [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libsuitesparseconfig 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/../libsuitesparseconfig 104074: search cache=/etc/ld.so.cache 104074: search path=/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/lib:/usr/lib (system search path) 104074: trying file=/lib/x86_64-linux-gnu/libsuitesparseconfig 104074: trying file=/usr/lib/x86_64-linux-gnu/libsuitesparseconfig 104074: trying file=/lib/libsuitesparseconfig 104074: trying file=/usr/lib/libsuitesparseconfig 104074: 104074: find library=libsuitesparseconfig.so [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libsuitesparseconfig.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Statistics/ERcPL_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/GPUArrays/v5u0T_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Requires/IyxeS_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/BFloat16s/iiZ8G_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/BFloat16sExt/B6KwG_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/LLVMLoopInfo/X7sqD_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/CUDA_Driver_jll/QJyk7_NzezC.so 104074: 104074: find library=libcuda.so.1 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcuda.so.1 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/../libcuda.so.1 104074: search cache=/etc/ld.so.cache 104074: trying file=/lib/x86_64-linux-gnu/libcuda.so.1 104074: 104074: 104074: calling init: /lib/x86_64-linux-gnu/libcuda.so.1 104074: 104074: 104074: calling fini: /lib/x86_64-linux-gnu/libcuda.so.1 [0] 104074: 104074: find library=libcuda.so.1 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcuda.so.1 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/../libcuda.so.1 104074: search cache=/etc/ld.so.cache 104074: trying file=/lib/x86_64-linux-gnu/libcuda.so.1 104074: 104074: find library=libcuda.so.1 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcuda.so.1 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/../libcuda.so.1 104074: search cache=/etc/ld.so.cache 104074: trying file=/lib/x86_64-linux-gnu/libcuda.so.1 104074: 104074: 104074: calling init: /lib/x86_64-linux-gnu/libcuda.so.1 104074: 104074: 104074: calling fini: /lib/x86_64-linux-gnu/libcuda.so.1 [0] 104074: 104074: find library=libcuda.so.1 [0]; searching 104074: search path=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia:/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/.. (RUNPATH from file /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/julia) 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libcuda.so.1 104074: trying file=/home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/../libcuda.so.1 104074: search cache=/etc/ld.so.cache 104074: trying file=/lib/x86_64-linux-gnu/libcuda.so.1 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/09eba544c107fcbe4c50dc34a32b398dd75d33fb/lib/libcuda.so 104074: 104074: 104074: calling fini: /home/christian/.julia/artifacts/09eba544c107fcbe4c50dc34a32b398dd75d33fb/lib/libcuda.so [0] 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/CUDA_Runtime_jll/Hs50y_NzezC.so 104074: 104074: find library=libcublasLt.so.12 [0]; searching 104074: search path=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/glibc-hwcaps/x86-64-v4:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/glibc-hwcaps/x86-64-v3:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/glibc-hwcaps/x86-64-v2:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/haswell/avx512_1/x86_64:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/haswell/avx512_1:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/haswell/x86_64:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/haswell:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/avx512_1/x86_64:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/avx512_1:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/x86_64:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/haswell/avx512_1/x86_64:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/haswell/avx512_1:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/haswell/x86_64:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/haswell:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/avx512_1/x86_64:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/avx512_1:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/x86_64:/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib (RUNPATH from file /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcublas.so) 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/glibc-hwcaps/x86-64-v4/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/glibc-hwcaps/x86-64-v3/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/glibc-hwcaps/x86-64-v2/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/haswell/avx512_1/x86_64/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/haswell/avx512_1/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/haswell/x86_64/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/haswell/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/avx512_1/x86_64/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/avx512_1/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/x86_64/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/tls/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/haswell/avx512_1/x86_64/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/haswell/avx512_1/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/haswell/x86_64/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/haswell/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/avx512_1/x86_64/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/avx512_1/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/x86_64/libcublasLt.so.12 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcublasLt.so.12 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcublasLt.so.12 104074: 104074: find library=libcuda.so.1 [0]; searching 104074: search path=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib (RUNPATH from file /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcublas.so) 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcuda.so.1 104074: search cache=/etc/ld.so.cache 104074: trying file=/lib/x86_64-linux-gnu/libcuda.so.1 104074: 104074: 104074: calling init: /lib/x86_64-linux-gnu/libcuda.so.1 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcublas.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcudart.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcufft.so 104074: 104074: find library=libutil.so.1 [0]; searching 104074: search cache=/etc/ld.so.cache 104074: trying file=/lib/x86_64-linux-gnu/libutil.so.1 104074: 104074: 104074: calling init: /lib/x86_64-linux-gnu/libutil.so.1 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcupti.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcurand.so 104074: 104074: find library=libnvJitLink.so.12 [0]; searching 104074: search path=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib (RUNPATH from file /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcublas.so) 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libnvJitLink.so.12 104074: 104074: find library=libcusparse.so.12 [0]; searching 104074: search path=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib (RUNPATH from file /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcublas.so) 104074: trying file=/home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcusparse.so.12 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libnvJitLink.so.12 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcusparse.so.12 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcusolver.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libcusolverMg.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libnvPTXCompiler.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libnvperf_host.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libnvperf_target.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libnvrtc.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libnvrtc-builtins.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/de13a04158724d6f718f0d35f2aca24a74b43a19/lib/libnvvm.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/FixedPointNumbers/Ple2W_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/ColorTypes/db21U_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Colors/NKjaT_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/NVTX_jll/a4RQP_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/0b6483311b60bc85d320f64bb66fd2e310916fb1/lib/libnvToolsExt.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/JuliaNVTXCallbacks_jll/EQzo6_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/artifacts/ff17ab283d1f7456123711f21d8b1d7e852f8024/lib/libjulia_nvtx_callbacks.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/NVTX/s1i2n_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/RandomNumbers/pgCpR_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Random123/1imiM_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/DataValueInterfaces/9Lpkp_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/DataAPI/3a8mN_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/IteratorInterfaceExtensions/N0h8q_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/TableTraits/I6SaN_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/OrderedCollections/LtT3J_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Tables/Z804B_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/PrecompileTools/AQ9Mk_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/StringManipulation/4nJQd_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Crayons/TXPcU_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/LaTeXStrings/H4HGh_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/PrettyTables/kRdcL_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Compat/GSFWK_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/CompatLinearAlgebraExt/Zxpzq_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/DataStructures/xKiwJ_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/SortingAlgorithms/6dCmw_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/PooledArrays/vi11X_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Missings/wLfgT_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/InvertedIndices/0P8C6_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/SentinelArrays/uMYVe_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Parsers/833b9_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/InlineStrings/prmY3_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/DataFrames/AR9oZ_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/AbstractFFTs/Di3HZ_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/UnsafeAtomics/OuhNJ_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/Atomix/3LdQ4_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/MacroTools/38lnR_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/StaticArraysCore/Tzw28_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/StaticArrays/yY9vm_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/AdaptStaticArraysExt/9bCdf_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/StaticArraysStatisticsExt/EfhbW_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/UnsafeAtomicsLLVM/auDvx_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/KernelAbstractions/aywHT_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/CUDA/oWw5k_NzezC.so 104074: 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: isyscall (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: isyscall (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: ijl_n_threads_per_pool (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: ijl_n_threads_per_pool (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: imemchr (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: imemchr (fatal) 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/CUDNN_jll/wC1gq_NzezC.so 104074: 104074: 104074: calling init: /home/christian/.julia/compiled/v1.10/cuDNN/awbES_NzezC.so 104074: 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: imemset (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: imemset (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: imemset (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: imemset (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: imemset (fatal) 104074: /home/christian/.julia/juliaup/julia-1.10.0-rc2+0.x64.linux.gnu/bin/../lib/julia/libjulia-internal.so.1.10: error: symbol lookup error: undefined symbol: imemset (fatal) ┌ Error: cuDNN is not available for your platform (x86_64-linux-gnu-libgfortran5-cxx11-libstdcxx30-cuda+none-julia_version+1.10.0) └ @ cuDNN ~/.julia/packages/cuDNN/E0AFc/src/cuDNN.jl:172 ```

Does this help?

maleadt commented 11 months ago

That log doesn't contain a segfault, it only throws a Julia error that presumably comes from the fact that precompiling the CUDA runtime failed, so no that doesn't really help. I'd be interested in a LD_DEBUG log of a process segfaulting when it's using the forwards-compatibility driver (i.e. without that env var set), to confirm that we aren't accidentally loading parts of the driver from the system still, which would be a potential cause for segfaults.