search

JuliaGPU / CUDA.jl

CUDA programming in Julia.
https://juliagpu.org/cuda/
Other
1.2k stars 218 forks source link

GC integration: multiple threads fail to free each others objects, lead to OOM #1522

Open lmh91 opened 2 years ago

lmh91 commented 2 years ago

The bug

GPU memory is not freed (fast enough?) when performing a little memory requiring computation in parallel on multiple threads.

MWE

I came across this issue when using a multi layer ML model via Flux.jl on a GPU in an multi-threaded optimizer. I was able to reproduce the issue with only CUDA.jl and Adapt.jl:

using CUDA
using Adapt

input_dims = 256
n = 1024
N = 100
T = Float32
x = ones(T, input_dims, N);
n_evals = 10000

l1 = (
    W = adapt(CuArray{T}, rand(n, input_dims)),
    b = adapt(CuArray{T}, rand(n))
);
l2 = (
    W = adapt(CuArray{T}, rand(1, n)),
    b = adapt(CuArray{T}, rand(1))
);
layers = (l1, l2);

sizeoflayer(l) = sizeof(l.W) + sizeof(l.b)
heavy_pseudo_computation(l::NamedTuple, x) = l.W * x .+ l.b

layers_memory_size = sum(sizeoflayer.(layers))
@info "$(round(layers_memory_size / 1024^2, digits = 3)) MiB GPU memory used by permanent layers.\n"
layers_memory_size == CUDA.used_memory()
println()
@info "GPU memory usage prior to any evaluation"
CUDA.memory_status(); println()

function eval_model(layers, X::Matrix{T}) where {T}
    X_gpu = adapt(CuArray{T}, X)
    Y_l1 = heavy_pseudo_computation(layers[1], X_gpu)
    Y_l2 = heavy_pseudo_computation(layers[2], Y_l1)
    Y_cpu = adapt(Array, Y_l2)
    return Y_cpu
end

eval_model(layers, x);

println()
@info "GPU memory usage after 1 evaluation"
CUDA.memory_status(); println()

for i in 1:10*n_evals-1 
    eval_model(layers, x)
end
println()
@info "GPU memory usage after $(n_evals) evaluations"
CUDA.memory_status(); println()

println()
@info "Now using $(Base.Threads.nthreads()) threads"
Base.Threads.@threads for i in 1:10*n_evals-1 
    # This results in GPU OOM
    eval_model(layers, x)
end

My output of running the script via julia --project=. main.jl: 

julia --project=. --threads=16 --startup-file=no main.jl 
[ Info: 1.008 MiB GPU memory used by permanent layers.

[ Info: GPU memory usage prior to any evaluation
Effective GPU memory usage: 3.99% (446.125 MiB/10.917 GiB)
Memory pool usage: 1.008 MiB (32.000 MiB reserved)

[ Info: GPU memory usage after 1 evaluation
Effective GPU memory usage: 4.96% (554.125 MiB/10.917 GiB)
Memory pool usage: 1.887 MiB (32.000 MiB reserved)

[ Info: GPU memory usage after 10000 evaluations
Effective GPU memory usage: 65.64% (7.166 GiB/10.917 GiB)
Memory pool usage: 997.184 MiB (6.656 GiB reserved)

[ Info: Now using 16 threads
ERROR: LoadError: TaskFailedException
Stacktrace:
 [1] wait
   @ ./task.jl:334 [inlined]
 [2] threading_run(func::Function)
   @ Base.Threads ./threadingconstructs.jl:38
 [3] top-level scope
   @ ./threadingconstructs.jl:97

    nested task error: Out of GPU memory trying to allocate 400.000 KiB
    Effective GPU memory usage: 99.75% (10.889 GiB/10.917 GiB)
    Memory pool usage: 10.247 GiB (10.250 GiB reserved)
    Stacktrace:
Manifest.toml

``` # This file is machine-generated - editing it directly is not advised julia_version = "1.7.2" manifest_format = "2.0" [[deps.AbstractFFTs]] deps = ["ChainRulesCore", "LinearAlgebra"] git-tree-sha1 = "6f1d9bc1c08f9f4a8fa92e3ea3cb50153a1b40d4" uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" version = "1.1.0" [[deps.Adapt]] deps = ["LinearAlgebra"] git-tree-sha1 = "af92965fb30777147966f58acb05da51c5616b5f" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" version = "3.3.3" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [[deps.BFloat16s]] deps = ["LinearAlgebra", "Printf", "Random", "Test"] git-tree-sha1 = "a598ecb0d717092b5539dbbe890c98bac842b072" uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" version = "0.2.0" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[deps.CEnum]] git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" version = "0.4.2" [[deps.CUDA]] deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"] git-tree-sha1 = "19fb33957a5f85efb3cc10e70cf4dd4e30174ac9" uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" version = "3.10.0" [[deps.ChainRulesCore]] deps = ["Compat", "LinearAlgebra", "SparseArrays"] git-tree-sha1 = "9950387274246d08af38f6eef8cb5480862a435f" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" version = "1.14.0" [[deps.ChangesOfVariables]] deps = ["ChainRulesCore", "LinearAlgebra", "Test"] git-tree-sha1 = "1e315e3f4b0b7ce40feded39c73049692126cf53" uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" version = "0.1.3" [[deps.Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] git-tree-sha1 = "b153278a25dd42c65abbf4e62344f9d22e59191b" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" version = "3.43.0" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[deps.DelimitedFiles]] deps = ["Mmap"] uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" [[deps.Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[deps.DocStringExtensions]] deps = ["LibGit2"] git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.8.6" [[deps.Downloads]] deps = ["ArgTools", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" [[deps.ExprTools]] git-tree-sha1 = "56559bbef6ca5ea0c0818fa5c90320398a6fbf8d" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" version = "0.1.8" [[deps.GPUArrays]] deps = ["Adapt", "LLVM", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"] git-tree-sha1 = "c783e8883028bf26fb05ed4022c450ef44edd875" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" version = "8.3.2" [[deps.GPUCompiler]] deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "TimerOutputs", "UUIDs"] git-tree-sha1 = "d8c5999631e1dc18d767883f621639c838f8e632" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" version = "0.15.2" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[deps.InverseFunctions]] deps = ["Test"] git-tree-sha1 = "336cc738f03e069ef2cac55a104eb823455dca75" uuid = "3587e190-3f89-42d0-90ee-14403ec27112" version = "0.1.4" [[deps.IrrationalConstants]] git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151" uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" version = "0.1.1" [[deps.JLLWrappers]] deps = ["Preferences"] git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" version = "1.4.1" [[deps.LLVM]] deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] git-tree-sha1 = "c8d47589611803a0f3b4813d9e267cd4e3dbcefb" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" version = "4.11.1" [[deps.LLVMExtra_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg", "TOML"] git-tree-sha1 = "771bfe376249626d3ca12bcd58ba243d3f961576" uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" version = "0.0.16+0" [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" [[deps.LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [[deps.LinearAlgebra]] deps = ["Libdl", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.LogExpFunctions]] deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"] git-tree-sha1 = "09e4b894ce6a976c354a69041a04748180d43637" uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" version = "0.3.15" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" version = "0.5.5+0" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[deps.Preferences]] deps = ["TOML"] git-tree-sha1 = "47e5f437cc0e7ef2ce8406ce1e7e24d44915f88d" uuid = "21216c6a-2e73-6563-6e65-726566657250" version = "1.3.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" [[deps.REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[deps.Random]] deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.Random123]] deps = ["Random", "RandomNumbers"] git-tree-sha1 = "afeacaecf4ed1649555a19cb2cad3c141bbc9474" uuid = "74087812-796a-5b5d-8853-05524746bad3" version = "1.5.0" [[deps.RandomNumbers]] deps = ["Random", "Requires"] git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" version = "1.5.3" [[deps.Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" uuid = "189a3867-3050-52da-a836-e630ba90ab69" version = "1.2.2" [[deps.Requires]] deps = ["UUIDs"] git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" uuid = "ae029012-a4dd-5104-9daa-d747884805df" version = "1.3.0" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" [[deps.SharedArrays]] deps = ["Distributed", "Mmap", "Random", "Serialization"] uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[deps.SparseArrays]] deps = ["LinearAlgebra", "Random"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.SpecialFunctions]] deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] git-tree-sha1 = "bc40f042cfcc56230f781d92db71f0e21496dffd" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" version = "2.1.5" [[deps.Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[deps.TimerOutputs]] deps = ["ExprTools", "Printf"] git-tree-sha1 = "7638550aaea1c9a1e86817a231ef0faa9aca79bd" uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" version = "0.5.19" [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" ```

Expected behavior

The "permanent" GPU memories (the layers) and the function eval_model only uses a very small amount of the GPU memory. Even when performing the function in parallel on multiple threads there should not be any GPU memory issue (with the size of the arrays in the above MWE). However, a OOM error is produced.

Version info

Details on Julia:

Julia Version 1.7.2
Commit bf53498635 (2022-02-06 15:21 UTC)
Platform Info:
  OS: Linux (x86_64-pc-linux-gnu)
  CPU: AMD EPYC 7702P 64-Core Processor
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-12.0.1 (ORCJIT, znver2)
Environment:
  JULIA_CUDA_USE_BINARYBUILDER = false
  JULIA_CXX_RTTI = 1
  JULIA_NUM_THREADS = auto

Details on CUDA:

CUDA toolkit 11.3, local installation
NVIDIA driver 510.47.3, for CUDA 11.6
CUDA driver 11.6

Libraries: 
- CUBLAS: 11.5.1
- CURAND: 10.2.4
- CUFFT: 10.4.2
- CUSOLVER: 11.1.2
- CUSPARSE: 11.6.0
- CUPTI: 14.0.0
- NVML: 11.0.0+510.47.3
- CUDNN: 8.20.0 (for CUDA 11.3.0)
- CUTENSOR: missing

Toolchain:
- Julia: 1.7.2
- LLVM: 12.0.1
- PTX ISA support: 3.2, 4.0, 4.1, 4.2, 4.3, 5.0, 6.0, 6.1, 6.3, 6.4, 6.5, 7.0
- Device capability support: sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80

Environment:
- JULIA_CUDA_USE_BINARYBUILDER: false

4 devices:
  0: NVIDIA GeForce RTX 3090 (sm_86, 23.651 GiB / 24.000 GiB available)
  1: NVIDIA GeForce RTX 3090 (sm_86, 23.680 GiB / 24.000 GiB available)
  2: NVIDIA GeForce RTX 3090 (sm_86, 23.680 GiB / 24.000 GiB available)
  3: NVIDIA GeForce RTX 3090 (sm_86, 23.062 GiB / 24.000 GiB available)
maleadt commented 2 years ago

After calling the function eval_model the amount of used GPU memory should be the same as before.

That's a wrong expectation. For one, memory allocations are garbage collected so it might take a while before they get freed, but secondly there's a caching layer in libcuda which will make that device memory usage looks higher while the memory is available for reuse. That's what the very line below what you're pointing at explains.

oschulz commented 2 years ago

@maleadt but why the overloading error - shouldn't GPU memory be GC'ed automatically if necessary?

maleadt commented 2 years ago

The issue didn't demonstrate an actual OOM, so I'm guessing that statement was a hypothetical? Either way, it shouldn't OOM, our allocator will forcibly free memory (by calling the GC) if some is needed.

oschulz commented 2 years ago

@lmh91 could you change your example to demo the OOM errors we've encountered?

lmh91 commented 2 years ago

I tried to make the MWE example as small as possible and it seems I actually removed the important part which seems to create the OOM error in my use case: using multiple threads. I updated the bug report and MWE. Sorry for that...

When running the for-loop single-threaded no OOM error is produced. (Though the "effective GPU memory" is 99,99% very quickly even if one iteration does not require much memory which I find a bit strange.)

The OOM error is produced running the loop over multiple threads via Base.Threads.@threads.

maleadt commented 2 years ago

Thanks, I can reproduce using this MWE. Will have a look.

maleadt commented 2 years ago

MWE:

using CUDA

function main()
    Threads.@threads for i in 1:100000
        CuArray{Float32}(undef, (1024, 100))
        nothing
    end
end

isinteractive() || main()

This looks like us calling into the GC being broken when using threads.

oschulz commented 2 years ago

I can reproduce using this MWE. Will have a look.

Thanks @maleadt !

maleadt commented 2 years ago

Even smaller:

using CUDA

function main()
    Threads.@threads for i in 1:30
        CuArray{UInt8}(undef, (1024, 1024, 1024))   # 1 GiB
        nothing
    end
end

isinteractive() || main()

OOMs on -t2, not on -t1. Looks like our calls to GC.gc are ineffective, or at least insufficient to free another thread's dead objects:

t1: try alloc 1024.000 MiB
t2: try alloc 1024.000 MiB
t1: alloc CuPtr{Nothing}(0x0000000302000000)
t1: try alloc 1024.000 MiB
t2: alloc CuPtr{Nothing}(0x0000000342000000)
t2: try alloc 1024.000 MiB
t1: alloc CuPtr{Nothing}(0x0000000382000000)
t1: try alloc 1024.000 MiB
t2: alloc CuPtr{Nothing}(0x00000003c2000000)
t2: try alloc 1024.000 MiB
t1: alloc CuPtr{Nothing}(0x0000000402000000)
t1: try alloc 1024.000 MiB
t2: alloc CuPtr{Nothing}(0x0000000442000000)
t2: try alloc 1024.000 MiB
t1: alloc CuPtr{Nothing}(0x0000000482000000)
t1: try alloc 1024.000 MiB
t2: alloc CuPtr{Nothing}(0x00000004c2000000)
t2: try alloc 1024.000 MiB
t1: alloc CuPtr{Nothing}(0x0000000502000000)
t1: try alloc 1024.000 MiB
t2: alloc CuPtr{Nothing}(0x0000000542000000)
t2: try alloc 1024.000 MiB
t1: alloc CuPtr{Nothing}(0x0000000582000000)
t1: try alloc 1024.000 MiB
t2: alloc CuPtr{Nothing}(0x00000005c2000000)
t2: try alloc 1024.000 MiB
t1: alloc CuPtr{Nothing}(0x0000000602000000)
t1: try alloc 1024.000 MiB
t2: alloc CuPtr{Nothing}(0x0000000642000000)
t2: try alloc 1024.000 MiB
t1: alloc re-try 1
t2: alloc re-try 1
t1: alloc re-try 2
t2: alloc re-try 2
t1: alloc re-try 3
t2: alloc re-try 3
t2: alloc re-try 4
t2: alloc re-try 5
t2: alloc re-try 6
t2: alloc failed
t1: free CuPtr{Nothing}(0x0000000642000000)
t1: free CuPtr{Nothing}(0x00000005c2000000)
t1: free CuPtr{Nothing}(0x0000000542000000)
t1: free CuPtr{Nothing}(0x00000004c2000000)
t1: free CuPtr{Nothing}(0x0000000442000000)
t1: free CuPtr{Nothing}(0x00000003c2000000)
t1: free CuPtr{Nothing}(0x0000000342000000)
t1: free CuPtr{Nothing}(0x0000000602000000)
t1: free CuPtr{Nothing}(0x0000000582000000)
t1: free CuPtr{Nothing}(0x0000000502000000)
t1: free CuPtr{Nothing}(0x0000000482000000)
t1: free CuPtr{Nothing}(0x0000000402000000)
t1: free CuPtr{Nothing}(0x0000000382000000)
t1: free CuPtr{Nothing}(0x0000000302000000)

During those retries, we do incrementally more effort to free memory, including calls to the GC. But as you can see from the trace, those don't free another thread's objects in time. I'm not sure how to proceed here, so I've asked @chflood to have a look.

maleadt commented 2 years ago

MWE without CUDA.jl:

const LIMIT = 14

# dummy atomic allocator
const memory = Threads.Atomic{Int}(0)
function alloc()
    println("thread $(Threads.threadid()): try alloc ($(memory[])/$(LIMIT) used)")
    while true
        old_memory = memory[]
        new_memory = old_memory + 1
        if new_memory > LIMIT
            printstyled("thread $(Threads.threadid()): alloc failure\n"; color=:yellow)
            return false
        end
        if Threads.atomic_cas!(memory, old_memory, new_memory) == old_memory
            println("thread $(Threads.threadid()): alloc success")
            return true
        end
    end
end
function free()
    printstyled("thread $(Threads.threadid()): free ($(memory[])/$(LIMIT) used)\n"; color=:green)
    while true
        old_memory = memory[]
        new_memory = old_memory - 1
        @assert new_memory >= 0
        if Threads.atomic_cas!(memory, old_memory, new_memory) == old_memory
            return
        end
    end
end

# dummy array
mutable struct CuArray
    function CuArray()
        success = alloc()
        if !success
            printstyled("thread $(Threads.threadid()): GC.gc(false)\n"; color=:magenta)
            GC.gc(false)
            success = alloc()
        end

        if !success
            printstyled("thread $(Threads.threadid()): GC.gc(true)\n"; color=:magenta)
            GC.gc(true)
            success = alloc()
        end

        if !success
            printstyled("thread $(Threads.threadid()): alloc really failed\n"; color=:red)
            throw(OutOfMemoryError())
        end

        obj = new()
        finalizer(obj) do _
            free()
        end
    end
end

function main()
    Threads.@threads for i in 1:30
        CuArray()
        nothing
    end
end
isinteractive() || main()
Alexander-Barth commented 10 months ago

I am seeing also an GPU OOM while training a neural network with CUDA.jl 5.1.1 and julia 1.9.4 and 2 threads (works fine with 1 thread). The CUDA MWE also fails on my system (NVIDIA A100-SXM4-40GB).

Is there any known work-around? I already tried to downgrade CUDA.jl, but without success. Thank you for your time!

Alexander-Barth commented 10 months ago

I am trying to add CUDA.reclaim(), but this leads to this error:

error in running finalizer: ArgumentError(msg="Attempt to release freed data.") error in running finalizer: ArgumentError(msg="Attempt to release freed data.") release at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/GPUArrays/dAUOE/src/host/abstractarray.jl:38 unsafe_free! at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/GPUArrays/dAUOE/src/host/abstractarray.jl:90 [inlined] unsafe_finalize! at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/CUDA/YIj5X/src/array.jl:113 unknown function (ip: 0x7f0fa044c512) _jl_invoke at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined] ijl_apply_generic at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940 run_finalizer at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gc.c:417 jl_gc_run_finalizers_in_list at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gc.c:507 run_finalizers at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gc.c:553 jl_mutex_unlock at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/julia_locks.h:81 [inlined] jl_generate_fptr_impl at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/jitlayers.cpp:467 jl_compile_method_internal at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2348 [inlined] jl_compile_method_internal at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2237 _jl_invoke at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2750 [inlined] ijl_apply_generic at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940 release at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/GPUArrays/dAUOE/src/host/abstractarray.jl:42 unsafe_free! at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/GPUArrays/dAUOE/src/host/abstractarray.jl:90 [inlined] unsafe_free! at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/CUDA/YIj5X/src/array.jl:112 [inlined] #scan!#1158 at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/CUDA/YIj5X/src/accumulate.jl:194 unknown function (ip: 0x7f0fa0427b1c) unknown function (ip: 0x7f0fa041cc69) unknown function (ip: 0x7f0fa041cc2e) scan! at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/CUDA/YIj5X/src/accumulate.jl:135 [inlined] _accumulate! at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/CUDA/YIj5X/src/accumulate.jl:203 [inlined] #accumulate!#877 at ./accumulate.jl:340 [inlined] accumulate! at ./accumulate.jl:337 [inlined] _cumsum! at ./accumulate.jl:61 [inlined] #cumsum!#869 at ./accumulate.jl:51 [inlined] cumsum! at ./accumulate.jl:49 [inlined] #cumsum#870 at ./accumulate.jl:113 [inlined] cumsum at ./accumulate.jl:111 [inlined] cumsum at ./accumulate.jl:144 [inlined] findall at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/CUDA/YIj5X/src/indexing.jl:25 to_index at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/CUDA/YIj5X/src/indexing.jl:14 [inlined] _to_indices1 at ./indices.jl:359 [inlined] to_indices at ./indices.jl:354 [inlined] to_indices at ./indices.jl:345 [inlined] view at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/GPUArrays/dAUOE/src/host/base.jl:312 [inlined] maybeview at ./views.jl:148 unknown function (ip: 0x7f0fa041f26b) _jl_invoke at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined] ijl_apply_generic at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940 dotview at ./broadcast.jl:1214 [inlined] getobs at /gpfs/home/acad/ulg-gher/abarth/projects-test-orig/Julia/share/diffusion_model.jl:452 getobs at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/MLUtils/n3C0h/src/obsview.jl:187 [inlined] _getbatch at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/MLUtils/n3C0h/src/batchview.jl:144 [inlined] getindex at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/MLUtils/n3C0h/src/batchview.jl:129 [inlined] getobs at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/MLUtils/n3C0h/src/observation.jl:110 [inlined] getobs at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/SimpleTraits/l1ZsK/src/SimpleTraits.jl:331 [inlined] #58 at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/MLUtils/n3C0h/src/parallel.jl:66 unknown function (ip: 0x7f14bc0ccc4c) _jl_invoke at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined] ijl_apply_generic at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940 macro expansion at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/MLUtils/n3C0h/src/parallel.jl:124 [inlined] ##reducing_function#293#68 at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/FLoops/6PVny/src/reduce.jl:817 [inlined] AdjoinIdentity at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/InitialValues/OWP8V/src/InitialValues.jl:306 next at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/Transducers/KcCBR/src/combinators.jl:290 [inlined] next at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/Transducers/KcCBR/src/core.jl:287 [inlined] macro expansion at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/Transducers/KcCBR/src/core.jl:181 [inlined] _foldl_array at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/Transducers/KcCBR/src/processes.jl:187 [inlined] __foldl__ at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/Transducers/KcCBR/src/processes.jl:182 [inlined] foldl_basecase at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/Transducers/KcCBR/src/processes.jl:361 [inlined] _reduce_basecase at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/Transducers/KcCBR/src/threading_utils.jl:58 _reduce at /gpfs/home/acad/ulg-gher/abarth/.julia/packages/Transducers/KcCBR/src/reduce.jl:139 #177 at ./threadingconstructs.jl:416 unknown function (ip: 0x7f14bc0ccc7f) _jl_invoke at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2758 [inlined] ijl_apply_generic at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/gf.c:2940 jl_apply at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/julia.h:1880 [inlined] start_task at /cache/build/builder-amdci4-4/julialang/julia-release-1-dot-9/src/task.c:1092 nb_parameters 2359169 WARNING: Error while freeing DeviceBuffer(240.000 KiB at 0x0000000c27443600): ErrorException("task switch not allowed from inside gc finalizer")
Manifest.toml # This file is machine-generated - editing it directly is not advised julia_version = "1.9.4" manifest_format = "2.0" project_hash = "c060f414b955daa2bf33d6e016d861292570ea4d" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" version = "1.5.0" weakdeps = ["ChainRulesCore", "Test"] [deps.AbstractFFTs.extensions] AbstractFFTsChainRulesCoreExt = "ChainRulesCore" AbstractFFTsTestExt = "Test" [[deps.Adapt]] deps = ["LinearAlgebra", "Requires"] git-tree-sha1 = "02f731463748db57cc2ebfbd9fbc9ce8280d3433" uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" version = "3.7.1" weakdeps = ["StaticArrays"] [deps.Adapt.extensions] AdaptStaticArraysExt = "StaticArrays" [[deps.ArgCheck]] git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4" uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" version = "2.3.0" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" version = "1.1.1" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [[deps.Atomix]] deps = ["UnsafeAtomics"] git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" version = "0.1.0" [[deps.BFloat16s]] deps = ["LinearAlgebra", "Printf", "Random", "Test"] git-tree-sha1 = "dbf84058d0a8cbbadee18d25cf606934b22d7c66" uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" version = "0.4.2" [[deps.BSON]] git-tree-sha1 = "2208958832d6e1b59e49f53697483a84ca8d664e" uuid = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" version = "0.3.7" [[deps.BangBang]] deps = ["Compat", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires", "Setfield", "Tables"] git-tree-sha1 = "e28912ce94077686443433c2800104b061a827ed" uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" version = "0.3.39" [deps.BangBang.extensions] BangBangChainRulesCoreExt = "ChainRulesCore" BangBangDataFramesExt = "DataFrames" BangBangStaticArraysExt = "StaticArrays" BangBangStructArraysExt = "StructArrays" BangBangTypedTablesExt = "TypedTables" [deps.BangBang.weakdeps] ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[deps.Baselet]] git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" uuid = "9718e550-a3fa-408a-8086-8db961cd8217" version = "0.1.1" [[deps.Bzip2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2" uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" version = "1.0.8+0" [[deps.CEnum]] git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90" uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" version = "0.4.2" [[deps.CFTime]] deps = ["Dates", "Printf"] git-tree-sha1 = "ed2e76c1c3c43fd9d0cb9248674620b29d71f2d1" uuid = "179af706-886a-5703-950a-314cd64e0468" version = "0.1.2" [[deps.CUDA]] deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "Crayons", "DataFrames", "ExprTools", "GPUArrays", "GPUCompiler", "KernelAbstractions", "LLVM", "LLVMLoopInfo", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "NVTX", "Preferences", "PrettyTables", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "Statistics", "UnsafeAtomicsLLVM"] git-tree-sha1 = "76582ae19006b1186e87dadd781747f76cead72c" uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" version = "5.1.1" weakdeps = ["ChainRulesCore", "SpecialFunctions"] [deps.CUDA.extensions] ChainRulesCoreExt = "ChainRulesCore" SpecialFunctionsExt = "SpecialFunctions" [[deps.CUDA_Driver_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] git-tree-sha1 = "1e42ef1bdb45487ff28de16182c0df4920181dc3" uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc" version = "0.7.0+0" [[deps.CUDA_Runtime_Discovery]] deps = ["Libdl"] git-tree-sha1 = "bcc4a23cbbd99c8535a5318455dcf0f2546ec536" uuid = "1af6417a-86b4-443c-805f-a4643ffb695f" version = "0.2.2" [[deps.CUDA_Runtime_jll]] deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] git-tree-sha1 = "9704e50c9158cf8896c2776b8dbc5edd136caf80" uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2" version = "0.10.1+0" [[deps.CUDNN_jll]] deps = ["Artifacts", "CUDA_Runtime_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] git-tree-sha1 = "75923dce4275ead3799b238e10178a68c07dbd3b" uuid = "62b44479-cb7b-5706-934f-f13b2eb2e645" version = "8.9.4+0" [[deps.ChainRules]] deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "SparseInverseSubset", "Statistics", "StructArrays", "SuiteSparse"] git-tree-sha1 = "006cc7170be3e0fa02ccac6d4164a1eee1fc8c27" uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" version = "1.58.0" [[deps.ChainRulesCore]] deps = ["Compat", "LinearAlgebra"] git-tree-sha1 = "e0af648f0692ec1691b5d094b8724ba1346281cf" uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" version = "1.18.0" weakdeps = ["SparseArrays"] [deps.ChainRulesCore.extensions] ChainRulesCoreSparseArraysExt = "SparseArrays" [[deps.ColorTypes]] deps = ["FixedPointNumbers", "Random"] git-tree-sha1 = "eb7f0f8307f71fac7c606984ea5fb2817275d6e4" uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" version = "0.11.4" [[deps.Colors]] deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] git-tree-sha1 = "fc08e5930ee9a4e03f84bfb5211cb54e7769758a" uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" version = "0.12.10" [[deps.CommonDataModel]] deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf"] git-tree-sha1 = "7f5717cbb2c1ce650cfd454451f282df33103596" uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" version = "0.2.5" [[deps.CommonSubexpressions]] deps = ["MacroTools", "Test"] git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" version = "0.3.0" [[deps.Compat]] deps = ["UUIDs"] git-tree-sha1 = "886826d76ea9e72b35fcd000e535588f7b60f21d" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" version = "4.10.1" weakdeps = ["Dates", "LinearAlgebra"] [deps.Compat.extensions] CompatLinearAlgebraExt = "LinearAlgebra" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" version = "1.0.5+0" [[deps.CompositionsBase]] git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" version = "0.1.2" [deps.CompositionsBase.extensions] CompositionsBaseInverseFunctionsExt = "InverseFunctions" [deps.CompositionsBase.weakdeps] InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" [[deps.ConstructionBase]] deps = ["LinearAlgebra"] git-tree-sha1 = "c53fc348ca4d40d7b371e71fd52251839080cbc9" uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" version = "1.5.4" [deps.ConstructionBase.extensions] ConstructionBaseIntervalSetsExt = "IntervalSets" ConstructionBaseStaticArraysExt = "StaticArrays" [deps.ConstructionBase.weakdeps] IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" [[deps.ContextVariablesX]] deps = ["Compat", "Logging", "UUIDs"] git-tree-sha1 = "25cc3803f1030ab855e383129dcd3dc294e322cc" uuid = "6add18c4-b38d-439d-96f6-d6bc489c04c5" version = "0.1.3" [[deps.Crayons]] git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" version = "4.1.1" [[deps.DataAPI]] git-tree-sha1 = "8da84edb865b0b5b0100c0666a9bc9a0b71c553c" uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" version = "1.15.0" [[deps.DataFrames]] deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] git-tree-sha1 = "04c738083f29f86e62c8afc341f0967d8717bdb8" uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" version = "1.6.1" [[deps.DataStructures]] deps = ["Compat", "InteractiveUtils", "OrderedCollections"] git-tree-sha1 = "3dbd312d370723b6bb43ba9d02fc36abade4518d" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" version = "0.18.15" [[deps.DataValueInterfaces]] git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" version = "1.0.0" [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[deps.DefineSingletons]] git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" version = "0.1.2" [[deps.DelimitedFiles]] deps = ["Mmap"] git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" version = "1.9.1" [[deps.DiffResults]] deps = ["StaticArraysCore"] git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" version = "1.1.0" [[deps.DiffRules]] deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" version = "1.15.1" [[deps.DiskArrays]] deps = ["OffsetArrays"] git-tree-sha1 = "1bfa9de80f35ac63c6c381b2d43c590875896a1f" uuid = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" version = "0.3.22" [[deps.Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[deps.DocStringExtensions]] deps = ["LibGit2"] git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.9.3" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" version = "1.6.0" [[deps.ExprTools]] git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" version = "0.1.10" [[deps.FLoops]] deps = ["BangBang", "Compat", "FLoopsBase", "InitialValues", "JuliaVariables", "MLStyle", "Serialization", "Setfield", "Transducers"] git-tree-sha1 = "ffb97765602e3cbe59a0589d237bf07f245a8576" uuid = "cc61a311-1640-44b5-9fba-1b764f453329" version = "0.2.1" [[deps.FLoopsBase]] deps = ["ContextVariablesX"] git-tree-sha1 = "656f7a6859be8673bf1f35da5670246b923964f7" uuid = "b9860ae5-e623-471e-878b-f6a53c775ea6" version = "0.1.1" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[deps.FillArrays]] deps = ["LinearAlgebra", "Random"] git-tree-sha1 = "01dba5dbad6b2766e2ddd7b9d64af0e6d68d95cd" uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" version = "1.9.1" [deps.FillArrays.extensions] FillArraysPDMatsExt = "PDMats" FillArraysSparseArraysExt = "SparseArrays" FillArraysStatisticsExt = "Statistics" [deps.FillArrays.weakdeps] PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [[deps.FixedPointNumbers]] deps = ["Statistics"] git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" version = "0.8.4" [[deps.Flux]] deps = ["Adapt", "ChainRulesCore", "Functors", "LinearAlgebra", "MLUtils", "MacroTools", "NNlib", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote"] git-tree-sha1 = "b97c3fc4f3628b8835d83789b09382961a254da4" uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" version = "0.14.6" [deps.Flux.extensions] FluxAMDGPUExt = "AMDGPU" FluxCUDAExt = "CUDA" FluxCUDAcuDNNExt = ["CUDA", "cuDNN"] FluxMetalExt = "Metal" [deps.Flux.weakdeps] AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" Metal = "dde4c033-4e86-420c-a63e-0dd931031962" cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" [[deps.ForwardDiff]] deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] git-tree-sha1 = "cf0fe81336da9fb90944683b8c41984b08793dad" uuid = "f6369f11-7733-5829-9624-2563aa707210" version = "0.10.36" weakdeps = ["StaticArrays"] [deps.ForwardDiff.extensions] ForwardDiffStaticArraysExt = "StaticArrays" [[deps.Functors]] deps = ["LinearAlgebra"] git-tree-sha1 = "9a68d75d466ccc1218d0552a8e1631151c569545" uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" version = "0.4.5" [[deps.Future]] deps = ["Random"] uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[deps.GPUArrays]] deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] git-tree-sha1 = "85d7fb51afb3def5dcb85ad31c3707795c8bccc1" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" version = "9.1.0" [[deps.GPUArraysCore]] deps = ["Adapt"] git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" uuid = "46192b85-c4d5-4398-a991-12ede77f4527" version = "0.1.5" [[deps.GPUCompiler]] deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Scratch", "TimerOutputs", "UUIDs"] git-tree-sha1 = "a846f297ce9d09ccba02ead0cae70690e072a119" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" version = "0.25.0" [[deps.HDF5_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739" uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" version = "1.14.2+1" [[deps.Hwloc_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "8ecb0b34472a3c98f945e3c75fc7d5428d165511" uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" version = "2.9.3+0" [[deps.IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] git-tree-sha1 = "8aa91235360659ca7560db43a7d57541120aa31d" uuid = "7869d1d1-7146-5819-86e3-90919afe41df" version = "0.4.11" [[deps.InitialValues]] git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" version = "0.3.1" [[deps.InlineStrings]] deps = ["Parsers"] git-tree-sha1 = "9cc2baf75c6d09f9da536ddf58eb2f29dedaf461" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" version = "1.4.0" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[deps.InvertedIndices]] git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" version = "1.3.0" [[deps.IrrationalConstants]] git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" version = "0.2.2" [[deps.IteratorInterfaceExtensions]] git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" uuid = "82899510-4779-5014-852e-03e436cf321d" version = "1.0.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" version = "1.5.0" [[deps.JSON3]] deps = ["Dates", "Mmap", "Parsers", "PrecompileTools", "StructTypes", "UUIDs"] git-tree-sha1 = "95220473901735a0f4df9d1ca5b171b568b2daa3" uuid = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" version = "1.13.2" [[deps.JuliaNVTXCallbacks_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "af433a10f3942e882d3c671aacb203e006a5808f" uuid = "9c1d0b0a-7046-5b2e-a33f-ea22f176ac7e" version = "0.2.1+0" [[deps.JuliaVariables]] deps = ["MLStyle", "NameResolution"] git-tree-sha1 = "49fb3cb53362ddadb4415e9b73926d6b40709e70" uuid = "b14d175d-62b4-44ba-8fb7-3064adc8c3ec" version = "0.2.4" [[deps.KernelAbstractions]] deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "Requires", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] git-tree-sha1 = "b0737cbbe1c8da6f1139d1c23e35e7cea129c0af" uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" version = "0.9.13" [deps.KernelAbstractions.extensions] EnzymeExt = "EnzymeCore" [deps.KernelAbstractions.weakdeps] EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" [[deps.LLVM]] deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Requires", "Unicode"] git-tree-sha1 = "0678579657515e88b6632a3a482d39adcbb80445" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" version = "6.4.1" weakdeps = ["BFloat16s"] [deps.LLVM.extensions] BFloat16sExt = "BFloat16s" [[deps.LLVMExtra_jll]] deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] git-tree-sha1 = "98eaee04d96d973e79c25d49167668c5c8fb50e2" uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" version = "0.0.27+1" [[deps.LLVMLoopInfo]] git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea" uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586" version = "1.0.0" [[deps.LLVMOpenMP_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "f689897ccbe049adb19a065c495e75f372ecd42b" uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" version = "15.0.4+0" [[deps.LaTeXStrings]] git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" version = "1.3.1" [[deps.LazyArtifacts]] deps = ["Artifacts", "Pkg"] uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" version = "8.4.0+0" [[deps.LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" version = "1.11.0+1" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "f9557a255370125b405568f9767d6d195822a175" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" version = "1.17.0+0" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.LogExpFunctions]] deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] git-tree-sha1 = "7d6dd4e9212aebaeed356de34ccf262a3cd415aa" uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" version = "0.3.26" [deps.LogExpFunctions.extensions] LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" LogExpFunctionsInverseFunctionsExt = "InverseFunctions" [deps.LogExpFunctions.weakdeps] ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.MLStyle]] git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8" uuid = "d8e11817-5142-5d16-987a-aa16d5891078" version = "0.4.17" [[deps.MLUtils]] deps = ["ChainRulesCore", "Compat", "DataAPI", "DelimitedFiles", "FLoops", "NNlib", "Random", "ShowCases", "SimpleTraits", "Statistics", "StatsBase", "Tables", "Transducers"] git-tree-sha1 = "3504cdb8c2bc05bde4d4b09a81b01df88fcbbba0" uuid = "f1d291b0-491e-4a28-83b9-f70985020b54" version = "0.4.3" [[deps.MPICH_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] git-tree-sha1 = "8a5b4d2220377d1ece13f49438d71ad20cf1ba83" uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" version = "4.1.2+0" [[deps.MPIPreferences]] deps = ["Libdl", "Preferences"] git-tree-sha1 = "8f6af051b9e8ec597fa09d8885ed79fd582f33c9" uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" version = "0.1.10" [[deps.MPItrampoline_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] git-tree-sha1 = "6979eccb6a9edbbb62681e158443e79ecc0d056a" uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" version = "5.3.1+0" [[deps.MacroTools]] deps = ["Markdown", "Random"] git-tree-sha1 = "9ee1618cbf5240e6d4e0371d6f24065083f60c48" uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" version = "0.5.11" [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" version = "2.28.2+0" [[deps.MicroCollections]] deps = ["BangBang", "InitialValues", "Setfield"] git-tree-sha1 = "629afd7d10dbc6935ec59b32daeb33bc4460a42e" uuid = "128add7d-3638-4c79-886c-908ea0c25c34" version = "0.1.4" [[deps.MicrosoftMPI_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "b01beb91d20b0d1312a9471a36017b5b339d26de" uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" version = "10.1.4+1" [[deps.Missings]] deps = ["DataAPI"] git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" version = "1.1.0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" version = "2022.10.11" [[deps.NCDatasets]] deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] git-tree-sha1 = "7fcb4378f9c648a186bcb996fa29acc929a179ed" uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" version = "0.13.1" [[deps.NNlib]] deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Pkg", "Random", "Requires", "Statistics"] git-tree-sha1 = "ac86d2944bf7a670ac8bf0f7ec099b5898abcc09" uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" version = "0.9.8" [deps.NNlib.extensions] NNlibAMDGPUExt = "AMDGPU" NNlibCUDACUDNNExt = ["CUDA", "cuDNN"] NNlibCUDAExt = "CUDA" NNlibEnzymeCoreExt = "EnzymeCore" [deps.NNlib.weakdeps] AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" [[deps.NVTX]] deps = ["Colors", "JuliaNVTXCallbacks_jll", "Libdl", "NVTX_jll"] git-tree-sha1 = "8bc9ce4233be3c63f8dcd78ccaf1b63a9c0baa34" uuid = "5da4648a-3479-48b8-97b9-01cb529c0a1f" version = "0.3.3" [[deps.NVTX_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "ce3269ed42816bf18d500c9f63418d4b0d9f5a3b" uuid = "e98f9f5b-d649-5603-91fd-7774390e6439" version = "3.1.0+2" [[deps.NaNMath]] deps = ["OpenLibm_jll"] git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4" uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" version = "1.0.2" [[deps.NameResolution]] deps = ["PrettyPrint"] git-tree-sha1 = "1a0fa0e9613f46c9b8c11eee38ebb4f590013c5e" uuid = "71a1bf82-56d0-4bbc-8a3c-48b961074391" version = "0.1.5" [[deps.NetCDF_jll]] deps = ["Artifacts", "Bzip2_jll", "HDF5_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "XML2_jll", "Zlib_jll", "Zstd_jll"] git-tree-sha1 = "10c612c81eaffdd6b7c28a45a554cdd9d2f40ff1" uuid = "7243133f-43d8-5620-bbf4-c2c921802cf3" version = "400.902.208+0" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.2.0" [[deps.OffsetArrays]] deps = ["Adapt"] git-tree-sha1 = "2ac17d29c523ce1cd38e27785a7d23024853a4bb" uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" version = "1.12.10" [[deps.OneHotArrays]] deps = ["Adapt", "ChainRulesCore", "Compat", "GPUArraysCore", "LinearAlgebra", "NNlib"] git-tree-sha1 = "5e4029759e8699ec12ebdf8721e51a659443403c" uuid = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" version = "0.2.4" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" version = "0.3.21+4" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" version = "0.8.1+0" [[deps.OpenMPI_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "PMIx_jll", "TOML", "Zlib_jll", "libevent_jll", "prrte_jll"] git-tree-sha1 = "694458ae803b684f09c07f90459cb79655fb377d" uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" version = "5.0.0+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "cc6e1927ac521b659af340e0ca45828a3ffc748f" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" version = "3.0.12+0" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" version = "0.5.5+0" [[deps.Optimisers]] deps = ["ChainRulesCore", "Functors", "LinearAlgebra", "Random", "Statistics"] git-tree-sha1 = "34205b1204cc83c43cd9cfe53ffbd3b310f6e8c5" uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" version = "0.3.1" [[deps.OrderedCollections]] git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" version = "1.6.3" [[deps.PMIx_jll]] deps = ["Artifacts", "Hwloc_jll", "JLLWrappers", "Libdl", "Zlib_jll", "libevent_jll"] git-tree-sha1 = "8b3b19351fa24791f94d7ae85faf845ca1362541" uuid = "32165bc3-0280-59bc-8c0b-c33b6203efab" version = "4.2.7+0" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] git-tree-sha1 = "a935806434c9d4c506ba941871b327b96d41f2bf" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "2.8.0" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" version = "1.9.2" [[deps.PooledArrays]] deps = ["DataAPI", "Future"] git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" version = "1.4.3" [[deps.PrecompileTools]] deps = ["Preferences"] git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" version = "1.2.0" [[deps.Preferences]] deps = ["TOML"] git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e" uuid = "21216c6a-2e73-6563-6e65-726566657250" version = "1.4.1" [[deps.PrettyPrint]] git-tree-sha1 = "632eb4abab3449ab30c5e1afaa874f0b98b586e4" uuid = "8162dcfd-2161-5ef2-ae6c-7681170c5f98" version = "0.2.0" [[deps.PrettyTables]] deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] git-tree-sha1 = "88b895d13d53b5577fd53379d913b9ab9ac82660" uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" version = "2.3.1" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" [[deps.ProgressLogging]] deps = ["Logging", "SHA", "UUIDs"] git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" version = "0.1.4" [[deps.REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[deps.Random]] deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[deps.Random123]] deps = ["Random", "RandomNumbers"] git-tree-sha1 = "552f30e847641591ba3f39fd1bed559b9deb0ef3" uuid = "74087812-796a-5b5d-8853-05524746bad3" version = "1.6.1" [[deps.RandomNumbers]] deps = ["Random", "Requires"] git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" version = "1.5.3" [[deps.RealDot]] deps = ["LinearAlgebra"] git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" version = "0.1.0" [[deps.Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" uuid = "189a3867-3050-52da-a836-e630ba90ab69" version = "1.2.2" [[deps.Requires]] deps = ["UUIDs"] git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" uuid = "ae029012-a4dd-5104-9daa-d747884805df" version = "1.3.0" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" [[deps.Scratch]] deps = ["Dates"] git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" uuid = "6c6a2e73-6563-6170-7368-637461726353" version = "1.2.1" [[deps.SentinelArrays]] deps = ["Dates", "Random"] git-tree-sha1 = "0e7508ff27ba32f26cd459474ca2ede1bc10991f" uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" version = "1.4.1" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" [[deps.Setfield]] deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" version = "1.1.1" [[deps.ShowCases]] git-tree-sha1 = "7f534ad62ab2bd48591bdeac81994ea8c445e4a5" uuid = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" version = "0.1.0" [[deps.SimpleTraits]] deps = ["InteractiveUtils", "MacroTools"] git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" version = "0.9.4" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[deps.SortingAlgorithms]] deps = ["DataStructures"] git-tree-sha1 = "5165dfb9fd131cf0c6957a3a7605dede376e7b63" uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" version = "1.2.0" [[deps.SparseArrays]] deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.SparseInverseSubset]] deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] git-tree-sha1 = "91402087fd5d13b2d97e3ef29bbdf9d7859e678a" uuid = "dc90abb0-5640-4711-901d-7e5b23a2fada" version = "0.1.1" [[deps.SpecialFunctions]] deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] git-tree-sha1 = "e2cfc4012a19088254b3950b85c3c1d8882d864d" uuid = "276daf66-3868-5448-9aa4-cd146d93841b" version = "2.3.1" weakdeps = ["ChainRulesCore"] [deps.SpecialFunctions.extensions] SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" [[deps.SplittablesBase]] deps = ["Setfield", "Test"] git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" uuid = "171d559e-b47b-412a-8079-5efa626c420e" version = "0.1.15" [[deps.StaticArrays]] deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] git-tree-sha1 = "5ef59aea6f18c25168842bded46b16662141ab87" uuid = "90137ffa-7385-5640-81b9-e52037218182" version = "1.7.0" weakdeps = ["Statistics"] [deps.StaticArrays.extensions] StaticArraysStatisticsExt = "Statistics" [[deps.StaticArraysCore]] git-tree-sha1 = "36b3d696ce6366023a0ea192b4cd442268995a0d" uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" version = "1.4.2" [[deps.Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" version = "1.9.0" [[deps.StatsAPI]] deps = ["LinearAlgebra"] git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" version = "1.7.0" [[deps.StatsBase]] deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] git-tree-sha1 = "1d77abd07f617c4868c33d4f5b9e1dbb2643c9cf" uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" version = "0.34.2" [[deps.StringManipulation]] deps = ["PrecompileTools"] git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" version = "0.3.4" [[deps.StructArrays]] deps = ["Adapt", "ConstructionBase", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] git-tree-sha1 = "0a3db38e4cce3c54fe7a71f831cd7b6194a54213" uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" version = "0.6.16" [[deps.StructTypes]] deps = ["Dates", "UUIDs"] git-tree-sha1 = "ca4bccb03acf9faaf4137a9abc1881ed1841aa70" uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" version = "1.10.0" [[deps.SuiteSparse]] deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" [[deps.SuiteSparse_jll]] deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" version = "5.10.1+6" [[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" version = "1.0.3" [[deps.TableTraits]] deps = ["IteratorInterfaceExtensions"] git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" version = "1.0.1" [[deps.Tables]] deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits"] git-tree-sha1 = "cb76cf677714c095e535e3501ac7954732aeea2d" uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" version = "1.11.1" [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" version = "1.10.0" [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[deps.TimerOutputs]] deps = ["ExprTools", "Printf"] git-tree-sha1 = "f548a9e9c490030e545f72074a41edfd0e5bcdd7" uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" version = "0.5.23" [[deps.Transducers]] deps = ["Adapt", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "ConstructionBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "Setfield", "SplittablesBase", "Tables"] git-tree-sha1 = "e579d3c991938fecbb225699e8f611fa3fbf2141" uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" version = "0.4.79" [deps.Transducers.extensions] TransducersBlockArraysExt = "BlockArrays" TransducersDataFramesExt = "DataFrames" TransducersLazyArraysExt = "LazyArrays" TransducersOnlineStatsBaseExt = "OnlineStatsBase" TransducersReferenceablesExt = "Referenceables" [deps.Transducers.weakdeps] BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" [[deps.UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[deps.UnsafeAtomics]] git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" version = "0.2.1" [[deps.UnsafeAtomicsLLVM]] deps = ["LLVM", "UnsafeAtomics"] git-tree-sha1 = "323e3d0acf5e78a56dfae7bd8928c989b4f3083e" uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" version = "0.1.3" [[deps.XML2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] git-tree-sha1 = "da69178aacc095066bad1f69d2f59a60a1dd8ad1" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" version = "2.12.0+0" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" version = "1.2.13+0" [[deps.Zstd_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "49ce682769cd5de6c72dcf1b94ed7790cd08974c" uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" version = "1.5.5+0" [[deps.Zygote]] deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArrays", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "Requires", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] git-tree-sha1 = "5ded212acd815612df112bb895ef3910c5a03f57" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" version = "0.6.67" [deps.Zygote.extensions] ZygoteColorsExt = "Colors" ZygoteDistancesExt = "Distances" ZygoteTrackerExt = "Tracker" [deps.Zygote.weakdeps] Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" [[deps.ZygoteRules]] deps = ["ChainRulesCore", "MacroTools"] git-tree-sha1 = "9d749cd449fb448aeca4feee9a2f4186dbb5d184" uuid = "700de1a5-db45-46bc-99cf-38207098b444" version = "0.2.4" [[deps.cuDNN]] deps = ["CEnum", "CUDA", "CUDA_Runtime_Discovery", "CUDNN_jll"] git-tree-sha1 = "c092c26591a851083ed3358890d0d916c58dde62" uuid = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" version = "1.2.1" [[deps.libaec_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] git-tree-sha1 = "eddd19a8dea6b139ea97bdc8a0e2667d4b661720" uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" version = "1.0.6+1" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" version = "5.8.0+0" [[deps.libevent_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll"] git-tree-sha1 = "f04ec6d9a186115fb38f858f05c0c4e1b7fc9dcb" uuid = "1080aeaf-3a6a-583e-a51c-c537b09f60ec" version = "2.1.13+1" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" version = "1.52.0+1" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" version = "17.4.0+0" [[deps.prrte_jll]] deps = ["Artifacts", "Hwloc_jll", "JLLWrappers", "Libdl", "PMIx_jll", "libevent_jll"] git-tree-sha1 = "5adb2d7a18a30280feb66cad6f1a1dfdca2dc7b0" uuid = "eb928a42-fffd-568d-ab9c-3f5d54fc65b9" version = "3.0.2+0"
maleadt commented 10 months ago

That is a different issue; what's described here is that our GC calls are ineffective with multiple threads, leading to an OOM. You're describing an error that shouldn't occur. Please file a new issue with an MWE so that I can take a look!