search

EnzymeAD / Enzyme.jl

Julia bindings for the Enzyme automatic differentiator
https://enzyme.mit.edu
MIT License
422 stars 58 forks source link

EnzymeTestUtils : fails on GPU Buffers #1427

Closed wsmoses closed 1 month ago

wsmoses commented 1 month ago

@sethaxen I think this should just simply be adding an extension to enzymetestutils for gpuarrays?

mind taking a look?

log


test_reverse: scatter! with return activity Const on (::typeof(-), Const), (::ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer}, BatchDuplicated), (::ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer}, BatchDuplicated), (::ROCArray{Int64, 1, AMDGPU.Runtime.Mem.HIPBuffer}, Const): Error During Test at /root/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/test_reverse.jl:89
--
  | Got exception outside of a @test
  | MethodError: no method matching Base.Threads.Atomic(::Int64)
  |  
  | Closest candidates are:
  | Base.Threads.Atomic()
  | @ Base atomics.jl:81
  |  
  | Stacktrace:
  | [1] map_fields_recursive(f::Function, x::Base.Threads.Atomic{Int64})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:9
  | [2] (::EnzymeTestUtils.var"#30#31"{Base.Fix1{typeof(EnzymeTestUtils.rand_tangent), TaskLocalRNG}})(xi::Base.Threads.Atomic{Int64})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:7
  | [3] map(f::EnzymeTestUtils.var"#30#31"{Base.Fix1{typeof(EnzymeTestUtils.rand_tangent), TaskLocalRNG}}, t::Tuple{AMDGPU.Runtime.Mem.HIPBuffer, typeof(AMDGPU._free_buf), Base.Threads.Atomic{Int64}})
  | @ Base ./tuple.jl:275
  | [4] map(::Function, ::NamedTuple{(:obj, :finalizer, :count), Tuple{AMDGPU.Runtime.Mem.HIPBuffer, typeof(AMDGPU._free_buf), Base.Threads.Atomic{Int64}}})
  | @ Base ./namedtuple.jl:219
  | [5] map_fields_recursive(f::Function, x::GPUArrays.RefCounted{AMDGPU.Runtime.Mem.HIPBuffer})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:6
  | [6] (::EnzymeTestUtils.var"#30#31"{Base.Fix1{typeof(EnzymeTestUtils.rand_tangent), TaskLocalRNG}})(xi::GPUArrays.RefCounted{AMDGPU.Runtime.Mem.HIPBuffer})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:7
  | [7] map
  | @ ./tuple.jl:274 [inlined]
  | [8] map(::Function, ::NamedTuple{(:rc, :freed), Tuple{GPUArrays.RefCounted{AMDGPU.Runtime.Mem.HIPBuffer}, Bool}})
  | @ Base ./namedtuple.jl:219
  | [9] map_fields_recursive(f::Function, x::GPUArrays.DataRef{AMDGPU.Runtime.Mem.HIPBuffer})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:6
  | [10] (::EnzymeTestUtils.var"#30#31"{Base.Fix1{typeof(EnzymeTestUtils.rand_tangent), TaskLocalRNG}})(xi::GPUArrays.DataRef{AMDGPU.Runtime.Mem.HIPBuffer})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:7
  | [11] map(f::EnzymeTestUtils.var"#30#31"{Base.Fix1{typeof(EnzymeTestUtils.rand_tangent), TaskLocalRNG}}, t::Tuple{GPUArrays.DataRef{AMDGPU.Runtime.Mem.HIPBuffer}, Tuple{Int64, Int64}, Int64})
  | @ Base ./tuple.jl:275
  | [12] map(::Function, ::NamedTuple{(:buf, :dims, :offset), Tuple{GPUArrays.DataRef{AMDGPU.Runtime.Mem.HIPBuffer}, Tuple{Int64, Int64}, Int64}})
  | @ Base ./namedtuple.jl:219
  | [13] map_fields_recursive(f::Function, x::ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:6
  | [14] rand_tangent(rng::TaskLocalRNG, x::ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:20
  | [15] rand_tangent(x::ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:19
  | [16] (::EnzymeTestUtils.var"#34#35"{ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer}})(#unused#::Int64)
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:42
  | [17] ntuple
  | @ ./ntuple.jl:19 [inlined]
  | [18] _build_activity(primal::ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer}, #unused#::Type{BatchDuplicated})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:42
  | [19] auto_activity(arg::Tuple{ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer}, UnionAll})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/generate_tangent.jl:31
  | [20] map
  | @ ./tuple.jl:275 [inlined]
  | [21] map
  | @ ./tuple.jl:276 [inlined]
  | [22] map(f::typeof(EnzymeTestUtils.auto_activity), t::Tuple{typeof(scatter!), Tuple{typeof(-), UnionAll}, Tuple{ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer}, UnionAll}, Tuple{ROCArray{Float64, 2, AMDGPU.Runtime.Mem.HIPBuffer}, UnionAll}, Tuple{ROCArray{Int64, 1, AMDGPU.Runtime.Mem.HIPBuffer}, UnionAll}})
  | @ Base ./tuple.jl:276
  | [23] macro expansion
  | @ ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/test_reverse.jl:91 [inlined]
  | [24] macro expansion
  | @ ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.9/julia-1.9-latest-linux-x86_64/share/julia/stdlib/v1.9/Test/src/Test.jl:1498 [inlined]
  | [25] test_reverse(::Function, ::Type, ::Tuple{typeof(-), UnionAll}, ::Vararg{Any}; fdm::FiniteDifferences.AdaptedFiniteDifferenceMethod{5, 1, FiniteDifferences.UnadaptedFiniteDifferenceMethod{7, 5}}, fkwargs::NamedTuple{(), Tuple{}}, rtol::Float64, atol::Float64, testset_name::Nothing)
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/test_reverse.jl:91
  | [26] test_reverse(::Function, ::Type, ::Tuple{typeof(-), UnionAll}, ::Vararg{Any})
  | @ EnzymeTestUtils ~/.cache/julia-buildkite-plugin/depots/dc18a9a2-eed5-4c7e-b514-fdcbd06a5a91/packages/EnzymeTestUtils/JLwtd/src/test_reverse.jl:74
  | [27] macro expansion
  | @ /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/scatter.jl:227 [inlined]
  | [28] macro expansion
  | @ ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.9/julia-1.9-latest-linux-x86_64/share/julia/stdlib/v1.9/Test/src/Test.jl:1498 [inlined]
  | [29] macro expansion
  | @ /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/scatter.jl:214 [inlined]
  | [30] macro expansion
  | @ ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.9/julia-1.9-latest-linux-x86_64/share/julia/stdlib/v1.9/Test/src/Test.jl:1498 [inlined]
  | [31] scatter_testsuite(Backend::Type{ROCBackend})
  | @ Main /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/scatter.jl:157
  | [32] macro expansion
  | @ /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/runtests.jl:53 [inlined]
  | [33] macro expansion
  | @ ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.9/julia-1.9-latest-linux-x86_64/share/julia/stdlib/v1.9/Test/src/Test.jl:1498 [inlined]
  | [34] nnlib_testsuite(Backend::Type; skip_tests::Set{String})
  | @ Main /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/runtests.jl:30
  | [35] nnlib_testsuite(Backend::Type)
  | @ Main /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/runtests.jl:45
  | [36] macro expansion
  | @ /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/runtests.jl:150 [inlined]
  | [37] macro expansion
  | @ ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.9/julia-1.9-latest-linux-x86_64/share/julia/stdlib/v1.9/Test/src/Test.jl:1498 [inlined]
  | [38] macro expansion
  | @ /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/runtests.jl:150 [inlined]
  | [39] macro expansion
  | @ ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.9/julia-1.9-latest-linux-x86_64/share/julia/stdlib/v1.9/Test/src/Test.jl:1498 [inlined]
  | [40] top-level scope
  | @ /var/lib/buildkite-agent/builds/leleka-pxlth-1/julialang/nnlib-dot-jl/test/runtests.jl:59
  | [41] include(fname::String)
  | @ Base.MainInclude ./client.jl:478
  | [42] top-level scope
  | @ none:6
  | [43] eval
  | @ ./boot.jl:370 [inlined]
  | [44] exec_options(opts::Base.JLOptions)
  | @ Base ./client.jl:280
  | [45] _start()
  | @ Base ./client.jl:522

from https://buildkite.com/julialang/nnlib-dot-jl/builds/1076#018ae5a9-762d-4f8d-9692-de592b97f01e

sethaxen commented 1 month ago

I'm traveling right now and won't be able to look at it until at least June. Also, I don't really have experience with GPU arrays in Julia. But if no one gets to it before I get back, I can take a look at it.

wsmoses commented 1 month ago

Apparently nnlib passes on amdgpu now, so closing