search

EnzymeAD / Enzyme.jl

Julia bindings for the Enzyme automatic differentiator
https://enzyme.mit.edu
MIT License
456 stars 64 forks source link

CUDA.jl reverse CI broke again #1889

Closed maleadt closed 1 month ago

maleadt commented 1 month ago

https://buildkite.com/julialang/cuda-dot-jl/builds/5439#01922996-97d0-4994-84d1-d50e8895ce21:

extensions/enzyme  (2) |         failed at 2024-09-25T15:58:07.546
Testing finished in 4 minutes, 11 seconds, 982 milliseconds
Worker 2 failed running test extensions/enzyme:
Some tests did not pass: 9 passed, 0 failed, 4 errored, 0 broken.
extensions/enzyme: Error During Test at /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/test/extensions/enzyme.jl:23
  Got exception outside of a @test
  AssertionError: Base.isconcretetype(typ)
  Stacktrace:
    [1] abs_typeof(arg::LLVM.LoadInst, partial::Bool)
      @ Enzyme.Compiler ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/absint.jl:381
    [2] codegen(output::Symbol, job::CompilerJob{Enzyme.Compiler.EnzymeTarget, Enzyme.Compiler.EnzymeCompilerParams}; libraries::Bool, deferred_codegen::Bool, optimize::Bool, toplevel::Bool, strip::Bool, validate::Bool, only_entry::Bool, parent_job::Nothing)
      @ Enzyme.Compiler ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/compiler.jl:8021
    [3] codegen
      @ ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/compiler.jl:7026 [inlined]
    [4] _thunk(job::CompilerJob{Enzyme.Compiler.EnzymeTarget, Enzyme.Compiler.EnzymeCompilerParams}, postopt::Bool)
      @ Enzyme.Compiler ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/compiler.jl:9290
    [5] _thunk
      @ ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/compiler.jl:9290 [inlined]
    [6] cached_compilation
      @ ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/compiler.jl:9331 [inlined]
    [7] thunkbase(ctx::LLVM.Context, mi::Core.MethodInstance, ::Val{0x0000000000007cab}, ::Type{Const{typeof(square!)}}, ::Type{Const{Nothing}}, tt::Type{Tuple{Duplicated{CuArray{Float32, 1, CUDA.DeviceMemory}}}}, ::Val{Enzyme.API.DEM_ForwardMode}, ::Val{1}, ::Val{(false, false)}, ::Val{false}, ::Val{false}, ::Type{FFIABI}, ::Val{true}, ::Val{false})
      @ Enzyme.Compiler ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/compiler.jl:9463
    [8] #s2064#19044
      @ ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/compiler.jl:9600 [inlined]
    [9] var"#s2064#19044"(FA::Any, A::Any, TT::Any, Mode::Any, ModifiedBetween::Any, width::Any, ReturnPrimal::Any, ShadowInit::Any, World::Any, ABI::Any, ErrIfFuncWritten::Any, RuntimeActivity::Any, ::Any, ::Type, ::Type, ::Type, tt::Any, ::Type, ::Type, ::Type, ::Type, ::Type, ::Type, ::Type, ::Any)
      @ Enzyme.Compiler ./none:0
   [10] (::Core.GeneratedFunctionStub)(::UInt64, ::LineNumberNode, ::Any, ::Vararg{Any})
      @ Core ./boot.jl:602
   [11] autodiff
      @ ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/Enzyme.jl:625 [inlined]
   [12] autodiff
      @ ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/Enzyme.jl:529 [inlined]
   [13] autodiff(mode::ForwardMode{false, FFIABI, false, false}, f::typeof(square!), args::Duplicated{CuArray{Float32, 1, CUDA.DeviceMemory}})
      @ Enzyme ~/.cache/julia-buildkite-plugin/depots/3cc01fab-3357-4a7a-9294-cde2d3115a97/dev/Enzyme/src/Enzyme.jl:496
   [14] macro expansion
      @ /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/test/extensions/enzyme.jl:28 [inlined]
   [15] macro expansion
      @ ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.10/julia-1.10-latest-linux-x86_64/share/julia/stdlib/v1.10/Test/src/Test.jl:1577 [inlined]
   [16] top-level scope
      @ /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/test/extensions/enzyme.jl:24
   [17] include
      @ ./client.jl:489 [inlined]
   [18] #11
      @ /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/test/runtests.jl:87 [inlined]
   [19] macro expansion
      @ /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/test/setup.jl:63 [inlined]
   [20] macro expansion
      @ ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.10/julia-1.10-latest-linux-x86_64/share/julia/stdlib/v1.10/Test/src/Test.jl:1577 [inlined]
   [21] macro expansion
      @ /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/test/setup.jl:63 [inlined]
   [22] macro expansion
      @ /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/src/utilities.jl:35 [inlined]
   [23] macro expansion
      @ /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/src/memory.jl:829 [inlined]
   [24] top-level scope
      @ /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/test/setup.jl:62
   [25] eval
      @ ./boot.jl:385 [inlined]
   [26] runtests(f::Function, name::String, time_source::Symbol)
      @ Main /var/lib/buildkite-agent/builds/gpuci-13/julialang/cuda-dot-jl/test/setup.jl:74
   [27] invokelatest(::Any, ::Any, ::Vararg{Any}; kwargs::@Kwargs{})
      @ Base ./essentials.jl:892
   [28] invokelatest(::Any, ::Any, ::Vararg{Any})
      @ Base ./essentials.jl:889
   [29] (::Distributed.var"#110#112"{Distributed.CallMsg{:call_fetch}})()
      @ Distributed ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.10/julia-1.10-latest-linux-x86_64/share/julia/stdlib/v1.10/Distributed/src/process_messages.jl:287
   [30] run_work_thunk(thunk::Distributed.var"#110#112"{Distributed.CallMsg{:call_fetch}}, print_error::Bool)
      @ Distributed ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.10/julia-1.10-latest-linux-x86_64/share/julia/stdlib/v1.10/Distributed/src/process_messages.jl:70
   [31] (::Distributed.var"#109#111"{Distributed.CallMsg{:call_fetch}, Distributed.MsgHeader, Sockets.TCPSocket})()
      @ Distributed ~/.cache/julia-buildkite-plugin/julia_installs/bin/linux/x64/1.10/julia-1.10-latest-linux-x86_64/share/julia/stdlib/v1.10/Distributed/src/process_messages.jl:287

https://github.com/JuliaGPU/CUDA.jl/commit/819e440a8321695363cd1ef848dba8d3cf8c40f3

wsmoses commented 1 month ago

yeah I just noticed this as well, investigating

wsmoses commented 1 month ago

I believe main should now be fixed.

The presumably breaking change was committed yesterday https://github.com/EnzymeAD/Enzyme.jl/pull/1884 and seemingly fixed today https://github.com/EnzymeAD/Enzyme.jl/pull/1890