search

FluxML / model-zoo

Please do not feed the models
https://fluxml.ai/
Other
918 stars 333 forks source link

cifar10 fails on GPU #125

Closed tbenst closed 5 years ago

tbenst commented 5 years ago

If I add using CuArrays to the top of the script, I get the following error:

$ julia vision/cifar10/cifar10.jl 
ERROR: LoadError: GPU compilation of #23(CuArrays.CuKernelState, CUDAnative.CuDeviceArray{Bool,1,CUDAnative.AS.Global}, Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CUDAnative.CuDeviceArray{Int64,1,CUDAnative.AS.Global},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}) failed
KernelError: passing and using non-bitstype argument

Argument 4 to your kernel function is of type Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CUDAnative.CuDeviceArray{Int64,1,CUDAnative.AS.Global},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}.
That type is not isbits, and such arguments are only allowed when they are unused by the kernel.

Stacktrace:
 [1] check_invocation(::CUDAnative.CompilerContext, ::LLVM.Function) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/validation.jl:35
 [2] compile(::CUDAnative.CompilerContext) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:94
 [3] #compile#109(::Bool, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::VersionNumber, ::Any, ::Any) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:45
 [4] compile at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:43 [inlined]
 [5] #compile#108(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::CUDAdrv.CuDevice, ::Function, ::Any) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:18
 [6] compile at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:16 [inlined]
 [7] macro expansion at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:269 [inlined]
 [8] #cufunction#123(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.cufunction), ::getfield(GPUArrays, Symbol("##23#24")), ::Type{Tuple{CuArrays.CuKernelState,CUDAnative.CuDeviceArray{Bool,1,CUDAnative.AS.Global},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CUDAnative.CuDeviceArray{Int64,1,CUDAnative.AS.Global},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}}}) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:240
 [9] cufunction(::Function, ::Type) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:240
 [10] macro expansion at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:208 [inlined]
 [11] macro expansion at ./gcutils.jl:87 [inlined]
 [12] macro expansion at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:205 [inlined]
 [13] _gpu_call(::CuArrays.CuArrayBackend, ::Function, ::CuArray{Bool,1}, ::Tuple{CuArray{Bool,1},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CuArray{Int64,1},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}}, ::Tuple{Tuple{Int64},Tuple{Int64}}) at /home/tyler/.julia/packages/CuArrays/qZCAt/src/gpuarray_interface.jl:59
 [14] gpu_call(::Function, ::CuArray{Bool,1}, ::Tuple{CuArray{Bool,1},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CuArray{Int64,1},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}}, ::Int64) at /home/tyler/.julia/packages/GPUArrays/t8tJB/src/abstract_gpu_interface.jl:151
 [15] gpu_call at /home/tyler/.julia/packages/GPUArrays/t8tJB/src/abstract_gpu_interface.jl:128 [inlined]
 [16] copyto! at /home/tyler/.julia/packages/GPUArrays/t8tJB/src/broadcast.jl:48 [inlined]
 [17] copyto! at ./broadcast.jl:797 [inlined]
 [18] copy(::Base.Broadcast.Broadcasted{Base.Broadcast.ArrayStyle{CuArray},Tuple{Base.OneTo{Int64}},typeof(==),Tuple{CuArray{Int64,1},Array{Int64,1}}}) at ./broadcast.jl:773
 [19] materialize(::Base.Broadcast.Broadcasted{Base.Broadcast.ArrayStyle{CuArray},Nothing,typeof(==),Tuple{CuArray{Int64,1},Array{Int64,1}}}) at ./broadcast.jl:753
 [20] accuracy(::CuArray{Float32,4}, ::Flux.OneHotMatrix{CuArray{Flux.OneHotVector,1}}) at /home/tyler/code/model-zoo/vision/cifar10/cifar10.jl:114
 [21] macro expansion at ./show.jl:555 [inlined]
 [22] (::getfield(Main, Symbol("##33#34")))() at /home/tyler/code/model-zoo/vision/cifar10/cifar10.jl:118
 [23] (::getfield(Flux, Symbol("##throttled#10#14")){Bool,Bool,getfield(Main, Symbol("##33#34")),Int64})(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function) at /home/tyler/.julia/packages/Flux/lz7S9/src/utils.jl:120
 [24] throttled at /home/tyler/.julia/packages/Flux/lz7S9/src/utils.jl:116 [inlined]
 [25] macro expansion at /home/tyler/.julia/packages/Flux/lz7S9/src/optimise/train.jl:75 [inlined]
 [26] macro expansion at /home/tyler/.julia/packages/Juno/TfNYn/src/progress.jl:133 [inlined]
 [27] #train!#12(::getfield(Flux, Symbol("#throttled#18")){getfield(Flux, Symbol("##throttled#10#14")){Bool,Bool,getfield(Main, Symbol("##33#34")),Int64}}, ::Function, ::Function, ::Tracker.Params, ::Array{Tuple{CuArray{Float32,4},Flux.OneHotMatrix{CuArray{Flux.OneHotVector,1}}},1}, ::ADAM) at /home/tyler/.julia/packages/Flux/lz7S9/src/optimise/train.jl:69
 [28] (::getfield(Flux.Optimise, Symbol("#kw##train!")))(::NamedTuple{(:cb,),Tuple{getfield(Flux, Symbol("#throttled#18")){getfield(Flux, Symbol("##throttled#10#14")){Bool,Bool,getfield(Main, Symbol("##33#34")),Int64}}}}, ::typeof(Flux.Optimise.train!), ::Function, ::Tracker.Params, ::Array{Tuple{CuArray{Float32,4},Flux.OneHotMatrix{CuArray{Flux.OneHotVector,1}}},1}, ::ADAM) at ./none:0
 [29] top-level scope at none:0
 [30] include at ./boot.jl:326 [inlined]
 [31] include_relative(::Module, ::String) at ./loading.jl:1038
 [32] include(::Module, ::String) at ./sysimg.jl:29
 [33] exec_options(::Base.JLOptions) at ./client.jl:267
 [34] _start() at ./client.jl:436
in expression starting at /home/tyler/code/model-zoo/vision/cifar10/cifar10.jl:124

Edit: hm, same problem on vision/mnist/conv.jl. The code does hit the GPU for a couple seconds however according to watch nvidia-smi

$ julia vision/mnist/conv.jl 
[ Info: activating new environment at ~/code/model-zoo/cuda.
  Updating registry at `~/.julia/registries/General`
  Updating git-repo `https://github.com/JuliaRegistries/General.git`
 Resolving package versions...
[ Info: Loading data set
[ Info: Constructing model...
[ Info: Beginning training loop...
ERROR: LoadError: GPU compilation of #23(CuArrays.CuKernelState, CUDAnative.CuDeviceArray{Bool,1,CUDAnative.AS.Global}, Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CUDAnative.CuDeviceArray{Int64,1,CUDAnative.AS.Global},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}) failed
KernelError: passing and using non-bitstype argument

Argument 4 to your kernel function is of type Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CUDAnative.CuDeviceArray{Int64,1,CUDAnative.AS.Global},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}.
That type is not isbits, and such arguments are only allowed when they are unused by the kernel.

Stacktrace:
 [1] check_invocation(::CUDAnative.CompilerContext, ::LLVM.Function) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/validation.jl:35
 [2] compile(::CUDAnative.CompilerContext) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:94
 [3] #compile#109(::Bool, ::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::VersionNumber, ::Any, ::Any) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:45
 [4] compile at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:43 [inlined]
 [5] #compile#108(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::Function, ::CUDAdrv.CuDevice, ::Function, ::Any) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:18
 [6] compile at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/compiler/driver.jl:16 [inlined]
 [7] macro expansion at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:269 [inlined]
 [8] #cufunction#123(::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}, ::typeof(CUDAnative.cufunction), ::getfield(GPUArrays, Symbol("##23#24")), ::Type{Tuple{CuArrays.CuKernelState,CUDAnative.CuDeviceArray{Bool,1,CUDAnative.AS.Global},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CUDAnative.CuDeviceArray{Int64,1,CUDAnative.AS.Global},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}}}) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:240
 [9] cufunction(::Function, ::Type) at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:240
 [10] macro expansion at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:208 [inlined]
 [11] macro expansion at ./gcutils.jl:87 [inlined]
 [12] macro expansion at /home/tyler/.julia/packages/CUDAnative/PFgO3/src/execution.jl:205 [inlined]
 [13] _gpu_call(::CuArrays.CuArrayBackend, ::Function, ::CuArray{Bool,1}, ::Tuple{CuArray{Bool,1},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CuArray{Int64,1},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}}, ::Tuple{Tuple{Int64},Tuple{Int64}}) at /home/tyler/.julia/packages/CuArrays/qZCAt/src/gpuarray_interface.jl:59
 [14] gpu_call(::Function, ::CuArray{Bool,1}, ::Tuple{CuArray{Bool,1},Base.Broadcast.Broadcasted{Nothing,Tuple{Base.OneTo{Int64}},typeof(==),Tuple{Base.Broadcast.Extruded{CuArray{Int64,1},Tuple{Bool},Tuple{Int64}},Base.Broadcast.Extruded{Array{Int64,1},Tuple{Bool},Tuple{Int64}}}}}, ::Int64) at /home/tyler/.julia/packages/GPUArrays/t8tJB/src/abstract_gpu_interface.jl:151
 [15] gpu_call at /home/tyler/.julia/packages/GPUArrays/t8tJB/src/abstract_gpu_interface.jl:128 [inlined]
 [16] copyto! at /home/tyler/.julia/packages/GPUArrays/t8tJB/src/broadcast.jl:48 [inlined]
 [17] copyto! at ./broadcast.jl:797 [inlined]
 [18] copy(::Base.Broadcast.Broadcasted{Base.Broadcast.ArrayStyle{CuArray},Tuple{Base.OneTo{Int64}},typeof(==),Tuple{CuArray{Int64,1},Array{Int64,1}}}) at ./broadcast.jl:773
 [19] materialize(::Base.Broadcast.Broadcasted{Base.Broadcast.ArrayStyle{CuArray},Nothing,typeof(==),Tuple{CuArray{Int64,1},Array{Int64,1}}}) at ./broadcast.jl:753
 [20] accuracy(::CuArray{Float32,4}, ::Flux.OneHotMatrix{CuArray{Flux.OneHotVector,1}}) at /home/tyler/code/model-zoo/vision/mnist/conv.jl:85
 [21] top-level scope at /home/tyler/code/model-zoo/vision/mnist/conv.jl:100 [inlined]
 [22] top-level scope at ./none:0
 [23] include at ./boot.jl:326 [inlined]
 [24] include_relative(::Module, ::String) at ./loading.jl:1038
 [25] include(::Module, ::String) at ./sysimg.jl:29
 [26] exec_options(::Base.JLOptions) at ./client.jl:267
 [27] _start() at ./client.jl:436
in expression starting at /home/tyler/code/model-zoo/vision/mnist/conv.jl:94

I'm not sure if this is a problem on ModelZoo or elsewhere. I'm on CUDA 10.0 and CUDNN 7.3.1. Here's the tests from CuArrays:

Test Summary:                          | Pass  Error  Total
CuArrays                               | 4011      4   4015
  GPUArrays test suite                 | 1023          1023
  Memory                               |    5             5
  Array                                |   19            19
  Adapt                                |    2             2
  Broadcast                            |   10            10
  Cufunc                               |    6             6
  Ref Broadcast                        |    1             1
  Broadcast Fix                        |    4             4
  Reduce                               |    6             6
  0D                                   |    2             2
  Slices                               |   17            17
  Reshape                              |    1             1
  LinearAlgebra.triu! with diagonal -2 |    1             1
  LinearAlgebra.triu! with diagonal -1 |    1             1
  LinearAlgebra.triu! with diagonal 0  |    1             1
  LinearAlgebra.triu! with diagonal 1  |    1             1
  LinearAlgebra.triu! with diagonal 2  |    1             1
  LinearAlgebra.tril! with diagonal -2 |    1             1
  LinearAlgebra.tril! with diagonal -1 |    1             1
  LinearAlgebra.tril! with diagonal 0  |    1             1
  LinearAlgebra.tril! with diagonal 1  |    1             1
  LinearAlgebra.tril! with diagonal 2  |    1             1
  Utilities                            |    2             2
  accumulate                           |    8             8
  logical indexing                     |   15            15
  CUDNN                                |   41            41
  CUBLAS                               | 1164          1164
  CUSPARSE                             | 1140          1140
  CUSOLVER                             |  269      4    273
    elty = Float32                     |   67      1     68
      Cholesky (po)                    |    8             8
      getrf!                           |           1      1
      getrs!                           |    3             3
      geqrf!                           |    1             1
      ormqr!                           |    4             4
      orgqr!                           |    2             2
      sytrf!                           |    4             4
      gebrd!                           |    4             4
      syevd!                           |    3             3
      sygvd!                           |    5             5
      syevj!                           |    4             4
      svd with QRAlgorithm method      |    7             7
      svd with QRAlgorithm method      |    1             1
      svd with JacobiAlgorithm method  |    7             7
      svd with JacobiAlgorithm method  |    7             7
      qr                               |    7             7
    elty = Float64                     |   67      1     68
      Cholesky (po)                    |    8             8
      getrf!                           |           1      1
      getrs!                           |    3             3
      geqrf!                           |    1             1
      ormqr!                           |    4             4
      orgqr!                           |    2             2
      sytrf!                           |    4             4
      gebrd!                           |    4             4
      syevd!                           |    3             3
      sygvd!                           |    5             5
      syevj!                           |    4             4
      svd with QRAlgorithm method      |    7             7
      svd with QRAlgorithm method      |    1             1
      svd with JacobiAlgorithm method  |    7             7
      svd with JacobiAlgorithm method  |    7             7
      qr                               |    7             7
    elty = Complex{Float32}            |   67      1     68
      Cholesky (po)                    |    8             8
      getrf!                           |           1      1
      getrs!                           |    3             3
      geqrf!                           |    1             1
      ormqr!                           |    4             4
      orgqr!                           |    2             2
      sytrf!                           |    4             4
      gebrd!                           |    4             4
      syevd!                           |    3             3
      sygvd!                           |    5             5
      syevj!                           |    4             4
      svd with QRAlgorithm method      |    7             7
      svd with QRAlgorithm method      |    1             1
      svd with JacobiAlgorithm method  |    7             7
      svd with JacobiAlgorithm method  |    7             7
      qr                               |    7             7
    elty = Complex{Float64}            |   67      1     68
      Cholesky (po)                    |    8             8
      getrf!                           |           1      1
      getrs!                           |    3             3
      geqrf!                           |    1             1
      ormqr!                           |    4             4
      orgqr!                           |    2             2
      sytrf!                           |    4             4
      gebrd!                           |    4             4
      syevd!                           |    3             3
      sygvd!                           |    5             5
      syevj!                           |    4             4
      svd with QRAlgorithm method      |    7             7
      svd with QRAlgorithm method      |    1             1
      svd with JacobiAlgorithm method  |    7             7
      svd with JacobiAlgorithm method  |    7             7
      qr                               |    7             7
  CUFFT                                |  150           150
  CURAND                               |   32            32
  CUSPARSE + CUSOLVER                  |   84            84
ERROR: LoadError: Some tests did not pass: 4011 passed, 0 failed, 4 errored, 0 broken.
in expression starting at /home/tyler/.julia/packages/CuArrays/qZCAt/test/runtests.jl:25
ERROR: Package CuArrays errored during testing

And here's the test results for Flux:

[ Info: Testing Flux/CUDNN
batch_size = 1: Error During Test at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:7
  Got exception outside of a @test
  CUDNNError(code 3, CUDNN_STATUS_BAD_PARAM)
  Stacktrace:
   [1] macro expansion at /home/tyler/.julia/packages/CuArrays/qZCAt/src/dnn/error.jl:19 [inlined]
   [2] cudnnRNNBackwardData(::Flux.CUDA.RNNDesc{Float32}, ::Int64, ::Array{CuArrays.CUDNN.TensorDesc,1}, ::CuArray{Float32,1}, ::Array{CuArrays.CUDNN.TensorDesc,1}, ::CuArray{Float32,1}, ::CuArrays.CUDNN.TensorDesc, ::CuArray{Float32,1}, ::Ptr{Nothing}, ::CUDAdrv.CuPtr{Nothing}, ::CuArrays.CUDNN.FilterDesc, ::CuArray{Float32,1}, ::CuArrays.CUDNN.TensorDesc, ::CuArray{Float32,1}, ::Ptr{Nothing}, ::CUDAdrv.CuPtr{Nothing}, ::Array{CuArrays.CUDNN.TensorDesc,1}, ::CuArray{Float32,1}, ::CuArrays.CUDNN.TensorDesc, ::CuArray{Float32,1}, ::Ptr{Nothing}, ::CUDAdrv.CuPtr{Nothing}, ::CuArray{UInt8,1}, ::CuArray{UInt8,1}) at /home/tyler/.julia/packages/Flux/lz7S9/src/cuda/curnn.jl:170
   [3] backwardData(::Flux.CUDA.RNNDesc{Float32}, ::CuArray{Float32,1}, ::CuArray{Float32,1}, ::CuArray{Float32,1}, ::Nothing, ::CuArray{Float32,1}, ::Nothing, ::CuArray{UInt8,1}) at /home/tyler/.julia/packages/Flux/lz7S9/src/cuda/curnn.jl:187
   [4] backwardData(::Flux.CUDA.RNNDesc{Float32}, ::CuArray{Float32,1}, ::CuArray{Float32,1}, ::CuArray{Float32,1}, ::CuArray{Float32,1}, ::CuArray{UInt8,1}) at /home/tyler/.julia/packages/Flux/lz7S9/src/cuda/curnn.jl:195
   [5] (::getfield(Flux.CUDA, Symbol("##8#9")){Flux.GRUCell{TrackedArray{…,CuArray{Float32,2}},TrackedArray{…,CuArray{Float32,1}}},TrackedArray{…,CuArray{Float32,1}},TrackedArray{…,CuArray{Float32,1}},CuArray{UInt8,1},Tuple{CuArray{Float32,1},CuArray{Float32,1}}})(::Tuple{CuArray{Float32,1},CuArray{Float32,1}}) at /home/tyler/.julia/packages/Flux/lz7S9/src/cuda/curnn.jl:306
   [6] back_(::Tracker.Call{getfield(Flux.CUDA, Symbol("##8#9")){Flux.GRUCell{TrackedArray{…,CuArray{Float32,2}},TrackedArray{…,CuArray{Float32,1}}},TrackedArray{…,CuArray{Float32,1}},TrackedArray{…,CuArray{Float32,1}},CuArray{UInt8,1},Tuple{CuArray{Float32,1},CuArray{Float32,1}}},Tuple{Tracker.Tracked{CuArray{Float32,1}},Tracker.Tracked{CuArray{Float32,1}},Tracker.Tracked{CuArray{Float32,2}},Tracker.Tracked{CuArray{Float32,2}},Tracker.Tracked{CuArray{Float32,1}}}}, ::Tuple{CuArray{Float32,1},CuArray{Float32,1}}, ::Bool) at /home/tyler/.julia/packages/Tracker/6wcYJ/src/back.jl:35
   [7] back(::Tracker.Tracked{Tuple{CuArray{Float32,1},CuArray{Float32,1}}}, ::Tuple{CuArray{Float32,1},Int64}, ::Bool) at /home/tyler/.julia/packages/Tracker/6wcYJ/src/back.jl:58
   [8] (::getfield(Tracker, Symbol("##13#14")){Bool})(::Tracker.Tracked{Tuple{CuArray{Float32,1},CuArray{Float32,1}}}, ::Tuple{CuArray{Float32,1},Int64}) at /home/tyler/.julia/packages/Tracker/6wcYJ/src/back.jl:38
   [9] foreach(::Function, ::Tuple{Tracker.Tracked{Tuple{CuArray{Float32,1},CuArray{Float32,1}}},Nothing}, ::Tuple{Tuple{CuArray{Float32,1},Int64},Nothing}) at ./abstractarray.jl:1867
   [10] back_(::Tracker.Call{getfield(Tracker, Symbol("##356#358")){Tracker.TrackedTuple{Tuple{CuArray{Float32,1},CuArray{Float32,1}}},Int64},Tuple{Tracker.Tracked{Tuple{CuArray{Float32,1},CuArray{Float32,1}}},Nothing}}, ::CuArray{Float32,1}, ::Bool) at /home/tyler/.julia/packages/Tracker/6wcYJ/src/back.jl:38
   [11] back(::Tracker.Tracked{CuArray{Float32,1}}, ::CuArray{Float32,1}, ::Bool) at /home/tyler/.julia/packages/Tracker/6wcYJ/src/back.jl:58
   [12] back!(::TrackedArray{…,CuArray{Float32,1}}, ::CuArray{Float32,1}) at /home/tyler/.julia/packages/Tracker/6wcYJ/src/back.jl:77
   [13] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:23
   [14] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
   [15] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:7
   [16] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
   [17] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
   [18] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1083
   [19] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
   [20] include at ./boot.jl:326 [inlined]
   [21] include_relative(::Module, ::String) at ./loading.jl:1038
   [22] include(::Module, ::String) at ./sysimg.jl:29
   [23] include(::String) at ./client.jl:403
   [24] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/cuda.jl:45
   [25] include at ./boot.jl:326 [inlined]
   [26] include_relative(::Module, ::String) at ./loading.jl:1038
   [27] include(::Module, ::String) at ./sysimg.jl:29
   [28] include(::String) at ./client.jl:403
   [29] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/runtests.jl:30
   [30] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1083
   [31] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/runtests.jl:11
   [32] include at ./boot.jl:326 [inlined]
   [33] include_relative(::Module, ::String) at ./loading.jl:1038
   [34] include(::Module, ::String) at ./sysimg.jl:29
   [35] include(::String) at ./client.jl:403
   [36] top-level scope at none:0
   [37] eval(::Module, ::Any) at ./boot.jl:328
   [38] exec_options(::Base.JLOptions) at ./client.jl:243
   [39] _start() at ./client.jl:436
batch_size = 5: Test Failed at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:26
  Expression: ((rnn.cell).Wi).grad ≈ collect(((curnn.cell).Wi).grad)
   Evaluated: Float32[0.0264221 0.0311623 … 0.0401529 0.0481648; -0.00580488 -0.00684245 … -0.000633113 -0.00512767; … ; -0.488995 -0.397307 … -0.543058 -0.593239; -1.59764 -1.72794 … -1.24722 -2.0625] ≈ Float32[0.0163969 0.0200817 … 0.0357488 0.036319; -0.00383122 -0.00466102 … 0.000233916 -0.00279561; … ; -0.655613 -0.581466 … -0.616253 -0.790114; -1.32217 -1.42347 … -1.1262 -1.737]
Stacktrace:
 [1] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:26
 [2] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
 [3] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:7
 [4] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
 [5] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
 [6] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1083
 [7] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
batch_size = 5: Test Failed at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:27
  Expression: ((rnn.cell).Wh).grad ≈ collect(((curnn.cell).Wh).grad)
   Evaluated: Float32[-0.0155654 0.00326925 … -0.0217709 -0.00794538; 0.00331486 -0.000961465 … 0.00208863 -0.000682507; … ; 0.0600955 -0.0379895 … 0.0877803 0.056216; 0.13519 -0.049922 … 0.1269 0.0464798] ≈ Float32[-0.0102084 0.00063507 … -0.0177858 -0.00776742; 0.00226024 -0.000442876 … 0.00130408 -0.000717541; … ; 0.0800566 -0.047805 … 0.10263 0.0568791; 0.120198 -0.04255 … 0.115747 0.0459818]
Stacktrace:
 [1] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:27
 [2] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
 [3] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:7
 [4] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
 [5] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
 [6] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1083
 [7] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
batch_size = 5: Test Failed at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:28
  Expression: ((rnn.cell).b).grad ≈ collect(((curnn.cell).b).grad)
   Evaluated: Float32[0.0431294, -0.00524064, 0.00328069, 0.0383973, -0.0277038, -0.190451, 0.378964, 0.113537, -1.10465, -0.109885, -0.380812, -0.806893, -0.591657, -0.840546, -2.16732] ≈ Float32[0.0309576, -0.0028444, 0.00784941, 0.0508012, -0.0251556, -0.116478, 0.196908, 0.156512, -1.30342, -0.109, -0.0338723, -0.537076, -0.840815, -1.04284, -1.83286]
Stacktrace:
 [1] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:28
 [2] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
 [3] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:7
 [4] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
 [5] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
 [6] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1083
 [7] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
batch_size = 5: Test Failed at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:29
  Expression: ((rnn.cell).h).grad ≈ collect(((curnn.cell).h).grad)
   Evaluated: Float32[-0.0128217, -0.706767, -0.358611, -1.41596, -0.547122] ≈ Float32[0.0667142, -0.127551, -0.454483, -1.79769, -0.414044]
Stacktrace:
 [1] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:29
 [2] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
 [3] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:7
 [4] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1156
 [5] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
 [6] top-level scope at /build/source/usr/share/julia/stdlib/v1.1/Test/src/Test.jl:1083
 [7] top-level scope at /home/tyler/.julia/packages/Flux/lz7S9/test/cuda/curnn.jl:4
Test Summary:        | Pass  Fail  Error  Total
Flux                 |  246     4      1    251
  Throttle           |   11                  11
  Jacobian           |    1                   1
  Initialization     |   12                  12
  Params             |    2                   2
  Basic Stacking     |    1                   1
  Precision          |    6                   6
  Stacking           |    3                   3
  onecold            |    4                   4
  Optimise           |   11                  11
  Optimiser          |    3                   3
  Training Loop      |    2                   2
  basic              |   25                  25
  Dropout            |    8                   8
  BatchNorm          |   14                  14
  InstanceNorm       |   16                  16
  GroupNorm          |   16                  16
  losses             |   30                  30
  Pooling            |    2                   2
  CNN                |    1                   1
  Depthwise Conv     |    4                   4
  Tracker            |    4                   4
  CuArrays           |    8                   8
  CUDNN BatchNorm    |   10                  10
  RNN                |   40     4      1     45
    R = Flux.RNN     |   16                  16
    R = Flux.GRU     |    6     4      1     11
      batch_size = 1 |    2            1      3
      batch_size = 5 |    4     4             8
    R = Flux.LSTM    |   18                  18
ERROR: LoadError: Some tests did not pass: 246 passed, 4 failed, 1 errored, 0 broken.
in expression starting at /home/tyler/.julia/packages/Flux/lz7S9/test/runtests.jl:9
ERROR: Package Flux errored during testing

This appears to be a known issue https://github.com/FluxML/Flux.jl/issues/267

tbenst commented 5 years ago

duplicate of https://github.com/FluxML/model-zoo/issues/102