search

locationtech / rasterframes

Geospatial Raster support for Spark DataFrames
http://rasterframes.io
Apache License 2.0
240 stars 46 forks source link

Python GeotrellisTests fail in CircleCI #479

Open vpipkt opened 4 years ago

vpipkt commented 4 years ago

Originally posted by @vpipkt in https://github.com/locationtech/rasterframes/pull/478#issuecomment-592126987

The tests run fine everywhere else, but on CircleCI they hang and timeout after a java.lang.NoClassDefFoundError: fs2/Stream

This error is triggered by taking action on the dataframe read from GeoTrellis.

Pardon the dust here, but I ssh'ed into the container running on circleci and then in target dir python setup.py install the package, the run python -m unittest -v tests/GeotrellisTests.py

I tweaked the test to print more stuff.. Observe:

writing done, calling spark.read
Calling count on the dataframe
[Stage 6:>                                                        (0 + 36) / 36]Exception in thread "geotrellis-default-io-30" java.lang.NoClassDefFoundError: fs2/Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169$$anonfun$apply$170
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2003)
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2001)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:139)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.run(Callback.scala:127)
        at cats.effect.internals.Trampoline.cats$effect$internals$Trampoline$$immediateLoop(Trampoline.scala:70)
        at cats.effect.internals.Trampoline.startLoop(Trampoline.scala:36)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.cats$effect$internals$TrampolineEC$JVMTrampoline$$super$startLoop(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply$mcV$sp(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.startLoop(TrampolineEC.scala:92)
        at cats.effect.internals.Trampoline.execute(Trampoline.scala:43)
        at cats.effect.internals.TrampolineEC.execute(TrampolineEC.scala:44)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.apply(Callback.scala:133)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.apply(Callback.scala:120)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$2.apply(Deferred.scala:205)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$2.apply(Deferred.scala:205)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply$mcV$sp(Deferred.scala:241)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply(Deferred.scala:241)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply(Deferred.scala:241)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:87)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.IOShift$Tick.run(IOShift.scala:36)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException: fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169$$anonfun$apply$170
        at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
        ... 32 more
Exception in thread "geotrellis-default-io-10" java.lang.NoClassDefFoundError: fs2/Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169$$anonfun$apply$170
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2003)
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2001)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:139)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.run(Callback.scala:127)
        at cats.effect.internals.Trampoline.cats$effect$internals$Trampoline$$immediateLoop(Trampoline.scala:70)
        at cats.effect.internals.Trampoline.startLoop(Trampoline.scala:36)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.cats$effect$internals$TrampolineEC$JVMTrampoline$$super$startLoop(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply$mcV$sp(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.startLoop(TrampolineEC.scala:92)
        at cats.effect.internals.Trampoline.execute(Trampoline.scala:43)
        at cats.effect.internals.TrampolineEC.execute(TrampolineEC.scala:44)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.apply(Callback.scala:133)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.apply(Callback.scala:120)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$2.apply(Deferred.scala:205)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$2.apply(Deferred.scala:205)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply$mcV$sp(Deferred.scala:241)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply(Deferred.scala:241)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply(Deferred.scala:241)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:87)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.IOShift$Tick.run(IOShift.scala:36)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Exception in thread "geotrellis-default-io-12" java.lang.NoClassDefFoundError: fs2/Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169$$anonfun$apply$170
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2003)
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2001)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:139)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.run(Callback.scala:127)
        at cats.effect.internals.Trampoline.cats$effect$internals$Trampoline$$immediateLoop(Trampoline.scala:70)
        at cats.effect.internals.Trampoline.startLoop(Trampoline.scala:36)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.cats$effect$internals$TrampolineEC$JVMTrampoline$$super$startLoop(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply$mcV$sp(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.startLoop(TrampolineEC.scala:92)
        at cats.effect.internals.Trampoline.execute(Trampoline.scala:43)
        at cats.effect.internals.TrampolineEC.execute(TrampolineEC.scala:44)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.apply(Callback.scala:133)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.apply(Callback.scala:120)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$2.apply(Deferred.scala:205)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$2.apply(Deferred.scala:205)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply$mcV$sp(Deferred.scala:241)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply(Deferred.scala:241)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply(Deferred.scala:241)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:87)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.IOShift$Tick.run(IOShift.scala:36)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Exception in thread "geotrellis-default-io-17" java.lang.NoClassDefFoundError: fs2/Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169$$anonfun$apply$170
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2003)
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2001)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:139)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.run(Callback.scala:127)
        at cats.effect.internals.Trampoline.cats$effect$internals$Trampoline$$immediateLoop(Trampoline.scala:70)
        at cats.effect.internals.Trampoline.startLoop(Trampoline.scala:36)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.cats$effect$internals$TrampolineEC$JVMTrampoline$$super$startLoop(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply$mcV$sp(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.startLoop(TrampolineEC.scala:92)
        at cats.effect.internals.Trampoline.execute(Trampoline.scala:43)
        at cats.effect.internals.TrampolineEC.execute(TrampolineEC.scala:44)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.apply(Callback.scala:133)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.apply(Callback.scala:120)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$2.apply(Deferred.scala:205)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$2.apply(Deferred.scala:205)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply$mcV$sp(Deferred.scala:241)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply(Deferred.scala:241)
        at cats.effect.concurrent.Deferred$ConcurrentDeferred$$anonfun$1.apply(Deferred.scala:241)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:87)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.IOShift$Tick.run(IOShift.scala:36)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
Exception in thread "geotrellis-default-io-1" java.lang.NoClassDefFoundError: fs2/Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169$$anonfun$apply$170
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2003)
        at fs2.Stream$$anonfun$parJoin$extension$2$$anonfun$apply$159$$anonfun$apply$160$$anonfun$apply$161$$anonfun$fs2$Stream$$anonfun$$anonfun$$anonfun$$anonfun$$runInner$2$1$$anonfun$apply$165$$anonfun$apply$168$$anonfun$apply$169.apply(Stream.scala:2001)
        at cats.effect.internals.IORunLoop$.cats$effect$internals$IORunLoop$$loop(IORunLoop.scala:139)
        at cats.effect.internals.IORunLoop$RestartCallback.signal(IORunLoop.scala:355)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:376)
        at cats.effect.internals.IORunLoop$RestartCallback.apply(IORunLoop.scala:316)
        at cats.effect.internals.Callback$AsyncIdempotentCallback.run(Callback.scala:127)
        at cats.effect.internals.Trampoline.cats$effect$internals$Trampoline$$immediateLoop(Trampoline.scala:70)
        at cats.effect.internals.Trampoline.startLoop(Trampoline.scala:36)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.cats$effect$internals$TrampolineEC$JVMTrampoline$$super$startLoop(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply$mcV$sp(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at cats.effect.internals.TrampolineEC$JVMTrampoline$$anonfun$startLoop$1.apply(TrampolineEC.scala:93)
        at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
        at cats.effect.internals.TrampolineEC$JVMTrampoline.startLoop(TrampolineEC.scala:92)
        at cats.effect.internals.Trampoline.