Stage failure

[ghislainfourny]

This query (from the JSONiq tutorial) fails. It is worth investigating, but may be complex to solve (it may come down to the nested for, which we can discuss)

let $stores := [ { "store number" : 1, "state" : "MA" }, { "store number" : 2, "state" : "MA" }, { "store number" : 3, "state" : "CA" }, { "store number" : 4, "state" : "CA" } ] let $sales := [ { "product" : "broiler", "store number" : 1, "quantity" : 20 }, { "product" : "toaster", "store number" : 2, "quantity" : 100 }, { "product" : "toaster", "store number" : 2, "quantity" : 50 }, { "product" : "toaster", "store number" : 3, "quantity" : 50 }, { "product" : "blender", "store number" : 3, "quantity" : 100 }, { "product" : "blender", "store number" : 3, "quantity" : 150 }, { "product" : "socks", "store number" : 1, "quantity" : 500 }, { "product" : "socks", "store number" : 2, "quantity" : 10 }, { "product" : "shirt", "store number" : 3, "quantity" : 10 } ] let $join := for $store in $stores[], $sale in $sales[] where $store."store number" eq $sale."store number" return { "nb" : $store."store number", "state" : $store.state, "sold" : $sale.product } return [$join]

[wscsprint3r]

When running the NestsVars.iq test file on a real cluster, this happens. Nests might be broken completely.

[wscsprint3r]

org.apache.spark.SparkException: Job aborted due to stage failure: Failed to serialize task 3, not attempting to retry it. Exception during serialization: java.io.NotSerializableException: sparksoniq.jsoniq.item.IntegerItem Serialization stack:

• object not serializable (class: sparksoniq.jsoniq.item.IntegerItem, value: sparksoniq.jsoniq.item.IntegerItem@2ea54d29)
• element of array (index: 0)
• array (class [Ljava.lang.Object;, size 1)
• field (class: scala.collection.mutable.WrappedArray$ofRef, name: array, type: class [Ljava.lang.Object;)
• object (class scala.collection.mutable.WrappedArray$ofRef, WrappedArray(sparksoniq.jsoniq.item.IntegerItem@2ea54d29))
• writeObject data (class: org.apache.spark.rdd.ParallelCollectionPartition)
• object (class org.apache.spark.rdd.ParallelCollectionPartition, org.apache.spark.rdd.ParallelCollectionPartition@694)
• field (class: org.apache.spark.scheduler.ResultTask, name: partition, type: interface org.apache.spark.Partition)
• object (class org.apache.spark.scheduler.ResultTask, ResultTask(0, 3)) at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802) at scala.Option.foreach(Option.scala:257) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1928) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1941) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1954) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1968) at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:362) at org.apache.spark.rdd.RDD.collect(RDD.scala:935) at org.apache.spark.api.java.JavaRDDLike$class.collect(JavaRDDLike.scala:361) at org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45) at sparksoniq.spark.iterator.SparkRuntimeIterator.next(SparkRuntimeIterator.java:70) at sparksoniq.spark.closures.LetClauseMapClosure.call(LetClauseMapClosure.java:44) at sparksoniq.spark.closures.LetClauseMapClosure.call(LetClauseMapClosure.java:31) at org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1.apply(JavaPairRDD.scala:1040) at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) at scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434) at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440) at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408) at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:389) at scala.collection.Iterator$class.foreach(Iterator.scala:893) at scala.collection.AbstractIterator.foreach(Iterator.scala:1336) at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48) at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310) at scala.collection.AbstractIterator.to(Iterator.scala:1336) at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302) at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1336) at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289) at scala.collection.AbstractIterator.toArray(Iterator.scala:1336) at org.apache.spark.rdd.RDD$$anonfun$take$1$$anonfun$29.apply(RDD.scala:1354) at org.apache.spark.rdd.RDD$$anonfun$take$1$$anonfun$29.apply(RDD.scala:1354) at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954) at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1954) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) at org.apache.spark.scheduler.Task.run(Task.scala:99) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:322) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)

[CanBerker]

Query now succeeds and returns: [ { "nb" : 1, "state" : "MA", "sold" : "broiler" }, { "nb" : 1, "state" : "MA", "sold" : "socks" }, { "nb" : 2, "state" : "MA", "sold" : "toaster" }, { "nb" : 2, "state" : "MA", "sold" : "toaster" }, { "nb" : 2, "state" : "MA", "sold" : "socks" }, { "nb" : 3, "state" : "CA", "sold" : "toaster" }, { "nb" : 3, "state" : "CA", "sold" : "blender" }, { "nb" : 3, "state" : "CA", "sold" : "blender" }, { "nb" : 3, "state" : "CA", "sold" : "shirt" } ]

[ghislainfourny]

Marvellous! So good to see so many issues solved. You did a great job with the local FLWORs.