search

NICTA / scoobi

A Scala productivity framework for Hadoop.
http://nicta.github.com/scoobi/
482 stars 97 forks source link

Scoobi 0.8.0-hadoop2-SNAPSHOT doesn't run on YARN on CDH5-beta1 #303

Open GrahamLea opened 11 years ago

GrahamLea commented 11 years ago

Using 0.8.0-hadoop2-SNAPSHOT to run the WordCount example from the Scoobi website on YARN/MR2 on a CDH5-beta cluster fails with a NoSuchMethodException on org.apache.hadoop.mapreduce.Job.getJobClient()

Exception in thread "main" java.lang.NoSuchMethodException: org.apache.hadoop.mapreduce.Job.getJobClient()
    at java.lang.Class.getDeclaredMethod(Class.java:1973)
    at com.nicta.scoobi.impl.reflect.Classes$class.invokeProtected(Classes.scala:135)
    at com.nicta.scoobi.impl.reflect.Classes$.invokeProtected(Classes.scala:165)
    at com.nicta.scoobi.impl.exec.TaskDetailsLogger.getJobClient$lzycompute(MapReduceJob.scala:314)
    at com.nicta.scoobi.impl.exec.TaskDetailsLogger.getJobClient(MapReduceJob.scala:314)
    at com.nicta.scoobi.impl.exec.TaskDetailsLogger.com$nicta$scoobi$impl$exec$TaskDetailsLogger$$getTaskCompletionEvents(MapReduceJob.scala:308)
    at com.nicta.scoobi.impl.exec.TaskDetailsLogger$$anonfun$logTaskCompletionDetails$1.apply(MapReduceJob.scala:285)
    at com.nicta.scoobi.impl.exec.TaskDetailsLogger$$anonfun$logTaskCompletionDetails$1.apply(MapReduceJob.scala:285)
    at scala.collection.Iterator$$anon$9.next(Iterator.scala:162)
    at scala.collection.Iterator$$anon$17.hasNext(Iterator.scala:511)
    at scala.collection.Iterator$class.foreach(Iterator.scala:727)
    at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
    at com.nicta.scoobi.impl.exec.TaskDetailsLogger.logTaskCompletionDetails(MapReduceJob.scala:285)
    at com.nicta.scoobi.impl.exec.MapReduceJob.executeJob(MapReduceJob.scala:215)
    at com.nicta.scoobi.impl.exec.MapReduceJob.execute(MapReduceJob.scala:69)
    at com.nicta.scoobi.impl.exec.HadoopMode$Execution$$anonfun$runMscrs$4.apply(HadoopMode.scala:141)
    at com.nicta.scoobi.impl.exec.HadoopMode$Execution$$anonfun$runMscrs$4.apply(HadoopMode.scala:141)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
    at scala.collection.immutable.List.foreach(List.scala:318)
    at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
    at scala.collection.AbstractTraversable.map(Traversable.scala:105)
    at com.nicta.scoobi.impl.exec.HadoopMode$Execution.runMscrs(HadoopMode.scala:141)
    at com.nicta.scoobi.impl.exec.HadoopMode$Execution.execute(HadoopMode.scala:119)
    at com.nicta.scoobi.impl.exec.HadoopMode$$anonfun$executeLayer$1.apply(HadoopMode.scala:109)
    at com.nicta.scoobi.impl.exec.HadoopMode$$anonfun$executeLayer$1.apply(HadoopMode.scala:108)
    at org.kiama.attribution.AttributionCore$CachedAttribute.apply(AttributionCore.scala:61)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
    at scala.collection.immutable.List.foreach(List.scala:318)
    at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
    at scala.collection.AbstractTraversable.map(Traversable.scala:105)
    at com.nicta.scoobi.impl.exec.HadoopMode.com$nicta$scoobi$impl$exec$HadoopMode$$executeLayers$1(HadoopMode.scala:68)
    at com.nicta.scoobi.impl.exec.HadoopMode$$anonfun$executeNode$1.apply(HadoopMode.scala:91)
    at com.nicta.scoobi.impl.exec.HadoopMode$$anonfun$executeNode$1.apply(HadoopMode.scala:84)
    at org.kiama.attribution.AttributionCore$CachedAttribute.apply(AttributionCore.scala:61)
    at scalaz.syntax.IdOps$class.$bar$greater(IdOps.scala:15)
    at scalaz.syntax.ToIdOps$$anon$1.$bar$greater(IdOps.scala:78)
    at com.nicta.scoobi.impl.exec.HadoopMode.execute(HadoopMode.scala:52)
    at com.nicta.scoobi.impl.exec.HadoopMode.execute(HadoopMode.scala:48)
    at com.nicta.scoobi.impl.Persister.persist(Persister.scala:44)
    at com.nicta.scoobi.impl.ScoobiConfigurationImpl.persist(ScoobiConfigurationImpl.scala:355)
    at com.nicta.scoobi.application.Persist$class.persist(Persist.scala:33)
    at WordCount$.persist(MyApp.scala:4)
    at com.nicta.scoobi.application.Persist$PersistableList.persist(Persist.scala:151)
    at WordCount$.run(MyApp.scala:12)
    at com.nicta.scoobi.application.ScoobiApp$$anonfun$main$1.apply$mcV$sp(ScoobiApp.scala:80)
    at com.nicta.scoobi.application.ScoobiApp$$anonfun$main$1.apply(ScoobiApp.scala:75)
    at com.nicta.scoobi.application.ScoobiApp$$anonfun$main$1.apply(ScoobiApp.scala:75)
    at com.nicta.scoobi.application.Hadoop$class.runOnCluster(Hadoop.scala:112)
    at WordCount$.runOnCluster(MyApp.scala:4)
    at com.nicta.scoobi.application.Hadoop$class.executeOnCluster(Hadoop.scala:69)
    at WordCount$.executeOnCluster(MyApp.scala:4)
    at com.nicta.scoobi.application.Hadoop$$anonfun$onCluster$1.apply(Hadoop.scala:55)
    at com.nicta.scoobi.application.InMemoryHadoop$class.withTimer(InMemory.scala:72)
    at WordCount$.withTimer(MyApp.scala:4)
    at com.nicta.scoobi.application.InMemoryHadoop$class.showTime(InMemory.scala:80)
    at WordCount$.showTime(MyApp.scala:4)
    at com.nicta.scoobi.application.Hadoop$class.onCluster(Hadoop.scala:55)
    at WordCount$.onCluster(MyApp.scala:4)
    at com.nicta.scoobi.application.Hadoop$class.onHadoop(Hadoop.scala:61)
    at WordCount$.onHadoop(MyApp.scala:4)
    at com.nicta.scoobi.application.ScoobiApp$class.main(ScoobiApp.scala:75)
    at WordCount$.main(MyApp.scala:4)
    at WordCount.main(MyApp.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.apache.hadoop.util.RunJar.main(RunJar.java:212)