search

NICTA / scoobi

A Scala productivity framework for Hadoop.
http://nicta.github.com/scoobi/
482 stars 97 forks source link

On wrong filesystem for FileUtil.copy #239

Closed daithiocrualaoich closed 11 years ago

daithiocrualaoich commented 11 years ago

Needs FileSystem.get in FileSystems.copyTo for use in mixed S3/HDFS. cf #185, #199.

Exception in thread "main" java.lang.IllegalArgumentException: This file system object (hdfs://10.212.231.95:9000) does not support access to the request path 's3://xxxxxx' You possibly called FileSystem.get(conf) when you should have called FileSystem.get(uri, conf) to obtain a file system supporting your path.
    at org.apache.hadoop.fs.FileSystem.checkPath(FileSystem.java:384)
    at org.apache.hadoop.hdfs.DistributedFileSystem.getPathName(DistributedFileSystem.java:129)
    at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:513)
    at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:770)
    at org.apache.hadoop.fs.FileUtil.checkDest(FileUtil.java:355)
    at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:211)
    at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:163)
    at com.nicta.scoobi.impl.io.FileSystems$$anonfun$copyTo$1.apply(FileSystems.scala:148)
    at com.nicta.scoobi.impl.io.FileSystems$$anonfun$copyTo$1.apply(FileSystems.scala:147)
    at com.nicta.scoobi.impl.plan.mscr.MscrOutputChannel$$anonfun$collectOutputs$3$$anonfun$apply$4.apply(OutputChannel.scala:106)
    at com.nicta.scoobi.impl.plan.mscr.MscrOutputChannel$$anonfun$collectOutputs$3$$anonfun$apply$4.apply(OutputChannel.scala:104)
    at scala.collection.immutable.List.foreach(List.scala:318)
    at com.nicta.scoobi.impl.plan.mscr.MscrOutputChannel$$anonfun$collectOutputs$3.apply(OutputChannel.scala:104)
    at com.nicta.scoobi.impl.plan.mscr.MscrOutputChannel$$anonfun$collectOutputs$3.apply(OutputChannel.scala:103)
    at scala.Option.foreach(Option.scala:236)
    at com.nicta.scoobi.impl.plan.mscr.MscrOutputChannel$class.collectOutputs(OutputChannel.scala:103)
    at com.nicta.scoobi.impl.plan.mscr.BypassOutputChannel.collectOutputs(OutputChannel.scala:205)
    at com.nicta.scoobi.impl.exec.MapReduceJob$$anonfun$collectOutputs$1.apply(MapReduceJob.scala:210)
    at com.nicta.scoobi.impl.exec.MapReduceJob$$anonfun$collectOutputs$1.apply(MapReduceJob.scala:210)
    at scala.collection.Iterator$class.foreach(Iterator.scala:727)
    at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
    at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
    at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
    at com.nicta.scoobi.impl.exec.MapReduceJob.collectOutputs(MapReduceJob.scala:210)
    at com.nicta.scoobi.impl.exec.MapReduceJob.execute(MapReduceJob.scala:65)
    at com.nicta.scoobi.impl.exec.HadoopMode$Execution$$anonfun$4.apply(HadoopMode.scala:115)
    at com.nicta.scoobi.impl.exec.HadoopMode$Execution$$anonfun$4.apply(HadoopMode.scala:115)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
    at scala.collection.immutable.List.foreach(List.scala:318)
    at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
    at scala.collection.AbstractTraversable.map(Traversable.scala:105)
    at com.nicta.scoobi.impl.exec.HadoopMode$Execution.runMscrs(HadoopMode.scala:115)
    at com.nicta.scoobi.impl.exec.HadoopMode$Execution.execute(HadoopMode.scala:98)
    at com.nicta.scoobi.impl.exec.HadoopMode$$anonfun$executeLayer$1.apply(HadoopMode.scala:88)
    at com.nicta.scoobi.impl.exec.HadoopMode$$anonfun$executeLayer$1.apply(HadoopMode.scala:87)
    at org.kiama.attribution.AttributionCore$CachedAttribute.apply(AttributionCore.scala:61)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
    at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
    at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
    at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
    at scala.collection.AbstractTraversable.map(Traversable.scala:105)
    at com.nicta.scoobi.impl.exec.HadoopMode.com$nicta$scoobi$impl$exec$HadoopMode$$executeLayers$1(HadoopMode.scala:66)
    at com.nicta.scoobi.impl.exec.HadoopMode$$anonfun$executeNode$1.apply(HadoopMode.scala:79)
    at com.nicta.scoobi.impl.exec.HadoopMode$$anonfun$executeNode$1.apply(HadoopMode.scala:78)
    at org.kiama.attribution.AttributionCore$CachedAttribute.apply(AttributionCore.scala:61)
    at scalaz.syntax.IdOps$class.$bar$greater(IdOps.scala:15)
    at scalaz.syntax.ToIdOps$$anon$1.$bar$greater(IdOps.scala:78)
    at com.nicta.scoobi.impl.exec.HadoopMode.execute(HadoopMode.scala:51)
    at com.nicta.scoobi.impl.exec.HadoopMode.execute(HadoopMode.scala:47)
    at com.nicta.scoobi.impl.Persister.persist(Persister.scala:44)
    at com.nicta.scoobi.impl.ScoobiConfigurationImpl.persist(ScoobiConfigurationImpl.scala:310)
    at com.nicta.scoobi.application.Persist$class.persist(Persist.scala:33)
daithiocrualaoich commented 11 years ago

Thanks.