val sparkConf = new SparkConf().setAppName(getClass.getName).setMaster("local")
val spark = SparkSession.builder().config(sparkConf).getOrCreate()
println("Loading Data...")
val path = "/files/1/ad2e3fc65f36398a66949e5d0c8d50ee.csv"
val df = spark.read.format("com.springml.spark.sftp")
.option("fileType", "csv")
.option("host", "shuying215")
.option("username", "ftpuser")
.option("password", "ftpuser123")
.option("header", "true")
.option("inferSchema", "true")
.option("ignoreLeadingWhiteSpace", "true")
.option("ignoreTrailingWhiteSpace", "true")
.load(path)
println("Showing Data...")
df.show(5)
println("Finished!")
spark.stop()
Error
Loading Data...
Exception in thread "main" com.jcraft.jsch.JSchException: java.io.IOException: Pipe closed
at com.jcraft.jsch.ChannelSftp.start(ChannelSftp.java:315)
at com.jcraft.jsch.Channel.connect(Channel.java:152)
at com.jcraft.jsch.Channel.connect(Channel.java:145)
at com.springml.sftp.client.SFTPClient.createSFTPChannel(SFTPClient.java:278)
at com.springml.sftp.client.SFTPClient.copy(SFTPClient.java:73)
at com.springml.spark.sftp.DefaultSource.copy(DefaultSource.scala:218)
at com.springml.spark.sftp.DefaultSource.createRelation(DefaultSource.scala:83)
at com.springml.spark.sftp.DefaultSource.createRelation(DefaultSource.scala:41)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:341)
at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:239)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:227)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:174)
at org.apache.spark.example.sql.FTPDemo$.main(FTPDemo.scala:29)
at org.apache.spark.example.sql.FTPDemo.main(FTPDemo.scala)
Caused by: java.io.IOException: Pipe closed
at java.io.PipedInputStream.read(PipedInputStream.java:307)
at java.io.PipedInputStream.read(PipedInputStream.java:377)
at com.jcraft.jsch.ChannelSftp.fill(ChannelSftp.java:2882)
at com.jcraft.jsch.ChannelSftp.header(ChannelSftp.java:2908)
at com.jcraft.jsch.ChannelSftp.start(ChannelSftp.java:262)
... 13 more
Unable to read files on ftp server
Code:
Error