search

springml / spark-sftp

Spark connector for SFTP
Apache License 2.0
100 stars 98 forks source link

com.jcraft.jsch.JSchException: java.io.IOException: Pipe closed #73

Open lonly197 opened 4 years ago

lonly197 commented 4 years ago

Unable to read files on ftp server

Code:

    val sparkConf = new SparkConf().setAppName(getClass.getName).setMaster("local")
    val spark = SparkSession.builder().config(sparkConf).getOrCreate()

    println("Loading Data...")
    val path = "/files/1/ad2e3fc65f36398a66949e5d0c8d50ee.csv"
    val df = spark.read.format("com.springml.spark.sftp")
      .option("fileType", "csv")
      .option("host", "shuying215")
      .option("username", "ftpuser")
      .option("password", "ftpuser123")
      .option("header", "true")
      .option("inferSchema", "true")
      .option("ignoreLeadingWhiteSpace", "true")
      .option("ignoreTrailingWhiteSpace", "true")
      .load(path)

    println("Showing Data...")
    df.show(5)

    println("Finished!")
    spark.stop()

Error

Loading Data...
Exception in thread "main" com.jcraft.jsch.JSchException: java.io.IOException: Pipe closed
    at com.jcraft.jsch.ChannelSftp.start(ChannelSftp.java:315)
    at com.jcraft.jsch.Channel.connect(Channel.java:152)
    at com.jcraft.jsch.Channel.connect(Channel.java:145)
    at com.springml.sftp.client.SFTPClient.createSFTPChannel(SFTPClient.java:278)
    at com.springml.sftp.client.SFTPClient.copy(SFTPClient.java:73)
    at com.springml.spark.sftp.DefaultSource.copy(DefaultSource.scala:218)
    at com.springml.spark.sftp.DefaultSource.createRelation(DefaultSource.scala:83)
    at com.springml.spark.sftp.DefaultSource.createRelation(DefaultSource.scala:41)
    at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:341)
    at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:239)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:227)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:174)
    at org.apache.spark.example.sql.FTPDemo$.main(FTPDemo.scala:29)
    at org.apache.spark.example.sql.FTPDemo.main(FTPDemo.scala)
Caused by: java.io.IOException: Pipe closed
    at java.io.PipedInputStream.read(PipedInputStream.java:307)
    at java.io.PipedInputStream.read(PipedInputStream.java:377)
    at com.jcraft.jsch.ChannelSftp.fill(ChannelSftp.java:2882)
    at com.jcraft.jsch.ChannelSftp.header(ChannelSftp.java:2908)
    at com.jcraft.jsch.ChannelSftp.start(ChannelSftp.java:262)
    ... 13 more