2018-06-19 32k dataframe

[xuchuanyin]

  test("write from dataframe with long string datatype") {
    prepareDF()
    longStringDF.show(numRows = 3, truncate = false)
    longStringDF.write
      .format("carbondata")
      .option("tableName", longStringTable)
      .option("single_pass", "false")
      .option("sort_columns", "name")
      .option("long_string_columns", "description, note")
      .mode(SaveMode.Overwrite)
    println("XU write dataframe finished.")
    sql("SHOW TABLES").show()
    checkQuery()
  }

[xuchuanyin]

  private def prepareDF(): Unit = {
    val schema = StructType(
      StructField("id", IntegerType, nullable = true) ::
      StructField("name", StringType, nullable = true) ::
      StructField("description", StringType, nullable = true) ::
      StructField("address", StringType, nullable = true) ::
      StructField("note", StringType, nullable = true) :: Nil
    )
    longStringDF = sqlContext.sparkSession.read
      .schema(schema)
      .csv(inputFile)
  }