Closed jinmu0410 closed 1 year ago
./spark-submit --class com.hs.test \
--master spark://jinmudeMacBook-Pro.local:7077 \
--name test_015 \
--jars "/Users/jinmu/Downloads/datahub-spark-lineage-0.10.0rc1.jar" \
--conf "spark.extraListeners=datahub.spark.DatahubSparkListener" \
--conf "spark.datahub.rest.server=http://localhost:8080" \
/Users/jinmu/Downloads/hs/lakehouse/target/hs-lakehouse-1.0-SNAPSHOT.jar
val spark = SparkSession.builder()
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
.config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
.appName("test1")
.master("local[*]")
.getOrCreate()
val path = "/Users/jinmu/Downloads/111.csv"
val readerOptions = Map(
"delimiter" ->",",
"header" -> "true",
"encoding" -> "utf-8",
"escape" -> "\""
)
import spark.implicits._
spark.read.format("csv").options(readerOptions).load(path)
.write.format("delta").mode("append")
.save("/Users/jinmu/Downloads/delta/jinmu.db/test33")
who can help me?