JackYumCha / QA

Question and Answer
0 stars 0 forks source link

Spark Sbt Configuration #21

Open JackYumCha opened 6 years ago

JackYumCha commented 6 years ago
name := "FirstSpark"
version := "0.1"
scalaVersion := "2.12.6"
organization := "org.example.ds"
publishMavenStyle := true
fork := true

libraryDependencies += "org.apache.spark" %% "spark-core" % "2.2.1" % "compile" exclude("org.apache.hadoop", "*") // 2.2.1 works
libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.2.1" % "compile" exclude("org.apache.hadoop", "*")

libraryDependencies += "org.apache.hadoop" % "hadoop-mapreduce-client-core" % "2.9.0" % "compile" // 2.9.0 works
libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.9.0" % "compile"
libraryDependencies += "org.apache.hadoop" % "hadoop-azure" % "2.9.0" % "compile" // 2.9.0 works for df write

libraryDependencies += "org.datasyslab" % "geospark" % "1.1.3"
libraryDependencies += "org.datasyslab" % "geospark-sql_".concat("2.2") % "1.1.3"
libraryDependencies += "org.datasyslab" % "geospark-viz" % "1.1.3"

// https://mvnrepository.com/artifact/com.typesafe.akka/akka-http
libraryDependencies += "com.typesafe.akka" %% "akka-http" % "10.1.1"
// https://mvnrepository.com/artifact/com.typesafe.akka/akka-stream
libraryDependencies += "com.typesafe.akka" %% "akka-stream" % "2.5.13"
// https://mvnrepository.com/artifact/com.typesafe.akka/akka-actor
libraryDependencies += "com.typesafe.akka" %% "akka-actor" % "2.5.13"
// https://mvnrepository.com/artifact/com.typesafe.akka/akka-http-core
libraryDependencies += "com.typesafe.akka" %% "akka-http-core" % "10.1.1"
// https://mvnrepository.com/artifact/com.typesafe.akka/akka-http-spray-json
libraryDependencies += "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.1"

libraryDependencies += "com.microsoft.azure" % "azure-storage" % "7.0.0" % "compile"
import sbt.Keys.{libraryDependencies, version}

resolvers +=
  "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots"

resolvers +=
  "Open Source Geospatial Foundation Repository" at "http://download.osgeo.org/webdav/geotools"