While Executing the spark job on EMR running into No Suitable Driver found issue

[cd-rnukala]

Below is my code snippet:

import com.typesafe.config.ConfigFactory import org.apache.spark.sql.{DataFrame, SQLContext} import org.apache.spark.{SparkConf, SparkContext} import org.slf4j.LoggerFactory

object SSPInventoryLoaderJob {

val logger = LoggerFactory.getLogger(getClass)

def main(args: Array[String]) { // create Spark context with Spark configuration val sc = new SparkContext(new SparkConf().setAppName("Spark Count").setMaster("local[*]")) val sql = new SQLContext(sc) val host = "xx.xx.xx.xx" val port = 3306 val dbName = "xxxx" val user = "xxxx" val password = "xxxx" val tableName = "xxx"

val conf = ConfigFactory.load
sql.sparkContext.hadoopConfiguration.set(Constants.S3_ACCESSKEY, conf.getString(Constants.SPARK_ETL_S3_ACCESSKEY))
sql.sparkContext.hadoopConfiguration.set(Constants.S3_SECRETKEY, conf.getString(Constants.SPARK_ETL_S3_SECRETKEY))
val start: Long = System.currentTimeMillis
val dfSSPInventoryList: DataFrame = sql.read
  .format(conf.getString(Constants.REDSHIFT_DRIVER))
  .option(Constants.URL, conf.getString(Constants.JDBC_URL))
  .option(Constants.QUERY, conf.getString(Constants.SSP_INVENTORY_LOADER_JOB_QUERY))
  .option(Constants.TEMP_DIR, conf.getString(Constants.S3_TMP_DIRECTORY) + "/ssp_inventory/")
  .load()
  val func_rt_acc = new com.memsql.spark.connector.DataFrameFunctions(dfSSPInventoryList)
func_rt_acc.saveToMemSQL(dbName, tableName, host, port, user, password)

} }

Error: Exception in thread "main" java.sql.SQLException: No suitable driver found for jdbc:mysql://xx.xx.xx.xxxx:3306 at java.sql.DriverManager.getConnection(DriverManager.java:689) at java.sql.DriverManager.getConnection(DriverManager.java:247) at com.memsql.spark.context.MemSQLContext$.getMemSQLConnection(MemSQLContext.scala:41) at com.memsql.spark.context.MemSQLContext$.getMemSQLChildAggregators(MemSQLContext.scala:49) at com.memsql.spark.context.MemSQLContext$.getMemSQLNodesAvailableForIngest(MemSQLContext.scala:101) at com.memsql.spark.connector.RDDFunctions.saveToMemSQL(RDDFunctions.scala:86) at com.memsql.spark.connector.DataFrameFunctions.saveToMemSQL(DataFrameFunctions.scala:65) at com.cadreon.atv.ranker.SSPInventoryLoaderJob$.main(SSPInventoryLoaderJob.scala:39) at com.cadreon.atv.ranker.SSPInventoryLoaderJob.main(SSPInventoryLoaderJob.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

[choochootrain]

how are you specifying the memsql-connector dependency in your build.sbt?

it appears to not have bundled the memsql and jdbc dependencies which may happen if you specify "provided" for the memsql-connector dependency.

a valid build.sbt could look like this:

lazy val root = (project in file(".")).
  settings(
    name := "my-memsql-spark-job",
    version := "0.0.1",
    scalaVersion := "2.10.5",
    parallelExecution in Test := false,
    libraryDependencies  ++= Seq(
        "org.apache.spark" %% "spark-core" % "1.5.2" % "provided",
        "org.apache.spark" %% "spark-sql" % "1.5.2"  % "provided",
        "org.apache.spark" %% "spark-streaming" % "1.5.2" % "provided",
        "com.memsql" %% "memsql-connector" % "1.3.3"
    )
)

[lucyyu]

Closing due to inactivity