search

sorenmacbeth / flambo

A Clojure DSL for Apache Spark
Eclipse Public License 1.0
606 stars 84 forks source link

Error using flambo sql #110

Closed saikatkumardey closed 7 years ago

saikatkumardey commented 7 years ago

Here's the exception:

java.lang.ClassNotFoundException: org.apache.spark.sql.functions, compiling:(sql_functions.clj:1:1)
Exception in thread "main" java.lang.ClassNotFoundException: org.apache.spark.sql.functions, compiling:(sql_functions.clj:1:1)
    at clojure.lang.Compiler$InvokeExpr.eval(Compiler.java:3657)
    at clojure.lang.Compiler.compile1(Compiler.java:7474)
    at clojure.lang.Compiler.compile1(Compiler.java:7464)
    at clojure.lang.Compiler.compile(Compiler.java:7541)
    at clojure.lang.RT.compile(RT.java:406)
    at clojure.lang.RT.load(RT.java:451)
    at clojure.lang.RT.load(RT.java:419)
    at clojure.core$load$fn__5677.invoke(core.clj:5893)
    at clojure.core$load.invokeStatic(core.clj:5892)
    at clojure.core$load.doInvoke(core.clj:5876)
    at clojure.lang.RestFn.invoke(RestFn.java:408)
    at clojure.core$load_one.invokeStatic(core.clj:5697)
    at clojure.core$load_one.invoke(core.clj:5692)
    at clojure.core$load_lib$fn__5626.invoke(core.clj:5737)
    at clojure.core$load_lib.invokeStatic(core.clj:5736)
    at clojure.core$load_lib.doInvoke(core.clj:5717)
    at clojure.lang.RestFn.applyTo(RestFn.java:142)
    at clojure.core$apply.invokeStatic(core.clj:648)
    at clojure.core$load_libs.invokeStatic(core.clj:5774)
    at clojure.core$load_libs.doInvoke(core.clj:5758)
    at clojure.lang.RestFn.applyTo(RestFn.java:137)
    at clojure.core$apply.invokeStatic(core.clj:648)
    at clojure.core$require.invokeStatic(core.clj:5796)
    at clojure.core$require.doInvoke(core.clj:5796)
    at clojure.lang.RestFn.invoke(RestFn.java:421)
    at flambo.sql$loading__5569__auto____55.invoke(sql.clj:4)
    at clojure.lang.AFn.applyToHelper(AFn.java:152)
    at clojure.lang.AFn.applyTo(AFn.java:144)
    at clojure.lang.Compiler$InvokeExpr.eval(Compiler.java:3652)
    at clojure.lang.Compiler.compile1(Compiler.java:7474)
    at clojure.lang.Compiler.compile1(Compiler.java:7464)
    at clojure.lang.Compiler.compile(Compiler.java:7541)
    at clojure.lang.RT.compile(RT.java:406)
    at clojure.lang.RT.load(RT.java:451)
    at clojure.lang.RT.load(RT.java:419)
    at clojure.core$load$fn__5677.invoke(core.clj:5893)
    at clojure.core$load.invokeStatic(core.clj:5892)
    at clojure.core$load.doInvoke(core.clj:5876)
    at clojure.lang.RestFn.invoke(RestFn.java:408)
    at clojure.core$load_one.invokeStatic(core.clj:5697)
    at clojure.core$load_one.invoke(core.clj:5692)
    at clojure.core$load_lib$fn__5626.invoke(core.clj:5737)
    at clojure.core$load_lib.invokeStatic(core.clj:5736)
    at clojure.core$load_lib.doInvoke(core.clj:5717)
    at clojure.lang.RestFn.applyTo(RestFn.java:142)
    at clojure.core$apply.invokeStatic(core.clj:648)
    at clojure.core$load_libs.invokeStatic(core.clj:5774)
    at clojure.core$load_libs.doInvoke(core.clj:5758)
    at clojure.lang.RestFn.applyTo(RestFn.java:137)
    at clojure.core$apply.invokeStatic(core.clj:648)
    at clojure.core$require.invokeStatic(core.clj:5796)
    at clojure.core$require.doInvoke(core.clj:5796)
    at clojure.lang.RestFn.invoke(RestFn.java:436)
    at churn.spark$loading__5569__auto____36.invoke(spark.clj:1)
    at clojure.lang.AFn.applyToHelper(AFn.java:152)
    at clojure.lang.AFn.applyTo(AFn.java:144)
    at clojure.lang.Compiler$InvokeExpr.eval(Compiler.java:3652)
    at clojure.lang.Compiler.compile1(Compiler.java:7474)
    at clojure.lang.Compiler.compile1(Compiler.java:7464)
    at clojure.lang.Compiler.compile(Compiler.java:7541)
    at clojure.lang.RT.compile(RT.java:406)
    at clojure.lang.RT.load(RT.java:451)
    at clojure.lang.RT.load(RT.java:419)
    at clojure.core$load$fn__5677.invoke(core.clj:5893)
    at clojure.core$load.invokeStatic(core.clj:5892)
    at clojure.core$load.doInvoke(core.clj:5876)
    at clojure.lang.RestFn.invoke(RestFn.java:408)
    at clojure.core$load_one.invokeStatic(core.clj:5697)
    at clojure.core$compile$fn__5682.invoke(core.clj:5903)
    at clojure.core$compile.invokeStatic(core.clj:5903)
    at clojure.core$compile.invoke(core.clj:5895)
    at user$eval20$fn__29.invoke(form-init5884722053706917671.clj:1)
    at user$eval20.invokeStatic(form-init5884722053706917671.clj:1)
    at user$eval20.invoke(form-init5884722053706917671.clj:1)
    at clojure.lang.Compiler.eval(Compiler.java:6927)
    at clojure.lang.Compiler.eval(Compiler.java:6917)
    at clojure.lang.Compiler.eval(Compiler.java:6917)
    at clojure.lang.Compiler.load(Compiler.java:7379)
    at clojure.lang.Compiler.loadFile(Compiler.java:7317)
    at clojure.main$load_script.invokeStatic(main.clj:275)
    at clojure.main$init_opt.invokeStatic(main.clj:277)
    at clojure.main$init_opt.invoke(main.clj:277)
    at clojure.main$initialize.invokeStatic(main.clj:308)
    at clojure.main$null_opt.invokeStatic(main.clj:342)
    at clojure.main$null_opt.invoke(main.clj:339)
    at clojure.main$main.invokeStatic(main.clj:421)
    at clojure.main$main.doInvoke(main.clj:384)
    at clojure.lang.RestFn.invoke(RestFn.java:421)
    at clojure.lang.Var.invoke(Var.java:383)
    at clojure.lang.AFn.applyToHelper(AFn.java:156)
    at clojure.lang.Var.applyTo(Var.java:700)
    at clojure.main.main(main.java:37)
Caused by: java.lang.ClassNotFoundException: org.apache.spark.sql.functions
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
    at clojure.lang.DynamicClassLoader.findClass(DynamicClassLoader.java:69)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at clojure.lang.DynamicClassLoader.loadClass(DynamicClassLoader.java:77)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    at java.lang.Class.forName0(Native Method)
    at java.lang.Class.forName(Class.java:348)
    at clojure.lang.RT.classForName(RT.java:2168)
    at clojure.lang.RT.classForNameNonLoading(RT.java:2181)
    at flambo.sql_functions$loading__5569__auto____57.invoke(sql_functions.clj:1)
    at clojure.lang.AFn.applyToHelper(AFn.java:152)
    at clojure.lang.AFn.applyTo(AFn.java:144)
    at clojure.lang.Compiler$InvokeExpr.eval(Compiler.java:3652)
    ... 91 more
Compilation failed: Subprocess failed

Here's what I had tried:

(ns churn.spark
  (:require [flambo.conf :as conf]
            [flambo.api :as f]
            [flambo.sql :as fsql]))

(def c (-> (conf/spark-conf)
           (conf/master "local")
           (conf/app-name "flame_princess")))

(def sc (f/spark-context c))

(def data (f/parallelize sc (range 10000000) 2))

;; project.clj

(defproject churn "0.1.0-SNAPSHOT"
  :dependencies [[org.clojure/clojure "1.8.0"]
                 [proto-repl "0.1.2"]
                 [proto-repl-charts "0.2.0"]
                 [org.clojure/tools.namespace "0.2.11"]
                 [org.clojure/java.jdbc "0.7.0-alpha1"]
                 [org.clojure/data.csv "0.1.3"]
                 [yieldbot/flambo "0.8.0"]]

  :resource-paths ["lib/redjdbc.jar"]
  :main ^:skip-aot churn.core
  :target-path "target/%s"
  :profiles {:dev {:aot :all}
             :uberjar {:aot :all}
             :provided
                 {:dependencies
                   [[org.apache.spark/spark-core_2.11 "2.0.1"]]}})
saikatkumardey commented 7 years ago

Fixed it.

Added these to :dependencies.

[org.apache.spark/spark-sql_2.11 "2.0.1"] [org.apache.spark/spark-hive_2.11 "2.0.1"]