search

holdenk / spark-testing-base

Base classes to use when writing tests with Spark
Apache License 2.0
1.52k stars 358 forks source link

SparkSession is null while using DataFrameSuiteBase #206

Closed fpopic closed 6 years ago

fpopic commented 7 years ago

I want to test/compare two dataframes using DataFrameSuiteBase but it appears that SparkSession object is null all the time and never gets created while tests are running?

package com.ta.geo.calculation

import com.holdenkarau.spark.testing.DataFrameSuiteBase
import com.ta.geo.calculation.GeodistanceCalculation.matchTrackingEventsWithNearestAirport
import com.ta.geo.datasource.ModelDatasource
import com.ta.geo.measure.{DistanceMeasure, HaversineDistance}
import com.ta.geo.model.{Airport, TrackingEvent}
import org.apache.spark.sql.{Dataset, SparkSession}
import org.scalatest.FlatSpec

class GeodistanceCalculationTest extends FlatSpec with DataFrameSuiteBase {

  override implicit def reuseContextIfPossible: Boolean = true

  implicit val sqlCtx: SparkSession = spark
  import sqlCtx.implicits._

  val ds: ModelDatasource = new ModelDatasource()(sqlCtx)
  val airports: Dataset[Airport] = ds.getAirports("data/optd-sample-20161201.csv")

  "GeolocationCalculation" should "return Berlin Tempelhof Airport (THF)." in {
    val events = Seq(TrackingEvent("TravelAudience", 52.4897337, 13.4554343)).toDS()

    val actual = matchTrackingEventsWithNearestAirport(events, airports, new HaversineDistance)
    val expected = Seq(("TravelAudience", "THF")).toDF("uuid", "iata_code")

    assertDataFrameEquals(actual, expected)
  }

}

my build.sbt

lazy val root = (project in file(".")).settings(
  ...
  assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
  parallelExecution in Test := false,
  fork in Test := true,
  javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-XX:+CMSClassUnloadingEnabled"),
)

val spark = "2.2.0"

libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-core" % spark,
  "org.apache.spark" %% "spark-sql" % spark,
  "org.scalactic" %% "scalactic" % "3.0.1",
  "com.holdenkarau" %% "spark-testing-base" % "2.2.0_0.7.2" % Test,
  "org.scalatest" %% "scalatest" % "3.0.1" % Test,
)

I run test within IntelliJ IDEA:

image

And everything works fine if I do it manually with BeforeAndAfter:

class GeodistanceCalculationTest extends FlatSpec with Matchers with BeforeAndAfter {

  implicit val spark: SparkSession =
    SparkSession.builder.master("local[*]")
      .config("spark.sql.shuffle.partitions", "4")
      .config("spark.driver.allowMultipleContexts", "true")
      .getOrCreate

  import spark.implicits._

  before {
    spark.newSession()
  }

  after {
    System.clearProperty("spark.driver.port")
  }

  val ds = new ModelDatasource()(spark)
  val airports = ds.getAirports("data/optd-sample-20161201.csv")

  "GeolocationCalculation" should "return Zagreb Airport (ZAG)." in {
    val events = Seq(TrackingEvent("Filip", 45.811289, 16.051475)).toDS

    val actual = matchTrackingEventsWithNearestAirport(events, airports, new HaversineDistance)
    val expected = Seq(("Filip", "ZAG")).toDF("uuid", "iata_code")

    actual.collect() shouldEqual expected.collect()
  }
}

Error (when I run debugger it shows that spark is null) :

/usr/lib/jvm/java-8-oracle/bin/java -Xms512M -Xmx2048M -javaagent:/home/fpopic/idea/lib/idea_rt.jar=44436:/home/fpopic/idea/bin -Dfile.encoding=UTF-8 -classpath /home/fpopic/.IntelliJIdea2017.2/config/plugins/Scala/lib/scala-plugin-runners.jar:/usr/lib/jvm/java-8-oracle/jre/lib/charsets.jar:/usr/lib/jvm/java-8-oracle/jre/lib/deploy.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/cldrdata.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/dnsns.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/jaccess.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/jfxrt.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/localedata.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/nashorn.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/sunec.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/sunjce_provider.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/sunpkcs11.jar:/usr/lib/jvm/java-8-oracle/jre/lib/ext/zipfs.jar:/usr/lib/jvm/java-8-oracle/jre/lib/javaws.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jce.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jfr.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jfxswt.jar:/usr/lib/jvm/java-8-oracle/jre/lib/jsse.jar:/usr/lib/jvm/java-8-oracle/jre/lib/management-agent.jar:/usr/lib/jvm/java-8-oracle/jre/lib/plugin.jar:/usr/lib/jvm/java-8-oracle/jre/lib/resources.jar:/usr/lib/jvm/java-8-oracle/jre/lib/rt.jar:/home/fpopic/Projects/ScalaProjects/TAChallenge/target/scala-2.11/test-classes:/home/fpopic/Projects/ScalaProjects/TAChallenge/target/scala-2.11/classes:/home/fpopic/.ivy2/cache/aopalliance/aopalliance/jars/aopalliance-1.0.jar:/home/fpopic/.ivy2/cache/org.scalacheck/scalacheck_2.11/jars/scalacheck_2.11-1.13.4.jar:/home/fpopic/.ivy2/cache/org.scala-sbt/test-interface/jars/test-interface-1.0.jar:/home/fpopic/.ivy2/cache/org.mortbay.jetty/jetty/jars/jetty-6.1.26.jar:/home/fpopic/.ivy2/cache/org.hamcrest/hamcrest-core/jars/hamcrest-core-1.3.jar:/home/fpopic/.ivy2/cache/org.eclipse.jetty/jetty-util/jars/jetty-util-9.3.11.v20160721.jar:/home/fpopic/.ivy2/cache/org.apache.htrace/htrace-core/jars/htrace-core-3.1.0-incubating.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-web-proxy/jars/hadoop-yarn-server-web-proxy-2.7.3-tests.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-web-proxy/jars/hadoop-yarn-server-web-proxy-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-tests/jars/hadoop-yarn-server-tests-2.7.3-tests.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-tests/jars/hadoop-yarn-server-tests-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-resourcemanager/jars/hadoop-yarn-server-resourcemanager-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-nodemanager/jars/hadoop-yarn-server-nodemanager-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-common/jars/hadoop-yarn-server-common-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-applicationhistoryservice/jars/hadoop-yarn-server-applicationhistoryservice-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-common/jars/hadoop-yarn-common-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-client/jars/hadoop-yarn-client-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-api/jars/hadoop-yarn-api-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-minicluster/jars/hadoop-minicluster-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-shuffle/jars/hadoop-mapreduce-client-shuffle-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-jobclient/jars/hadoop-mapreduce-client-jobclient-2.7.3-tests.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-jobclient/jars/hadoop-mapreduce-client-jobclient-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-hs/jars/hadoop-mapreduce-client-hs-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-core/jars/hadoop-mapreduce-client-core-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-common/jars/hadoop-mapreduce-client-common-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-app/jars/hadoop-mapreduce-client-app-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/jars/hadoop-hdfs-2.7.3-tests.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/jars/hadoop-hdfs-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-common/jars/hadoop-common-2.7.3-tests.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-common/jars/hadoop-common-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-client/jars/hadoop-client-2.7.3-tests.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-client/jars/hadoop-client-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-auth/jars/hadoop-auth-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-annotations/jars/hadoop-annotations-2.7.3.jar:/home/fpopic/.ivy2/cache/org.apache.curator/curator-recipes/bundles/curator-recipes-2.7.1.jar:/home/fpopic/.ivy2/cache/org.apache.curator/curator-framework/bundles/curator-framework-2.7.1.jar:/home/fpopic/.ivy2/cache/org.apache.curator/curator-client/bundles/curator-client-2.7.1.jar:/home/fpopic/.ivy2/cache/junit/junit/jars/junit-4.12.jar:/home/fpopic/.ivy2/cache/javax.servlet.jsp/jsp-api/jars/jsp-api-2.1.jar:/home/fpopic/.ivy2/cache/io.github.nicolasstucki/multisets_2.11/jars/multisets_2.11-0.4.jar:/home/fpopic/.ivy2/cache/commons-logging/commons-logging/jars/commons-logging-1.1.3.jar:/home/fpopic/.ivy2/cache/commons-daemon/commons-daemon/jars/commons-daemon-1.0.13.jar:/home/fpopic/.ivy2/cache/com.sun.xml.bind/jaxb-impl/jars/jaxb-impl-2.2.3-1.jar:/home/fpopic/.ivy2/cache/com.sun.jersey.contribs/jersey-guice/jars/jersey-guice-1.9.jar:/home/fpopic/.ivy2/cache/com.sun.jersey/jersey-server/bundles/jersey-server-1.9.jar:/home/fpopic/.ivy2/cache/com.sun.jersey/jersey-json/bundles/jersey-json-1.9.jar:/home/fpopic/.ivy2/cache/com.sun.jersey/jersey-core/bundles/jersey-core-1.9.jar:/home/fpopic/.ivy2/cache/com.sun.jersey/jersey-client/bundles/jersey-client-1.9.jar:/home/fpopic/.ivy2/cache/com.jcraft/jsch/jars/jsch-0.1.42.jar:/home/fpopic/.ivy2/cache/com.holdenkarau/spark-testing-base_2.11/jars/spark-testing-base_2.11-2.2.0_0.7.2.jar:/home/fpopic/.ivy2/cache/com.google.inject.extensions/guice-servlet/jars/guice-servlet-3.0.jar:/home/fpopic/.ivy2/cache/com.google.code.findbugs/jsr305/jars/jsr305-3.0.0.jar:/home/fpopic/.ivy2/cache/asm/asm/jars/asm-3.1.jar:/home/fpopic/.ivy2/cache/xmlenc/xmlenc/jars/xmlenc-0.52.jar:/home/fpopic/.ivy2/cache/xml-apis/xml-apis/jars/xml-apis-1.3.04.jar:/home/fpopic/.ivy2/cache/xerces/xercesImpl/jars/xercesImpl-2.9.1.jar:/home/fpopic/.ivy2/cache/oro/oro/jars/oro-2.0.8.jar:/home/fpopic/.ivy2/cache/org.xerial.snappy/snappy-java/bundles/snappy-java-1.1.2.6.jar:/home/fpopic/.ivy2/cache/org.tukaani/xz/jars/xz-1.0.jar:/home/fpopic/.ivy2/cache/org.spark-project.spark/unused/jars/unused-1.0.0.jar:/home/fpopic/.ivy2/cache/org.sonatype.sisu.inject/cglib/jars/cglib-2.2.1-v20090111.jar:/home/fpopic/.ivy2/cache/org.slf4j/slf4j-log4j12/jars/slf4j-log4j12-1.7.16.jar:/home/fpopic/.ivy2/cache/org.slf4j/slf4j-api/jars/slf4j-api-1.7.16.jar:/home/fpopic/.ivy2/cache/org.slf4j/jul-to-slf4j/jars/jul-to-slf4j-1.7.16.jar:/home/fpopic/.ivy2/cache/org.slf4j/jcl-over-slf4j/jars/jcl-over-slf4j-1.7.16.jar:/home/fpopic/.ivy2/cache/org.scalatest/scalatest_2.11/bundles/scalatest_2.11-3.0.1.jar:/home/fpopic/.ivy2/cache/org.scalactic/scalactic_2.11/bundles/scalactic_2.11-3.0.1.jar:/home/fpopic/.ivy2/cache/org.scala-lang.modules/scala-xml_2.11/bundles/scala-xml_2.11-1.0.5.jar:/home/fpopic/.ivy2/cache/org.scala-lang.modules/scala-parser-combinators_2.11/bundles/scala-parser-combinators_2.11-1.0.4.jar:/home/fpopic/.ivy2/cache/org.scala-lang/scalap/jars/scalap-2.11.11.jar:/home/fpopic/.ivy2/cache/org.scala-lang/scala-reflect/jars/scala-reflect-2.11.11.jar:/home/fpopic/.ivy2/cache/org.scala-lang/scala-compiler/jars/scala-compiler-2.11.11.jar:/home/fpopic/.ivy2/cache/org.roaringbitmap/RoaringBitmap/bundles/RoaringBitmap-0.5.11.jar:/home/fpopic/.ivy2/cache/org.objenesis/objenesis/jars/objenesis-2.1.jar:/home/fpopic/.ivy2/cache/org.mortbay.jetty/jetty-util/jars/jetty-util-6.1.26.jar:/home/fpopic/.ivy2/cache/org.json4s/json4s-jackson_2.11/jars/json4s-jackson_2.11-3.2.11.jar:/home/fpopic/.ivy2/cache/org.json4s/json4s-core_2.11/jars/json4s-core_2.11-3.2.11.jar:/home/fpopic/.ivy2/cache/org.json4s/json4s-ast_2.11/jars/json4s-ast_2.11-3.2.11.jar:/home/fpopic/.ivy2/cache/org.javassist/javassist/bundles/javassist-3.18.1-GA.jar:/home/fpopic/.ivy2/cache/org.htrace/htrace-core/jars/htrace-core-3.0.4.jar:/home/fpopic/.ivy2/cache/org.glassfish.jersey.media/jersey-media-jaxb/jars/jersey-media-jaxb-2.22.2.jar:/home/fpopic/.ivy2/cache/org.glassfish.jersey.core/jersey-server/jars/jersey-server-2.22.2.jar:/home/fpopic/.ivy2/cache/org.glassfish.jersey.core/jersey-common/jars/jersey-common-2.22.2.jar:/home/fpopic/.ivy2/cache/org.glassfish.jersey.core/jersey-client/jars/jersey-client-2.22.2.jar:/home/fpopic/.ivy2/cache/org.glassfish.jersey.containers/jersey-container-servlet-core/jars/jersey-container-servlet-core-2.22.2.jar:/home/fpopic/.ivy2/cache/org.glassfish.jersey.containers/jersey-container-servlet/jars/jersey-container-servlet-2.22.2.jar:/home/fpopic/.ivy2/cache/org.glassfish.jersey.bundles.repackaged/jersey-guava/bundles/jersey-guava-2.22.2.jar:/home/fpopic/.ivy2/cache/org.glassfish.hk2.external/javax.inject/jars/javax.inject-2.4.0-b34.jar:/home/fpopic/.ivy2/cache/org.glassfish.hk2.external/aopalliance-repackaged/jars/aopalliance-repackaged-2.4.0-b34.jar:/home/fpopic/.ivy2/cache/org.glassfish.hk2/osgi-resource-locator/jars/osgi-resource-locator-1.0.1.jar:/home/fpopic/.ivy2/cache/org.glassfish.hk2/hk2-utils/jars/hk2-utils-2.4.0-b34.jar:/home/fpopic/.ivy2/cache/org.glassfish.hk2/hk2-locator/jars/hk2-locator-2.4.0-b34.jar:/home/fpopic/.ivy2/cache/org.glassfish.hk2/hk2-api/jars/hk2-api-2.4.0-b34.jar:/home/fpopic/.ivy2/cache/org.fusesource.leveldbjni/leveldbjni-all/bundles/leveldbjni-all-1.8.jar:/home/fpopic/.ivy2/cache/org.codehaus.jettison/jettison/bundles/jettison-1.1.jar:/home/fpopic/.ivy2/cache/org.codehaus.janino/janino/jars/janino-3.0.0.jar:/home/fpopic/.ivy2/cache/org.codehaus.janino/commons-compiler/jars/commons-compiler-3.0.0.jar:/home/fpopic/.ivy2/cache/org.codehaus.jackson/jackson-xc/jars/jackson-xc-1.9.13.jar:/home/fpopic/.ivy2/cache/org.codehaus.jackson/jackson-mapper-asl/jars/jackson-mapper-asl-1.9.13.jar:/home/fpopic/.ivy2/cache/org.codehaus.jackson/jackson-jaxrs/jars/jackson-jaxrs-1.9.13.jar:/home/fpopic/.ivy2/cache/org.codehaus.jackson/jackson-core-asl/jars/jackson-core-asl-1.9.13.jar:/home/fpopic/.ivy2/cache/org.bouncycastle/bcprov-jdk15on/jars/bcprov-jdk15on-1.51.jar:/home/fpopic/.ivy2/cache/org.apache.zookeeper/zookeeper/jars/zookeeper-3.4.6.jar:/home/fpopic/.ivy2/cache/org.apache.xbean/xbean-asm5-shaded/bundles/xbean-asm5-shaded-4.4.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-unsafe_2.11/jars/spark-unsafe_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-tags_2.11/jars/spark-tags_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-sql_2.11/jars/spark-sql_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-sketch_2.11/jars/spark-sketch_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-network-shuffle_2.11/jars/spark-network-shuffle_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-network-common_2.11/jars/spark-network-common_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-launcher_2.11/jars/spark-launcher_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-core_2.11/jars/spark-core_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.spark/spark-catalyst_2.11/jars/spark-catalyst_2.11-2.2.0.jar:/home/fpopic/.ivy2/cache/org.apache.parquet/parquet-jackson/jars/parquet-jackson-1.8.2.jar:/home/fpopic/.ivy2/cache/org.apache.parquet/parquet-hadoop/jars/parquet-hadoop-1.8.2.jar:/home/fpopic/.ivy2/cache/org.apache.parquet/parquet-format/jars/parquet-format-2.3.1.jar:/home/fpopic/.ivy2/cache/org.apache.parquet/parquet-encoding/jars/parquet-encoding-1.8.2.jar:/home/fpopic/.ivy2/cache/org.apache.parquet/parquet-common/jars/parquet-common-1.8.2.jar:/home/fpopic/.ivy2/cache/org.apache.parquet/parquet-column/jars/parquet-column-1.8.2.jar:/home/fpopic/.ivy2/cache/org.apache.ivy/ivy/jars/ivy-2.4.0.jar:/home/fpopic/.ivy2/cache/org.apache.httpcomponents/httpcore/jars/httpcore-4.3.3.jar:/home/fpopic/.ivy2/cache/org.apache.httpcomponents/httpclient/jars/httpclient-4.3.6.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-server-common/jars/hadoop-yarn-server-common-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-common/jars/hadoop-yarn-common-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-client/jars/hadoop-yarn-client-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-yarn-api/jars/hadoop-yarn-api-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-shuffle/jars/hadoop-mapreduce-client-shuffle-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-jobclient/jars/hadoop-mapreduce-client-jobclient-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-core/jars/hadoop-mapreduce-client-core-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-common/jars/hadoop-mapreduce-client-common-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-mapreduce-client-app/jars/hadoop-mapreduce-client-app-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/jars/hadoop-hdfs-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-common/jars/hadoop-common-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-client/jars/hadoop-client-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-auth/jars/hadoop-auth-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.hadoop/hadoop-annotations/jars/hadoop-annotations-2.6.5.jar:/home/fpopic/.ivy2/cache/org.apache.directory.server/apacheds-kerberos-codec/bundles/apacheds-kerberos-codec-2.0.0-M15.jar:/home/fpopic/.ivy2/cache/org.apache.directory.server/apacheds-i18n/bundles/apacheds-i18n-2.0.0-M15.jar:/home/fpopic/.ivy2/cache/org.apache.directory.api/api-util/bundles/api-util-1.0.0-M20.jar:/home/fpopic/.ivy2/cache/org.apache.directory.api/api-asn1-api/bundles/api-asn1-api-1.0.0-M20.jar:/home/fpopic/.ivy2/cache/org.apache.curator/curator-recipes/bundles/curator-recipes-2.6.0.jar:/home/fpopic/.ivy2/cache/org.apache.curator/curator-framework/bundles/curator-framework-2.6.0.jar:/home/fpopic/.ivy2/cache/org.apache.curator/curator-client/bundles/curator-client-2.6.0.jar:/home/fpopic/.ivy2/cache/org.apache.commons/commons-math3/jars/commons-math3-3.4.1.jar:/home/fpopic/.ivy2/cache/org.apache.commons/commons-lang3/jars/commons-lang3-3.5.jar:/home/fpopic/.ivy2/cache/org.apache.commons/commons-crypto/jars/commons-crypto-1.0.0.jar:/home/fpopic/.ivy2/cache/org.apache.commons/commons-compress/jars/commons-compress-1.4.1.jar:/home/fpopic/.ivy2/cache/org.apache.avro/avro-mapred/jars/avro-mapred-1.7.7-hadoop2.jar:/home/fpopic/.ivy2/cache/org.apache.avro/avro-ipc/jars/avro-ipc-1.7.7-tests.jar:/home/fpopic/.ivy2/cache/org.apache.avro/avro-ipc/jars/avro-ipc-1.7.7.jar:/home/fpopic/.ivy2/cache/org.apache.avro/avro/jars/avro-1.7.7.jar:/home/fpopic/.ivy2/cache/org.antlr/antlr4-runtime/jars/antlr4-runtime-4.5.3.jar:/home/fpopic/.ivy2/cache/net.sf.py4j/py4j/jars/py4j-0.10.4.jar:/home/fpopic/.ivy2/cache/net.razorvine/pyrolite/jars/pyrolite-4.13.jar:/home/fpopic/.ivy2/cache/net.jpountz.lz4/lz4/jars/lz4-1.3.0.jar:/home/fpopic/.ivy2/cache/net.java.dev.jets3t/jets3t/jars/jets3t-0.9.3.jar:/home/fpopic/.ivy2/cache/net.iharder/base64/jars/base64-2.3.8.jar:/home/fpopic/.ivy2/cache/mx4j/mx4j/jars/mx4j-3.0.2.jar:/home/fpopic/.ivy2/cache/log4j/log4j/bundles/log4j-1.2.17.jar:/home/fpopic/.ivy2/cache/jline/jline/jars/jline-0.9.94.jar:/home/fpopic/.ivy2/cache/javax.xml.stream/stax-api/jars/stax-api-1.0-2.jar:/home/fpopic/.ivy2/cache/javax.xml.bind/jaxb-api/jars/jaxb-api-2.2.2.jar:/home/fpopic/.ivy2/cache/javax.ws.rs/javax.ws.rs-api/jars/javax.ws.rs-api-2.0.1.jar:/home/fpopic/.ivy2/cache/javax.validation/validation-api/jars/validation-api-1.1.0.Final.jar:/home/fpopic/.ivy2/cache/javax.servlet/javax.servlet-api/jars/javax.servlet-api-3.1.0.jar:/home/fpopic/.ivy2/cache/javax.mail/mail/jars/mail-1.4.7.jar:/home/fpopic/.ivy2/cache/javax.inject/javax.inject/jars/javax.inject-1.jar:/home/fpopic/.ivy2/cache/javax.annotation/javax.annotation-api/jars/javax.annotation-api-1.2.jar:/home/fpopic/.ivy2/cache/javax.activation/activation/jars/activation-1.1.1.jar:/home/fpopic/.ivy2/cache/io.netty/netty-all/jars/netty-all-4.0.43.Final.jar:/home/fpopic/.ivy2/cache/io.netty/netty/bundles/netty-3.9.9.Final.jar:/home/fpopic/.ivy2/cache/io.dropwizard.metrics/metrics-jvm/bundles/metrics-jvm-3.1.2.jar:/home/fpopic/.ivy2/cache/io.dropwizard.metrics/metrics-json/bundles/metrics-json-3.1.2.jar:/home/fpopic/.ivy2/cache/io.dropwizard.metrics/metrics-graphite/bundles/metrics-graphite-3.1.2.jar:/home/fpopic/.ivy2/cache/io.dropwizard.metrics/metrics-core/bundles/metrics-core-3.1.2.jar:/home/fpopic/.ivy2/cache/commons-net/commons-net/jars/commons-net-2.2.jar:/home/fpopic/.ivy2/cache/commons-lang/commons-lang/jars/commons-lang-2.6.jar:/home/fpopic/.ivy2/cache/commons-io/commons-io/jars/commons-io-2.4.jar:/home/fpopic/.ivy2/cache/commons-httpclient/commons-httpclient/jars/commons-httpclient-3.1.jar:/home/fpopic/.ivy2/cache/commons-digester/commons-digester/jars/commons-digester-1.8.jar:/home/fpopic/.ivy2/cache/commons-configuration/commons-configuration/jars/commons-configuration-1.6.jar:/home/fpopic/.ivy2/cache/commons-collections/commons-collections/jars/commons-collections-3.2.2.jar:/home/fpopic/.ivy2/cache/commons-codec/commons-codec/jars/commons-codec-1.10.jar:/home/fpopic/.ivy2/cache/commons-cli/commons-cli/jars/commons-cli-1.2.jar:/home/fpopic/.ivy2/cache/commons-beanutils/commons-beanutils-core/jars/commons-beanutils-core-1.8.0.jar:/home/fpopic/.ivy2/cache/commons-beanutils/commons-beanutils/jars/commons-beanutils-1.7.0.jar:/home/fpopic/.ivy2/cache/com.univocity/univocity-parsers/jars/univocity-parsers-2.2.1.jar:/home/fpopic/.ivy2/cache/com.twitter/chill_2.11/jars/chill_2.11-0.8.0.jar:/home/fpopic/.ivy2/cache/com.twitter/chill-java/jars/chill-java-0.8.0.jar:/home/fpopic/.ivy2/cache/com.thoughtworks.paranamer/paranamer/jars/paranamer-2.6.jar:/home/fpopic/.ivy2/cache/com.ning/compress-lzf/bundles/compress-lzf-1.0.3.jar:/home/fpopic/.ivy2/cache/com.jamesmurty.utils/java-xmlbuilder/jars/java-xmlbuilder-1.0.jar:/home/fpopic/.ivy2/cache/com.google.protobuf/protobuf-java/bundles/protobuf-java-2.5.0.jar:/home/fpopic/.ivy2/cache/com.google.inject/guice/jars/guice-3.0.jar:/home/fpopic/.ivy2/cache/com.google.guava/guava/jars/guava-11.0.2.jar:/home/fpopic/.ivy2/cache/com.google.code.gson/gson/jars/gson-2.2.4.jar:/home/fpopic/.ivy2/cache/com.google.code.findbugs/jsr305/jars/jsr305-1.3.9.jar:/home/fpopic/.ivy2/cache/com.fasterxml.jackson.module/jackson-module-scala_2.11/bundles/jackson-module-scala_2.11-2.6.5.jar:/home/fpopic/.ivy2/cache/com.fasterxml.jackson.module/jackson-module-paranamer/bundles/jackson-module-paranamer-2.6.5.jar:/home/fpopic/.ivy2/cache/com.fasterxml.jackson.core/jackson-databind/bundles/jackson-databind-2.6.5.jar:/home/fpopic/.ivy2/cache/com.fasterxml.jackson.core/jackson-core/bundles/jackson-core-2.6.5.jar:/home/fpopic/.ivy2/cache/com.fasterxml.jackson.core/jackson-annotations/bundles/jackson-annotations-2.6.5.jar:/home/fpopic/.ivy2/cache/com.esotericsoftware/minlog/bundles/minlog-1.3.0.jar:/home/fpopic/.ivy2/cache/com.esotericsoftware/kryo-shaded/bundles/kryo-shaded-3.0.3.jar:/home/fpopic/.ivy2/cache/com.clearspring.analytics/stream/jars/stream-2.7.0.jar:/home/fpopic/.ivy2/cache/org.scala-lang/scala-library/jars/scala-library-2.11.11.jar org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner -s com.ta.geo.measure.HaversineDistanceTest -s com.ta.geo.calculation.GeodistanceCalculationTest -C org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestReporter -showProgressMessages true
Testing started at 14:02 ...

An exception or error caused a run to abort. 
java.lang.NullPointerException
    at com.ta.geo.datasource.ModelDatasource.getAirports(ModelDatasource.scala:13)
    at com.ta.geo.calculation.GeodistanceCalculationTest.<init>(GeodistanceCalculationTest.scala:25)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    at java.lang.Class.newInstance(Class.java:442)
    at org.scalatest.tools.Runner$.genSuiteConfig(Runner.scala:1422)
    at org.scalatest.tools.Runner$$anonfun$31.apply(Runner.scala:1236)
    at org.scalatest.tools.Runner$$anonfun$31.apply(Runner.scala:1235)
    at scala.collection.immutable.List.map(List.scala:284)
    at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1235)
    at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1011)
    at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1010)
    at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1500)
    at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1010)
    at org.scalatest.tools.Runner$.run(Runner.scala:850)
    at org.scalatest.tools.Runner.run(Runner.scala)
    at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138)
    at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28)

Process finished with exit code 0
holdenk commented 6 years ago

So if you move the creating the dataset and test inside of the test function instead of globally for the class does that still have the issue?

fpopic commented 6 years ago

OK, it means that Spark things can live inside test function only, I can live with that.

I added: "org.apache.spark" %% "spark-hive" % spark % Test to library dependencies and now everything works as expected.

Thank you very much!

minnieshi commented 2 years ago

@fpopic or @holdenk does this mean we can not really fake 'shared' data frame to be used in different test functions?

minnieshi commented 2 years ago

I found another issued talked about using super.beforeAll see #180