search

AbsaOSS / spline-spark-agent

Spline agent for Apache Spark
https://absaoss.github.io/spline/
Apache License 2.0
183 stars 93 forks source link

NPE in `NonPersistentActionsCapturePlugin` #634

Closed wajda closed 1 year ago

wajda commented 1 year ago 
ERROR za.co.absa.spline.agent.SplineAgent  - Unexpected error occurred during lineage processing for application: 478a0385-d1e7-4a4d-a78e-e83a718ae09d #local-1679916415137
java.lang.RuntimeException: Write extraction failed for: class org.apache.spark.sql.catalyst.plans.logical.Aggregate
    at za.co.absa.spline.harvester.LineageHarvester.tryExtractWriteCommand(LineageHarvester.scala:153)
    at za.co.absa.spline.harvester.LineageHarvester.harvest(LineageHarvester.scala:61)
    at za.co.absa.spline.agent.SplineAgent$$anon$1.$anonfun$handle$1(SplineAgent.scala:91)
    at za.co.absa.spline.agent.SplineAgent$$anon$1.withErrorHandling(SplineAgent.scala:100)
    at za.co.absa.spline.agent.SplineAgent$$anon$1.handle(SplineAgent.scala:72)
    at za.co.absa.spline.harvester.listener.QueryExecutionListenerDelegate.onSuccess(QueryExecutionListenerDelegate.scala:28)
    at org.apache.spark.sql.util.ExecutionListenerManager.$anonfun$onSuccess$2(QueryExecutionListener.scala:129)
    at org.apache.spark.sql.util.ExecutionListenerManager.$anonfun$onSuccess$2$adapted(QueryExecutionListener.scala:128)
    at org.apache.spark.sql.util.ExecutionListenerManager.$anonfun$withErrorHandling$1(QueryExecutionListener.scala:157)
    at org.apache.spark.sql.util.ExecutionListenerManager.$anonfun$withErrorHandling$1$adapted(QueryExecutionListener.scala:155)
    at scala.collection.immutable.List.foreach(List.scala:431)
    at scala.collection.generic.TraversableForwarder.foreach(TraversableForwarder.scala:38)
    at scala.collection.generic.TraversableForwarder.foreach$(TraversableForwarder.scala:38)
    at scala.collection.mutable.ListBuffer.foreach(ListBuffer.scala:47)
    at org.apache.spark.sql.util.ExecutionListenerManager.withErrorHandling(QueryExecutionListener.scala:155)
    at org.apache.spark.sql.util.ExecutionListenerManager.$anonfun$onSuccess$1(QueryExecutionListener.scala:128)
    at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
    at org.apache.spark.sql.util.ExecutionListenerManager.readLock(QueryExecutionListener.scala:168)
    at org.apache.spark.sql.util.ExecutionListenerManager.onSuccess(QueryExecutionListener.scala:128)
    at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3372)
    at org.apache.spark.sql.Dataset.count(Dataset.scala:2835)
    at za.co.absa.spline.NonPersistentActionsSpec.$anonfun$new$1(NonPersistentActionsSpec.scala:34)
    at za.co.absa.spline.NonPersistentActionsSpec.$anonfun$new$1$adapted(NonPersistentActionsSpec.scala:34)
    at za.co.absa.spline.NonPersistentActionsSpec.$anonfun$new$12(NonPersistentActionsSpec.scala:48)
    at za.co.absa.spline.test.fixture.spline.LineageCaptor.lineageOf(LineageCaptor.scala:47)
    at za.co.absa.spline.NonPersistentActionsSpec.$anonfun$new$11(NonPersistentActionsSpec.scala:45)
    at za.co.absa.spline.test.fixture.spline.SplineFixture.withLineageTracking(SplineFixture.scala:24)
    at za.co.absa.spline.test.fixture.spline.SplineFixture.withLineageTracking$(SplineFixture.scala:23)
    at za.co.absa.spline.NonPersistentActionsSpec.withLineageTracking(NonPersistentActionsSpec.scala:27)
    at za.co.absa.spline.NonPersistentActionsSpec.$anonfun$new$10(NonPersistentActionsSpec.scala:43)
    at za.co.absa.spline.test.fixture.SparkFixture.withIsolatedSparkSession(SparkFixture.scala:57)
    at za.co.absa.spline.test.fixture.SparkFixture.withIsolatedSparkSession$(SparkFixture.scala:54)
    at za.co.absa.spline.NonPersistentActionsSpec.withIsolatedSparkSession(NonPersistentActionsSpec.scala:27)
    at za.co.absa.spline.NonPersistentActionsSpec.$anonfun$new$8(NonPersistentActionsSpec.scala:42)
    at org.scalatest.flatspec.AsyncFlatSpecLike.transformToOutcomeParam$1(AsyncFlatSpecLike.scala:138)
    at org.scalatest.flatspec.AsyncFlatSpecLike.$anonfun$registerTestToRun$1(AsyncFlatSpecLike.scala:144)
    at org.scalatest.AsyncTestSuite.$anonfun$transformToOutcome$1(AsyncTestSuite.scala:240)
    at org.scalatest.flatspec.AsyncFlatSpecLike$$anon$5.apply(AsyncFlatSpecLike.scala:1697)
    at org.scalatest.AsyncTestSuite.withFixture(AsyncTestSuite.scala:313)
    at org.scalatest.AsyncTestSuite.withFixture$(AsyncTestSuite.scala:312)
    at org.scalatest.flatspec.AsyncFlatSpec.withFixture(AsyncFlatSpec.scala:2221)
    at org.scalatest.flatspec.AsyncFlatSpecLike.invokeWithAsyncFixture$1(AsyncFlatSpecLike.scala:1695)
    at org.scalatest.flatspec.AsyncFlatSpecLike.$anonfun$runTest$1(AsyncFlatSpecLike.scala:1709)
    at org.scalatest.AsyncSuperEngine.runTestImpl(AsyncEngine.scala:374)
    at org.scalatest.flatspec.AsyncFlatSpecLike.runTest(AsyncFlatSpecLike.scala:1709)
    at org.scalatest.flatspec.AsyncFlatSpecLike.runTest$(AsyncFlatSpecLike.scala:1688)
    at org.scalatest.flatspec.AsyncFlatSpec.runTest(AsyncFlatSpec.scala:2221)
    at org.scalatest.flatspec.AsyncFlatSpecLike.$anonfun$runTests$1(AsyncFlatSpecLike.scala:1767)
    at org.scalatest.AsyncSuperEngine.$anonfun$runTestsInBranch$1(AsyncEngine.scala:432)
    at scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
    at scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
    at scala.collection.immutable.List.foldLeft(List.scala:91)
    at org.scalatest.AsyncSuperEngine.traverseSubNodes$1(AsyncEngine.scala:406)
    at org.scalatest.AsyncSuperEngine.runTestsInBranch(AsyncEngine.scala:487)
    at org.scalatest.AsyncSuperEngine.runTestsImpl(AsyncEngine.scala:555)
    at org.scalatest.flatspec.AsyncFlatSpecLike.runTests(AsyncFlatSpecLike.scala:1767)
    at org.scalatest.flatspec.AsyncFlatSpecLike.runTests$(AsyncFlatSpecLike.scala:1766)
    at org.scalatest.flatspec.AsyncFlatSpec.runTests(AsyncFlatSpec.scala:2221)
    at org.scalatest.Suite.run(Suite.scala:1112)
    at org.scalatest.Suite.run$(Suite.scala:1094)
    at org.scalatest.flatspec.AsyncFlatSpec.org$scalatest$flatspec$AsyncFlatSpecLike$$super$run(AsyncFlatSpec.scala:2221)
    at org.scalatest.flatspec.AsyncFlatSpecLike.$anonfun$run$1(AsyncFlatSpecLike.scala:1812)
    at org.scalatest.AsyncSuperEngine.runImpl(AsyncEngine.scala:625)
    at org.scalatest.flatspec.AsyncFlatSpecLike.run(AsyncFlatSpecLike.scala:1812)
    at org.scalatest.flatspec.AsyncFlatSpecLike.run$(AsyncFlatSpecLike.scala:1810)
    at org.scalatest.flatspec.AsyncFlatSpec.run(AsyncFlatSpec.scala:2221)
    at org.scalatest.Suite.callExecuteOnSuite$1(Suite.scala:1175)
    at org.scalatest.Suite.$anonfun$runNestedSuites$1(Suite.scala:1222)
    at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
    at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
    at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
    at org.scalatest.Suite.runNestedSuites(Suite.scala:1220)
    at org.scalatest.Suite.runNestedSuites$(Suite.scala:1154)
    at org.scalatest.tools.DiscoverySuite.runNestedSuites(DiscoverySuite.scala:30)
    at org.scalatest.Suite.run(Suite.scala:1109)
    at org.scalatest.Suite.run$(Suite.scala:1094)
    at org.scalatest.tools.DiscoverySuite.run(DiscoverySuite.scala:30)
    at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
    at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13(Runner.scala:1314)
    at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13$adapted(Runner.scala:1308)
    at scala.collection.immutable.List.foreach(List.scala:431)
    at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1308)
    at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:993)
    at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:971)
    at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1474)
    at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:971)
    at org.scalatest.tools.Runner$.main(Runner.scala:775)
    at org.scalatest.tools.Runner.main(Runner.scala)
Caused by: java.lang.NullPointerException
    at za.co.absa.spline.harvester.plugin.embedded.NonPersistentActionsCapturePlugin$.LocalMacAddressString$lzycompute(NonPersistentActionsCapturePlugin.scala:64)
    at za.co.absa.spline.harvester.plugin.embedded.NonPersistentActionsCapturePlugin$.za$co$absa$spline$harvester$plugin$embedded$NonPersistentActionsCapturePlugin$$LocalMacAddressString(NonPersistentActionsCapturePlugin.scala:63)
    at za.co.absa.spline.harvester.plugin.embedded.NonPersistentActionsCapturePlugin$$anonfun$1.applyOrElse(NonPersistentActionsCapturePlugin.scala:47)
    at za.co.absa.spline.harvester.plugin.embedded.NonPersistentActionsCapturePlugin$$anonfun$1.applyOrElse(NonPersistentActionsCapturePlugin.scala:44)
    at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:175)
    at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:176)
    at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:176)
    at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:176)
    at scala.PartialFunction$Lifted.apply(PartialFunction.scala:228)
    at scala.PartialFunction$Lifted.apply(PartialFunction.scala:224)
    at za.co.absa.spline.harvester.builder.write.PluggableWriteCommandExtractor.asWriteCommand(PluggableWriteCommandExtractor.scala:45)
    at za.co.absa.spline.harvester.LineageHarvester.$anonfun$tryExtractWriteCommand$1(LineageHarvester.scala:145)
    at scala.util.Try$.apply(Try.scala:213)
    at za.co.absa.spline.harvester.LineageHarvester.tryExtractWriteCommand(LineageHarvester.scala:145)
    ... 87 more
wajda commented 1 year ago

the error happens seemingly intermittently