search

AbsaOSS / ABRiS

Avro SerDe for Apache Spark structured APIs.
Apache License 2.0
231 stars 76 forks source link

Issues running inside Scala notebook on databricks #349

Closed luisvicenteatprima closed 11 months ago

luisvicenteatprima commented 11 months ago

I'm having problems with this bit of code:

import za.co.absa.abris.avro.functions.from_avro
import za.co.absa.abris.config.{AbrisConfig, FromAvroConfig}

val schemaRegistryKey = ""//dbutils.secrets.get("", "")
val schemaRegistrySecret = ""//dbutils.secrets.get("", "")

val schemaRegistryConfig = Map(
  AbrisConfig.SCHEMA_REGISTRY_URL -> "",
  "confluent.schema.registry.basic.auth.credentials.source" -> "USER_INFO",
  "confluent.schema.registry.basic.auth.user.info" -> s"$schemaRegistryKey:$schemaRegistrySecret",
)

val abrisConfig: FromAvroConfig = AbrisConfig.fromConfluentAvro.downloadReaderSchemaByLatestVersion.andTopicNameStrategy(topic).usingSchemaRegistry(schemaRegistryConfig)

The last line triggers a:

Caused by: java.lang.ClassNotFoundException: org.apache.kafka.common.config.AbstractConfig
    at java.net.URLClassLoader.findClass(URLClassLoader.java:387)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
    at com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader.loadClass(ClassLoaders.scala:151)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
    at java.lang.ClassLoader.defineClass1(Native Method)
    at java.lang.ClassLoader.defineClass(ClassLoader.java:757)
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:473)
    at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
    at com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader.loadClass(ClassLoaders.scala:151)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
    at java.lang.ClassLoader.defineClass1(Native Method)
    at java.lang.ClassLoader.defineClass(ClassLoader.java:757)
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:473)
    at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
    at com.databricks.backend.daemon.driver.ClassLoaders$LibraryClassLoader.loadClass(ClassLoaders.scala:151)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
    at za.co.absa.abris.avro.registry.ConfluentRegistryClient$.za$co$absa$abris$avro$registry$ConfluentRegistryClient$$createClient(ConfluentRegistryClient.scala:31)
    at za.co.absa.abris.avro.registry.ConfluentRegistryClient.<init>(ConfluentRegistryClient.scala:25)
    at za.co.absa.abris.avro.read.confluent.SchemaManagerFactory$.$anonfun$getOrCreateRegistryClient$1(SchemaManagerFactory.scala:73)
    at scala.collection.concurrent.TrieMap.getOrElseUpdate(TrieMap.scala:895)
    at za.co.absa.abris.avro.read.confluent.SchemaManagerFactory$.getOrCreateRegistryClient(SchemaManagerFactory.scala:53)
    at za.co.absa.abris.avro.read.confluent.SchemaManagerFactory$.create(SchemaManagerFactory.scala:49)
    at za.co.absa.abris.config.FromSchemaDownloadingConfigFragment.usingSchemaRegistry(Config.scala:250)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(command-3834262893998825:13)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw$$iw$$iw$$iw.<init>(command-3834262893998825:60)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw$$iw$$iw.<init>(command-3834262893998825:62)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw$$iw.<init>(command-3834262893998825:64)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw$$iw.<init>(command-3834262893998825:66)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$$iw.<init>(command-3834262893998825:68)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read.<init>(command-3834262893998825:70)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$.<init>(command-3834262893998825:74)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$read$.<clinit>(command-3834262893998825)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$eval$.$print$lzycompute(<notebook>:7)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$eval$.$print(<notebook>:6)
    at $line7f3bcdbfa6fd408ab358418ea95bdd4929.$eval.$print(<notebook>)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:747)
    at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1020)
    at scala.tools.nsc.interpreter.IMain.$anonfun$interpret$1(IMain.scala:568)
    at scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:36)
    at scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:116)
    at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:41)
    at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:567)
    at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:594)
    at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:564)
    at com.databricks.backend.daemon.driver.DriverILoop.execute(DriverILoop.scala:219)
    at com.databricks.backend.daemon.driver.ScalaDriverLocal.$anonfun$repl$1(ScalaDriverLocal.scala:225)
    at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
    at com.databricks.backend.daemon.driver.DriverLocal$TrapExitInternal$.trapExit(DriverLocal.scala:1013)
    at com.databricks.backend.daemon.driver.DriverLocal$TrapExit$.apply(DriverLocal.scala:966)
    at com.databricks.backend.daemon.driver.ScalaDriverLocal.repl(ScalaDriverLocal.scala:225)
    at com.databricks.backend.daemon.driver.DriverLocal.$anonfun$execute$13(DriverLocal.scala:644)
    at com.databricks.logging.Log4jUsageLoggingShim$.$anonfun$withAttributionContext$1(Log4jUsageLoggingShim.scala:33)
    at scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)
    at com.databricks.logging.AttributionContext$.withValue(AttributionContext.scala:94)
    at com.databricks.logging.Log4jUsageLoggingShim$.withAttributionContext(Log4jUsageLoggingShim.scala:31)
    at com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:205)
    at com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:204)
    at com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:59)
    at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:240)
    at com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:240)
    at com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:225)
    at com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:59)
    at com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:621)
    at com.databricks.backend.daemon.driver.DriverWrapper.$anonfun$tryExecutingCommand$1(DriverWrapper.scala:615)
    at scala.util.Try$.apply(Try.scala:213)
    at com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:607)
    at com.databricks.backend.daemon.driver.DriverWrapper.executeCommandAndGetError(DriverWrapper.scala:526)
    at com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:561)
    at com.databricks.backend.daemon.driver.DriverWrapper.runInnerLoop(DriverWrapper.scala:431)
    at com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:374)
    at com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:225)
    at java.lang.Thread.run(Thread.java:750)

This happens with their runtimes that support Spark 3.3.2 and 3.2.1; the library is properly installed on startup. Has you ever seen something like that?

luisvicenteatprima commented 11 months ago

Just replying to myself... you need to install org.apache.kafka:kafka-clients:6.2.1-ccs