search

sbt / sbt-assembly

Deploy über-JARs. Restart processes. (port of codahale/assembly-sbt)
MIT License
1.95k stars 224 forks source link

ASM SignatureReader java.lang.IllegalArgumentException #401

Closed zzvara closed 1 year ago

zzvara commented 4 years ago

The below shading worked correctly earlier.

SBT 1.3.10, Scala 2.12.11 openjdk 11.0.6 2020-01-14 OpenJDK Runtime Environment AdoptOpenJDK (build 11.0.6+10) OpenJDK 64-Bit Server VM AdoptOpenJDK (build 11.0.6+10, mixed mode)

Plugins:

addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
addSbtPlugin("com.github.cb372" % "sbt-explicit-dependencies" % "0.2.13")
addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1")

Build model:

import Dependencies._
import sbtassembly.MergeStrategy

import scala.io.Source

updateOptions := updateOptions.value.withCachedResolution(false).withLatestSnapshots(true)

Global / onChangedBuildSource := ReloadOnSourceChanges

scalaVersion := "2.12.11"

parallelExecution := false

Project.inConfig(Test)(baseAssemblySettings)

lazy val shadeRules = Seq()

lazy val mergeStrategy: PartialFunction[String, MergeStrategy] = {
  case x if x.endsWith("module-info.class") => MergeStrategy.rename
  case x if x.endsWith("actor.conf") => MergeStrategy.first
  case x if x.endsWith("scala-java8-compat.properties") => MergeStrategy.first
}

lazy val commonSettings = Seq(
  organizationName := "redacted",
  organization := "redacted",
  version := {
    val v = Source.fromFile("version", "UTF-8").mkString
    if (!v.matches("""^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(-SNAPSHOT)?$""")) {
      throw new RuntimeException("Invalid version format!")
    }
    v
  },
  scalaVersion := "2.12.11",

  fork := true,

  logLevel in test := Level.Debug,
  test in assembly := {},
  test in assemblyPackageDependency := {},

  unmanagedBase := baseDirectory.value / "library",

  sources in (Compile, doc) := Seq.empty,

  publishArtifact in (Compile, packageDoc) := false,
  publishArtifact in (Test, packageDoc) := false,
  publishArtifact in (Test, packageBin) := true,
  publishArtifact in (Test, packageSrc) := true,

  assemblyShadeRules in assembly := Seq(
    ShadeRule
      .zap("org.apache.spark.unused.UnusedStubClass")
      .inAll
  ),
  assemblyShadeRules in assemblyPackageDependency := Seq(),

  assemblyMergeStrategy in assembly := mergeStrategy.orElse {
    case x =>
      val oldStrategy = (assemblyMergeStrategy in assembly).value
      oldStrategy(x)
  },
  assemblyMergeStrategy in assemblyPackageDependency := mergeStrategy.orElse {
    case x =>
      val oldStrategy = (assemblyMergeStrategy in assembly).value
      oldStrategy(x)
  },

  publishConfiguration := publishConfiguration.value.withOverwrite(true),
  publishLocalConfiguration := publishLocalConfiguration.value.withOverwrite(true),

  publishTo := Some("Artifactory Realm" at s"redacted"),

  credentials += Credentials(Path.userHome / ".sbt" / ".credentials"),

  resolvers ++= Seq(
    "Maven Central" at "https://repo1.maven.org/maven2/",
    "Artifactory Realm" at s"redacted"
  )
)

lazy val core = (project in file("core")).
  settings(commonSettings: _*).
  settings(
    name := "core",
    libraryDependencies ++= coreDependencies
  )

lazy val processing = (project in file("processing")).
  settings(commonSettings: _*).
  dependsOn(
    core % "test->test;compile->compile").
  settings(
    name := "processing"
  )

lazy val root = (project in file(".")).
  settings(commonSettings: _*).
  aggregate(core, processing).
  dependsOn(
    core % "test->test;compile->compile",
    processing % "test->test;compile->compile")

Command:

sbt assembly

Error:

[error] java.io.IOException: Unable to transform redacted/common/payment/Transaction.class
[error]         at org.pantsbuild.jarjar.util.JarTransformer.process(JarTransformer.java:41)
[error]         at org.pantsbuild.jarjar.util.JarProcessorChain.process(JarProcessorChain.java:38)
[error]         at org.pantsbuild.jarjar.JJProcessor.process(JJProcessor.scala:108)
[error]         at com.eed3si9n.jarjarabrams.Shader$.$anonfun$shadeDirectory$7(Shader.scala:57)
[error]         at scala.collection.Iterator.foreach(Iterator.scala:941)
[error]         at scala.collection.Iterator.foreach$(Iterator.scala:941)
[error]         at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
[error]         at scala.collection.IterableLike.foreach(IterableLike.scala:74)
[error]         at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
[error]         at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
[error]         at com.eed3si9n.jarjarabrams.Shader$.shadeDirectory(Shader.scala:51)
[error]         at sbtassembly.Assembly$.$anonfun$assembleMappings$11(Assembly.scala:252)
[error]         at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
[error]         at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
[error]         at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
[error]         at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1056)
[error]         at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
[error]         at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[error]         at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
[error]         at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
[error]         at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
[error]         at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1053)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:160)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal$(Tasks.scala:157)
[error]         at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.internal(Tasks.scala:440)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:150)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
[error]         at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
[error]         at java.base/java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
[error]         at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
[error]         at java.base/java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:396)
[error]         at java.base/java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:721)
[error]         at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
[error]         at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
[error]         at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
[error]         at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
[error]         at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
[error]         at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
[error]         at scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult(Tasks.scala:555)
[error]         at scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult$(Tasks.scala:555)
[error]         at scala.collection.parallel.ExecutionContextTaskSupport.executeAndWaitResult(TaskSupport.scala:84)
[error]         at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:960)
[error]         at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
[error]         at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[error]         at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
[error]         at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
[error]         at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
[error]         at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:955)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
[error]         at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
[error]         at java.base/java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
[error]         at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
[error]         at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
[error]         at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
[error]         at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
[error]         at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:177)
[error] Caused by: java.lang.IllegalArgumentException
[error]         at org.objectweb.asm.signature.SignatureReader.parseType(SignatureReader.java:249)
[error]         at org.objectweb.asm.signature.SignatureReader.accept(SignatureReader.java:124)
[error]         at org.objectweb.asm.commons.Remapper.mapSignature(Remapper.java:210)
[error]         at org.objectweb.asm.commons.ClassRemapper.visit(ClassRemapper.java:92)
[error]         at org.objectweb.asm.ClassReader.accept(ClassReader.java:524)
[error]         at org.objectweb.asm.ClassReader.accept(ClassReader.java:391)
[error]         at org.pantsbuild.jarjar.util.JarTransformer.process(JarTransformer.java:39)
[error]         at org.pantsbuild.jarjar.util.JarProcessorChain.process(JarProcessorChain.java:38)
[error]         at org.pantsbuild.jarjar.JJProcessor.process(JJProcessor.scala:108)
[error]         at com.eed3si9n.jarjarabrams.Shader$.$anonfun$shadeDirectory$7(Shader.scala:57)
[error]         at scala.collection.Iterator.foreach(Iterator.scala:941)
[error]         at scala.collection.Iterator.foreach$(Iterator.scala:941)
[error]         at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)
[error]         at scala.collection.IterableLike.foreach(IterableLike.scala:74)
[error]         at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
[error]         at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
[error]         at com.eed3si9n.jarjarabrams.Shader$.shadeDirectory(Shader.scala:51)
[error]         at sbtassembly.Assembly$.$anonfun$assembleMappings$11(Assembly.scala:252)
[error]         at scala.collection.parallel.AugmentedIterableIterator.map2combiner(RemainsIterator.scala:116)
[error]         at scala.collection.parallel.AugmentedIterableIterator.map2combiner$(RemainsIterator.scala:113)
[error]         at scala.collection.parallel.immutable.ParVector$ParVectorIterator.map2combiner(ParVector.scala:66)
[error]         at scala.collection.parallel.ParIterableLike$Map.leaf(ParIterableLike.scala:1056)
[error]         at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
[error]         at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[error]         at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
[error]         at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
[error]         at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
[error]         at scala.collection.parallel.ParIterableLike$Map.tryLeaf(ParIterableLike.scala:1053)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:160)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal$(Tasks.scala:157)
[error]         at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.internal(Tasks.scala:440)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:150)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
[error]         at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
[error]         at java.base/java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
[error]         at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
[error]         at java.base/java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:396)
[error]         at java.base/java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:721)
[error]         at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:379)
[error]         at scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:379)
[error]         at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:440)
[error]         at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:423)
[error]         at scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:416)
[error]         at scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:60)
[error]         at scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult(Tasks.scala:555)
[error]         at scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult$(Tasks.scala:555)
[error]         at scala.collection.parallel.ExecutionContextTaskSupport.executeAndWaitResult(TaskSupport.scala:84)
[error]         at scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:960)
[error]         at scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:53)
[error]         at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
[error]         at scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:67)
[error]         at scala.collection.parallel.Task.tryLeaf(Tasks.scala:56)
[error]         at scala.collection.parallel.Task.tryLeaf$(Tasks.scala:50)
[error]         at scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:955)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:153)
[error]         at scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:149)
[error]         at scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:440)
[error]         at java.base/java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
[error]         at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
[error]         at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
[error]         at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
[error]         at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
[error]         at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:177)
eed3si9n commented 4 years ago

@zzvara Thanks for the report. Could you tell us which version of JDK you're using plz? ASM is often sensitive to bytecode instructions/flags that gets added in JDK.

zzvara commented 4 years ago

@eed3si9n Thanks for asking!

openjdk 11.0.6 2020-01-14
OpenJDK Runtime Environment AdoptOpenJDK (build 11.0.6+10)
OpenJDK 64-Bit Server VM AdoptOpenJDK (build 11.0.6+10, mixed mode)

running in WSL.

zzvara commented 4 years ago

Removing:

ShadeRule
      .zap("org.apache.spark.unused.UnusedStubClass")
      .inAll

and moving Spark dependency to provided fixed it.

Not a hard problem, but this will force us to deal with classpath conflicts in runtime in the Spark container (in this case the Kubernetes Spark image and the /jars directory there that is filled with unshaded dependencies.)

zzvara commented 4 years ago

I stumbled upon the same problem in a separate project with SBT assembly 1.5.0.

Spark is already "provided"-scope in the project.

yruslan commented 3 years ago

I had the same error for a class defined as:

@throws(classOf[IllegalArgumentException])
class Foo[T: ClassTag](bar: Seq[String]) extends Something { ... }

After the exception annotation was removed, the assembly proceeded without errors:

class Foo[T: ClassTag](bar: Seq[String]) extends Something { ... }

This annotation is not needed anyway since IllegalArgumentException is an unchecked exception.

eed3si9n commented 3 years ago

I'm guessing that ASM version needs to be updated by jarjar but there's no owner for the project anymore.

eed3si9n commented 1 year ago

Should be fixed now.