search

AppThreat / atom

Atom is a novel intermediate representation for applications and a standalone tool that is powered by chen.
https://appthreat.com
Apache License 2.0
43 stars 2 forks source link

Sporadic "key not found errors" in reaching def pass #26

Closed prabhu closed 1 year ago

prabhu commented 1 year ago

We need a solution or a workaround for this exception. So far, tried adding JAVA_OPTS to increase heap memory which didn't work. Merely running parsedeps command against a repo like scipy or requests is enough to replicate this.

atom parsedeps -l python -o /tmp/atom-deps-sIUOi7/app.atom --slice-outfile /tmp/atom-deps-sIUOi7/slices.json /home/almalinux/work/sandbox/scipy -Dlog4j.configurationFile=/tmp/atom-deps-sIUOi7/log4j2.xml
Data-flow overlay is not detected, applying now
Failure: java.util.NoSuchElementException: key not found: io.shiftleft.codepropertygraph.generated.nodes.Identifier[label=IDENTIFIER; id=2346266]
 2023-06-11 00:02:29.712 ERROR CpgPassBase: Pass io.joern.dataflowengineoss.passes.reachingdef.ReachingDefPass failed
java.util.NoSuchElementException: java.util.NoSuchElementException: key not found: io.shiftleft.codepropertygraph.generated.nodes.Identifier[label=IDENTIFIER; id=2346266]
    at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
    at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
    at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
    at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
    at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
    at java.util.concurrent.ForkJoinTask.getThrowableException(ForkJoinTask.java:562) ~[?:?]
    at java.util.concurrent.ForkJoinTask.reportException(ForkJoinTask.java:591) ~[?:?]
    at java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:689) ~[?:?]
    at java.util.stream.ReduceOps$ReduceOp.evaluateParallel(ReduceOps.java:927) ~[?:?]
    at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233) ~[?:?]
    at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:693) ~[?:?]
    at io.shiftleft.passes.NewStyleCpgPassBase.runWithBuilder(CpgPass.scala:152) ~[io.shiftleft.codepropertygraph_3-1.3.600.jar:1.3.600]
    at io.shiftleft.passes.ForkJoinParallelCpgPass.createApplySerializeAndStore(CpgPass.scala:74) ~[io.shiftleft.codepropertygraph_3-1.3.600.jar:1.3.600]
    at io.shiftleft.semanticcpg.layers.LayerCreator.runPass(LayerCreator.scala:53) ~[io.joern.semanticcpg_3-1.1.1742.jar:1.1.1742]
    at io.joern.dataflowengineoss.layers.dataflows.OssDataFlow.create$$anonfun$1(OssDataFlow.scala:31) ~[io.joern.dataflowengineoss_3-1.1.1742.jar:1.1.1742]
    at scala.runtime.function.JProcedure1.apply(JProcedure1.java:15) ~[org.scala-lang.scala3-library_3-3.3.0.jar:3.3.0]
    at scala.runtime.function.JProcedure1.apply(JProcedure1.java:10) ~[org.scala-lang.scala3-library_3-3.3.0.jar:3.3.0]
    at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:575) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:573) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at scala.collection.AbstractIterator.foreach(Iterator.scala:1300) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at io.joern.dataflowengineoss.layers.dataflows.OssDataFlow.create(OssDataFlow.scala:32) ~[io.joern.dataflowengineoss_3-1.1.1742.jar:1.1.1742]
    at io.shiftleft.semanticcpg.layers.LayerCreator.run(LayerCreator.scala:32) ~[io.joern.semanticcpg_3-1.1.1742.jar:1.1.1742]
    at io.appthreat.atom.parsedeps.package$.parseDependencies(package.scala:32) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at io.appthreat.atom.Atom$.generateSlice$$anonfun$2(Atom.scala:250) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at scala.util.Using$.resource(Using.scala:261) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at io.appthreat.atom.Atom$.generateSlice(Atom.scala:250) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at io.appthreat.atom.Atom$.run$$anonfun$1$$anonfun$1$$anonfun$1(Atom.scala:114) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at scala.util.Either.flatMap(Either.scala:352) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at io.appthreat.atom.Atom$.run$$anonfun$1$$anonfun$1(Atom.scala:115) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at scala.util.Either.flatMap(Either.scala:352) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at io.appthreat.atom.Atom$.run$$anonfun$1(Atom.scala:115) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at scala.util.Either.flatMap(Either.scala:352) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at io.appthreat.atom.Atom$.run(Atom.scala:115) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at io.appthreat.atom.Atom$.run(Atom.scala:99) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at io.appthreat.atom.Atom$.main(Atom.scala:55) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
    at io.appthreat.atom.Atom.main(Atom.scala) ~[io.appthreat.atom-1.0.0.jar:1.0.0]
Caused by: java.util.NoSuchElementException: key not found: io.shiftleft.codepropertygraph.generated.nodes.Identifier[label=IDENTIFIER; id=2346266]
    at scala.collection.immutable.BitmapIndexedMapNode.apply(HashMap.scala:635) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at scala.collection.immutable.BitmapIndexedMapNode.apply(HashMap.scala:633) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at scala.collection.immutable.HashMap.apply(HashMap.scala:132) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at io.joern.dataflowengineoss.passes.reachingdef.ReachingDefFlowGraph.pred(ReachingDefProblem.scala:72) ~[io.joern.dataflowengineoss_3-1.1.1742.jar:1.1.1742]
    at io.joern.dataflowengineoss.passes.reachingdef.ReachingDefFlowGraph.pred(ReachingDefProblem.scala:71) ~[io.joern.dataflowengineoss_3-1.1.1742.jar:1.1.1742]
    at io.joern.dataflowengineoss.passes.reachingdef.DataFlowSolver.$anonfun$1(DataFlowSolver.scala:20) ~[io.joern.dataflowengineoss_3-1.1.1742.jar:1.1.1742]
    at scala.collection.StrictOptimizedIterableOps.flatMap(StrictOptimizedIterableOps.scala:118) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at scala.collection.StrictOptimizedIterableOps.flatMap$(StrictOptimizedIterableOps.scala:105) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at scala.collection.mutable.ListBuffer.flatMap(ListBuffer.scala:39) ~[org.scala-lang.scala-library-2.13.10.jar:?]
    at io.joern.dataflowengineoss.passes.reachingdef.DataFlowSolver.calculateMopSolutionForwards(DataFlowSolver.scala:33) ~[io.joern.dataflowengineoss_3-1.1.1742.jar:1.1.1742]
    at io.joern.dataflowengineoss.passes.reachingdef.ReachingDefPass.runOnPart(ReachingDefPass.scala:31) ~[io.joern.dataflowengineoss_3-1.1.1742.jar:1.1.1742]
    at io.joern.dataflowengineoss.passes.reachingdef.ReachingDefPass.runOnPart(ReachingDefPass.scala:23) ~[io.joern.dataflowengineoss_3-1.1.1742.jar:1.1.1742]
    at io.shiftleft.passes.NewStyleCpgPassBase$$anon$2.accept(CpgPass.scala:147) ~[io.shiftleft.codepropertygraph_3-1.3.600.jar:1.3.600]
    at io.shiftleft.passes.NewStyleCpgPassBase$$anon$2.accept(CpgPass.scala:146) ~[io.shiftleft.codepropertygraph_3-1.3.600.jar:1.3.600]
    at java.util.stream.ReduceOps$4ReducingSink.accept(ReduceOps.java:220) ~[?:?]
    at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:992) ~[?:?]
    at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509) ~[?:?]
    at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499) ~[?:?]
    at java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:960) ~[?:?]
    at java.util.stream.ReduceOps$ReduceTask.doLeaf(ReduceOps.java:934) ~[?:?]
    at java.util.stream.AbstractTask.compute(AbstractTask.java:327) ~[?:?]
    at java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:754) ~[?:?]
    at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:373) ~[?:?]
    at java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1182) ~[?:?]
    at java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1655) ~[?:?]
    at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1622) ~[?:?]
    at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) ~[?:?]

cc: @mpollmeier

prabhu commented 1 year ago

Added a temporary working by adding custom passes that set queue length to 1