search

logstash-plugins / logstash-output-kafka

Kafka Output for Logstash
Apache License 2.0
75 stars 76 forks source link

Unable to override partitioner_class without creating a fat jar with all dependencies #26

Open steotia opened 9 years ago

steotia commented 9 years ago

I created a partitioner_class and replaced the partitioner_class appropriately in the conf file. The jar containing the partitioner_class was added to the CLASSPATH.

However, when I started logstash, logstash errored out with a trace I have appended at the end of this comment.

The only way I can avoid the error is if I create a fat jar (jar-with-dependencies) for the partitioner_class, even though all the dependencies are included in the plugin.

Is this expected? I was under the impression that all that was required was to create a jar containing the relevant file and include that in the CLASSPATH.

  ava/lang/ClassLoader.java:-2:in `defineClass1': java.lang.NoClassDefFoundError: kafka/producer/Partitioner
    from java/lang/ClassLoader.java:800:in `defineClass'
    from java/security/SecureClassLoader.java:142:in `defineClass'
    from java/net/URLClassLoader.java:449:in `defineClass'
    from java/net/URLClassLoader.java:71:in `access$100'
    from java/net/URLClassLoader.java:361:in `run'
    from java/net/URLClassLoader.java:355:in `run'
    from java/security/AccessController.java:-2:in `doPrivileged'
    from java/net/URLClassLoader.java:354:in `findClass'
    from java/lang/ClassLoader.java:425:in `loadClass'
    from sun/misc/Launcher.java:308:in `loadClass'
    from java/lang/ClassLoader.java:412:in `loadClass'
    from java/lang/ClassLoader.java:358:in `loadClass'
    from java/lang/Class.java:-2:in `forName0'
    from java/lang/Class.java:190:in `forName'
    from Utils.scala:435:in `createObject'
    from kafka/utils/Utils.scala:435:in `createObject'
    from Producer.scala:61:in `<init>'
    from kafka/producer/Producer.scala:61:in `<init>'
    from Producer.scala:26:in `<init>'
    from kafka/javaapi/producer/Producer.scala:26:in `<init>'
    from java/lang/reflect/Constructor.java:526:in `newInstance'
    from <SOME_DIR>/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/jruby-kafka-1.4.0-java/lib/jruby-kafka/producer.rb:64:in `connect'
    from <SOME_DIR>/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-output-kafka-1.0.0/lib/logstash/outputs/kafka.rb:142:in `register'
    from org/jruby/RubyArray.java:1613:in `each'
    from <SOME_DIR>/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.4-java/lib/logstash/pipeline.rb:164:in `start_outputs'
    from <SOME_DIR>/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.4-java/lib/logstash/pipeline.rb:83:in `run'
    from <SOME_DIR>/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.4-java/lib/logstash/agent.rb:155:in `execute'
    from <SOME_DIR>/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.4-java/lib/logstash/runner.rb:90:in `run'
    from org/jruby/RubyProc.java:271:in `call'
    from <SOME_DIR>/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/logstash-core-1.5.4-java/lib/logstash/runner.rb:95:in `run'
    from org/jruby/RubyProc.java:271:in `call'
    from <SOME_DIR>/logstash-1.5.4/vendor/bundle/jruby/1.9/gems/stud-0.0.21/lib/stud/task.rb:12:in `initialize'
    from java/lang/Thread.java:745:in `run'
  Caused by:
  URLClassLoader.java:366:in `run': java.lang.ClassNotFoundException: kafka.producer.Partitioner
    from URLClassLoader.java:355:in `run'
    from AccessController.java:-2:in `doPrivileged'
    from URLClassLoader.java:354:in `findClass'
    from ClassLoader.java:425:in `loadClass'
    from Launcher.java:308:in `loadClass'
    from ClassLoader.java:358:in `loadClass'
    from ClassLoader.java:-2:in `defineClass1'
    from ClassLoader.java:800:in `defineClass'
    from SecureClassLoader.java:142:in `defineClass'
    from URLClassLoader.java:449:in `defineClass'
    from URLClassLoader.java:71:in `access$100'
    from URLClassLoader.java:361:in `run'
    from URLClassLoader.java:355:in `run'
    from AccessController.java:-2:in `doPrivileged'
    from URLClassLoader.java:354:in `findClass'
    from ClassLoader.java:425:in `loadClass'
    from Launcher.java:308:in `loadClass'
    from ClassLoader.java:412:in `loadClass'
    from ClassLoader.java:358:in `loadClass'
    from Class.java:-2:in `forName0'
    from Class.java:190:in `forName'
    from Utils.scala:435:in `createObject'
    from Producer.scala:61:in `<init>'
    from Producer.scala:26:in `<init>'
    from NativeConstructorAccessorImpl.java:-2:in `newInstance0'
    from NativeConstructorAccessorImpl.java:57:in `newInstance'
    from DelegatingConstructorAccessorImpl.java:45:in `newInstance'
    from Constructor.java:526:in `newInstance'
    from JavaConstructor.java:268:in `newInstanceDirect'
    from ConstructorInvoker.java:83:in `call'
    from ConstructorInvoker.java:174:in `call'
    from CachingCallSite.java:336:in `cacheAndCall'
    from CachingCallSite.java:179:in `callBlock'
    from CachingCallSite.java:183:in `call'
    from ConcreteJavaProxy.java:56:in `call'
    from CachingCallSite.java:336:in `cacheAndCall'
    from CachingCallSite.java:179:in `callBlock'
    from CachingCallSite.java:183:in `call'
    from RubyClass.java:856:in `newInstance'
    from RubyClass$INVOKER$i$newInstance.gen:-1:in `call'
    from JavaMethod.java:301:in `call'
    from ConcreteJavaProxy.java:155:in `call'
    from CachingCallSite.java:326:in `cacheAndCall'
    from CachingCallSite.java:170:in `call'
    from CallOneArgNode.java:57:in `interpret'
    from InstAsgnNode.java:95:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from BlockNode.java:71:in `interpret'
    from ASTInterpreter.java:74:in `INTERPRET_METHOD'
    from InterpretedMethod.java:139:in `call'
    from DefaultMethod.java:187:in `call'
    from CachingCallSite.java:306:in `cacheAndCall'
    from CachingCallSite.java:136:in `call'
    from CallNoArgNode.java:60:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from BlockNode.java:71:in `interpret'
    from ASTInterpreter.java:74:in `INTERPRET_METHOD'
    from InterpretedMethod.java:112:in `call'
    from DefaultMethod.java:169:in `call'
    from CachingCallSite.java:286:in `cacheAndCall'
    from CachingCallSite.java:81:in `callBlock'
    from CachingCallSite.java:85:in `call'
    from RubySymbol.java:461:in `yieldInner'
    from RubySymbol.java:481:in `yield'
    from Block.java:142:in `yield'
    from RubyArray.java:1606:in `eachCommon'
    from RubyArray.java:1613:in `each'
    from RubyArray$INVOKER$i$0$0$each.gen:-1:in `call'
    from CachingCallSite.java:316:in `cacheAndCall'
    from CachingCallSite.java:145:in `callBlock'
    from CachingCallSite.java:149:in `call'
    from CallNoArgBlockPassNode.java:53:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from BlockNode.java:71:in `interpret'
    from ASTInterpreter.java:74:in `INTERPRET_METHOD'
    from InterpretedMethod.java:139:in `call'
    from DefaultMethod.java:187:in `call'
    from CachingCallSite.java:306:in `cacheAndCall'
    from CachingCallSite.java:136:in `call'
    from VCallNode.java:88:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from BlockNode.java:71:in `interpret'
    from ASTInterpreter.java:74:in `INTERPRET_METHOD'
    from InterpretedMethod.java:139:in `call'
    from DefaultMethod.java:187:in `call'
    from CachingCallSite.java:306:in `cacheAndCall'
    from CachingCallSite.java:136:in `call'
    from CallNoArgNode.java:60:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from BlockNode.java:71:in `interpret'
    from RescueNode.java:221:in `executeBody'
    from RescueNode.java:116:in `interpret'
    from EnsureNode.java:96:in `interpret'
    from ASTInterpreter.java:74:in `INTERPRET_METHOD'
    from InterpretedMethod.java:139:in `call'
    from DefaultMethod.java:187:in `call'
    from CachingCallSite.java:306:in `cacheAndCall'
    from CachingCallSite.java:136:in `call'
    from CallNoArgNode.java:60:in `interpret'
    from ReturnNode.java:92:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from BlockNode.java:71:in `interpret'
    from ASTInterpreter.java:112:in `INTERPRET_BLOCK'
    from Interpreted19Block.java:206:in `evalBlockBody'
    from Interpreted19Block.java:194:in `yield'
    from Interpreted19Block.java:125:in `call'
    from Block.java:101:in `call'
    from RubyProc.java:290:in `call'
    from RubyProc.java:271:in `call19'
    from RubyProc$INVOKER$i$0$0$call19.gen:-1:in `call'
    from DynamicMethod.java:202:in `call'
    from DynamicMethod.java:198:in `call'
    from CachingCallSite.java:306:in `cacheAndCall'
    from CachingCallSite.java:136:in `call'
    from CallNoArgNode.java:60:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from ASTInterpreter.java:112:in `INTERPRET_BLOCK'
    from Interpreted19Block.java:206:in `evalBlockBody'
    from Interpreted19Block.java:194:in `yield'
    from Interpreted19Block.java:125:in `call'
    from Block.java:101:in `call'
    from RubyProc.java:290:in `call'
    from RubyProc.java:271:in `call19'
    from RubyProc$INVOKER$i$0$0$call19.gen:-1:in `call'
    from DynamicMethod.java:202:in `call'
    from DynamicMethod.java:198:in `call'
    from CachingCallSite.java:306:in `cacheAndCall'
    from CachingCallSite.java:136:in `call'
    from CallSpecialArgNode.java:65:in `interpret'
    from DAsgnNode.java:110:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from BlockNode.java:71:in `interpret'
    from RescueNode.java:221:in `executeBody'
    from RescueNode.java:116:in `interpret'
    from BeginNode.java:83:in `interpret'
    from NewlineNode.java:105:in `interpret'
    from ASTInterpreter.java:112:in `INTERPRET_BLOCK'
    from Interpreted19Block.java:206:in `evalBlockBody'
    from Interpreted19Block.java:194:in `yield'
    from Interpreted19Block.java:125:in `call'
    from Block.java:101:in `call'
    from RubyProc.java:290:in `call'
    from RubyProc.java:228:in `call'
    from RubyRunnable.java:99:in `run'
    from Thread.java:745:in `run'
talevy commented 9 years ago

was your class name kafka/producer/Partitioner or kafka.producer.Partitioner

mind sharing your config?