logstash-plugins / logstash-filter-jdbc_streaming

A Logstash filter that can enrich events with data from a database
Apache License 2.0
12 stars 23 forks source link

JDBC streaming filter seems not working properly #7

Closed barnendu closed 6 years ago

barnendu commented 7 years ago

This is my .conf file configuration

input{
  jdbc {
    jdbc_connection_string => "jdbc:sqlserver://testserver;databaseName=TEST_DEV;user=me;password=secret;"
    jdbc_user => "me"
    jdbc_password => "secret"
    jdbc_driver_library => "../jre8/sqljdbc42.jar"
    jdbc_driver_class => "com.microsoft.sqlserver.jdbc.SQLServerDriver"
    # Shedule is once per hour or customise in cron type syntax
    schedule => "* * * * *"
    statement => "SELECT * from PROPERTY_MASTER_ORS.C_B_LEASE_COMPARABLES"
    # Uncomment and experiement for large tables
    #jdbc_paging_enabled => true
    #jdbc_page_size => 15000
    #jdbc_fetch_size => 10000
  }
}
filter {
  jdbc_streaming {
    jdbc_driver_library => "../jre8/sqljdbc42.jar"
    jdbc_driver_class => "com.microsoft.sqlserver.jdbc.SQLServerDriver"
    jdbc_connection_string => "jdbc:sqlserver://testserver;databaseName=TEST_DEV;user=me;password=secret;"
    jdbc_user => "me"
    jdbc_password => "secret"
    statement => "SELECT * from PROPERTY_MASTER_ORS.C_B_LEASE_COMP_FLR where lease_comaparable_id = :id"
    parameters => { "id" =>"rowid_object"}
    target => "lease_comp_floor"
  }
}

output {
  elasticsearch {
    hosts => "http://estestcloud.cloudapp.net:9200"
    index => "rpdi-es-1.0"
    document_type => "lease-comp"
    document_id => "%{rowid_object}"
  }
  stdout {
    codec => rubydebug
  }
}

So it is fetching few records, after that i am getting the below error.

[2017-07-13T18:33:47,081][INFO ]logstash.inputs.jdbc SELECT * from PROPERTY_MASTER_ORS.C_B_LEASE_COMPARABLES
[2017-07-13T18:33:47,867][ERROR][logstash.pipeline ] t.ruby_set_field(JrubyEventExtLibrary.java:128)", "org.logstash.ext.JrubyEventExtLibrary$RubyEvent$INVOKER$i$2$0$ruby_set_field.call(JrubyEventExtLibrary$RubyEvent$INVOKER$i$2$0$ruby_set_field.gen)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:202)", "org.jruby.ast.CallTwoArgNode.interpret(CallTwoArgNode.java:59)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.BlockNode.interpret(BlockNode.java:71)", "org.jruby.evaluator.ASTInterpreter.INTERPRET_METHOD(ASTInterpreter.java:74)", "org.jruby.internal.runtime.methods.InterpretedMethod.call(InterpretedMethod.java:225)", "org.jruby.internal.runtime.methods.DefaultMethod.call(DefaultMethod.java:219)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:202)", "org.jruby.ast.FCallTwoArgNode.interpret(FCallTwoArgNode.java:38)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.IfNode.interpret(IfNode.java:118)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.BlockNode.interpret(BlockNode.java:71)", "org.jruby.evaluator.ASTInterpreter.INTERPRET_METHOD(ASTInterpreter.java:74)", "org.jruby.internal.runtime.methods.InterpretedMethod.call(InterpretedMethod.java:204)", "org.jruby.internal.runtime.methods.DefaultMethod.call(DefaultMethod.java:211)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:177)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:183)", "org.jruby.ast.FCallOneArgBlockPassNode.interpret(FCallOneArgBlockPassNode.java:32)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.BlockNode.interpret(BlockNode.java:71)", "org.jruby.evaluator.ASTInterpreter.INTERPRET_METHOD(ASTInterpreter.java:74)", "org.jruby.internal.runtime.methods.InterpretedMethod.call(InterpretedMethod.java:204)", "org.jruby.internal.runtime.methods.DefaultMethod.call(DefaultMethod.java:211)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:177)", "org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:188)", "org.jruby.ast.FCallOneArgBlockNode.interpret(FCallOneArgBlockNode.java:34)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.BlockNode.interpret(BlockNode.java:71)", "org.jruby.ast.IfNode.interpret(IfNode.java:118)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.evaluator.ASTInterpreter.INTERPRET_BLOCK(ASTInterpreter.java:112)", "org.jruby.runtime.Interpreted19Block.evalBlockBody(Interpreted19Block.java:206)", "org.jruby.runtime.Interpreted19Block.yield(Interpreted19Block.java:157)", "org.jruby.runtime.Block.yield(Block.java:142)", "org.jruby.RubyArray.eachCommon(RubyArray.java:1606)", "org.jruby.RubyArray.each(RubyArray.java:1613)", "org.jruby.RubyArray$INVOKER$i$0$0$each.call(RubyArray$INVOKER$i$0$0$each.gen)", "org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:143)", "org.jruby.runtime.callsite.CachingCallSite.callIter(CachingCallSite.java:154)", "org.jruby.ast.CallNoArgBlockNode.interpret(CallNoArgBlockNode.java:64)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.BlockNode.interpret(BlockNode.java:71)", "org.jruby.evaluator.ASTInterpreter.INTERPRET_METHOD(ASTInterpreter.java:74)", "org.jruby.internal.runtime.methods.InterpretedMethod.call(InterpretedMethod.java:182)", "org.jruby.internal.runtime.methods.DefaultMethod.call(DefaultMethod.java:203)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:168)", "org.jruby.ast.CallOneArgNode.interpret(CallOneArgNode.java:57)", "org.jruby.ast.LocalAsgnNode.interpret(LocalAsgnNode.java:123)", "org.jruby.ast.NewlineNode.interpret(NewlineNode.java:105)", "org.jruby.ast.BlockNode.interpret(BlockNode.java:71)"}]

Any suggestion please.

guyboertje commented 6 years ago

I asked you for more info in this comment on the discussion forum but you did not reply. https://discuss.elastic.co/t/jdbc-streaming-filter-plugin-seems-not-working-properly/93022/2?u=guyboertje Closing