search

embulk / embulk-output-elasticsearch

Apache License 2.0
32 stars 26 forks source link

Update embulk-base-restclient to fix ArrayIndexOutOfBoundsException #37

Closed sakama closed 7 years ago

sakama commented 7 years ago

Problem

I sometimes got ArrayIndexOutOfBoundsException with this plugin. This problem never happens when input source is CSV or JSON. Only happens when MessagePack input.

I sent a PR https://github.com/embulk/embulk-base-restclient/pull/87 to embulk-base-rest-client to fix this problem and v0.5.2 was released.

Changes

Stacktrace

org.embulk.exec.PartialExecutionException: java.lang.ArrayIndexOutOfBoundsException: 123456789
  at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(BulkLoader.java:373) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.exec.BulkLoader.doRun(BulkLoader.java:591) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.exec.BulkLoader.access$000(BulkLoader.java:33) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.exec.BulkLoader$1.run(BulkLoader.java:389) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.exec.BulkLoader$1.run(BulkLoader.java:385) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.spi.Exec.doWith(Exec.java:25) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.exec.BulkLoader.run(BulkLoader.java:385) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.EmbulkEmbed.run(EmbulkEmbed.java:180) ~[embulk-core-0.8.18.jar:na]
  at com.treasuredata.worker.calls.EmbulkResult.run(EmbulkResult.java:69) ~[td-worker-calls-0.1.0-SNAPSHOT.jar:0.1.0-SNAPSHOT]
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.7.0_80]
  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) ~[na:1.7.0_80]
  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.7.0_80]
  at java.lang.reflect.Method.invoke(Method.java:606) ~[na:1.7.0_80]
  at com.treasuredata.java_call.JavaCall.processRequest(JavaCall.java:147) [td-worker-calls-0.1.0-SNAPSHOT.jar:0.1.0-SNAPSHOT]
  at com.treasuredata.java_call.JavaCall.run(JavaCall.java:111) [td-worker-calls-0.1.0-SNAPSHOT.jar:0.1.0-SNAPSHOT]
  at com.treasuredata.worker.JavaCallMain.main(JavaCallMain.java:21) [td-worker-calls-0.1.0-SNAPSHOT.jar:0.1.0-SNAPSHOT]
Caused by: java.lang.ArrayIndexOutOfBoundsException: 123456789
  at java.util.Arrays$ArrayList.get(Arrays.java:2866) ~[na:1.7.0_80]
  at org.embulk.spi.Page.getStringReference(Page.java:53) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.spi.PageReader.getString(PageReader.java:111) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.spi.PageReader.getString(PageReader.java:105) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.base.restclient.record.SinglePageRecordReader.getString(SinglePageRecordReader.java:67) ~[na:na]
  at org.embulk.base.restclient.jackson.scope.JacksonAllInObjectScope$1.stringColumn(JacksonAllInObjectScope.java:62) ~[na:na]
  at org.embulk.spi.Column.visit(Column.java:58) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.spi.Schema.visitColumns(Schema.java:81) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.base.restclient.jackson.scope.JacksonAllInObjectScope.scopeObject(JacksonAllInObjectScope.java:36) ~[na:na]
  at org.embulk.base.restclient.jackson.scope.JacksonObjectScopeBase.scopeEmbulkValues(JacksonObjectScopeBase.java:17) ~[na:na]
  at org.embulk.base.restclient.jackson.scope.JacksonObjectScopeBase.scopeEmbulkValues(JacksonObjectScopeBase.java:9) ~[na:na]
  at org.embulk.base.restclient.record.ValueExporter.exportValueToBuildRecord(ValueExporter.java:14) ~[na:na]
  at org.embulk.base.restclient.record.RecordExporter.exportRecord(RecordExporter.java:18) ~[na:na]
  at org.embulk.base.restclient.RestClientPageOutput.add(RestClientPageOutput.java:43) ~[na:na]
  at org.embulk.spi.PageBuilder.doFlush(PageBuilder.java:244) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.spi.PageBuilder.flush(PageBuilder.java:250) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.spi.PageBuilder.addRecord(PageBuilder.java:227) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.parser.msgpack.MsgpackParserPlugin.run(MsgpackParserPlugin.java:280) ~[na:na]
  at org.embulk.spi.FileInputRunner.run(FileInputRunner.java:153) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.spi.util.Executors.process(Executors.java:67) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.spi.util.Executors.process(Executors.java:42) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.exec.LocalExecutorPlugin$DirectExecutor$1.call(LocalExecutorPlugin.java:184) ~[embulk-core-0.8.18.jar:na]
  at org.embulk.exec.LocalExecutorPlugin$DirectExecutor$1.call(LocalExecutorPlugin.java:180) ~[embulk-core-0.8.18.jar:na]
  at java.util.concurrent.FutureTask.run(FutureTask.java:262) ~[na:1.7.0_80]
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[na:1.7.0_80]
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) ~[na:1.7.0_80]
  at java.lang.Thread.run(Thread.java:745) ~[na:1.7.0_80]
java.lang.ArrayIndexOutOfBoundsException: 123456789
org.embulk.exec.PartialExecutionException: java.lang.ArrayIndexOutOfBoundsException: 123456789
  at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(BulkLoader.java:373)
  at org.embulk.exec.BulkLoader.doRun(BulkLoader.java:591)
  at org.embulk.exec.BulkLoader.access$000(BulkLoader.java:33)
  at org.embulk.exec.BulkLoader$1.run(BulkLoader.java:389)
  at org.embulk.exec.BulkLoader$1.run(BulkLoader.java:385)
  at org.embulk.spi.Exec.doWith(Exec.java:25)
  at org.embulk.exec.BulkLoader.run(BulkLoader.java:385)
  at org.embulk.EmbulkEmbed.run(EmbulkEmbed.java:180)
  at com.treasuredata.worker.calls.EmbulkResult.run(EmbulkResult.java:69)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:606)
  at com.treasuredata.java_call.JavaCall.processRequest(JavaCall.java:147)
  at com.treasuredata.java_call.JavaCall.run(JavaCall.java:111)
  at com.treasuredata.worker.JavaCallMain.main(JavaCallMain.java:21)
Caused by: java.lang.ArrayIndexOutOfBoundsException: 123456789
  at java.util.Arrays$ArrayList.get(Arrays.java:2866)
  at org.embulk.spi.Page.getStringReference(Page.java:53)
  at org.embulk.spi.PageReader.getString(PageReader.java:111)
  at org.embulk.spi.PageReader.getString(PageReader.java:105)
  at org.embulk.base.restclient.record.SinglePageRecordReader.getString(SinglePageRecordReader.java:67)
  at org.embulk.base.restclient.jackson.scope.JacksonAllInObjectScope$1.stringColumn(JacksonAllInObjectScope.java:62)
  at org.embulk.spi.Column.visit(Column.java:58)
  at org.embulk.spi.Schema.visitColumns(Schema.java:81)
  at org.embulk.base.restclient.jackson.scope.JacksonAllInObjectScope.scopeObject(JacksonAllInObjectScope.java:36)
  at org.embulk.base.restclient.jackson.scope.JacksonObjectScopeBase.scopeEmbulkValues(JacksonObjectScopeBase.java:17)
  at org.embulk.base.restclient.jackson.scope.JacksonObjectScopeBase.scopeEmbulkValues(JacksonObjectScopeBase.java:9)
  at org.embulk.base.restclient.record.ValueExporter.exportValueToBuildRecord(ValueExporter.java:14)
  at org.embulk.base.restclient.record.RecordExporter.exportRecord(RecordExporter.java:18)
  at org.embulk.base.restclient.RestClientPageOutput.add(RestClientPageOutput.java:43)
  at org.embulk.spi.PageBuilder.doFlush(PageBuilder.java:244)
  at org.embulk.spi.PageBuilder.flush(PageBuilder.java:250)
  at org.embulk.spi.PageBuilder.addRecord(PageBuilder.java:227)
  at org.embulk.parser.msgpack.MsgpackParserPlugin.run(MsgpackParserPlugin.java:280)
  at org.embulk.spi.FileInputRunner.run(FileInputRunner.java:153)
  at org.embulk.spi.util.Executors.process(Executors.java:67)
  at org.embulk.spi.util.Executors.process(Executors.java:42)
  at org.embulk.exec.LocalExecutorPlugin$DirectExecutor$1.call(LocalExecutorPlugin.java:184)
  at org.embulk.exec.LocalExecutorPlugin$DirectExecutor$1.call(LocalExecutorPlugin.java:180)
  at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  at java.lang.Thread.run(Thread.java:745)
sakama commented 7 years ago

@kietdo360 Could you take a look?

kietdo360 commented 7 years ago

LGTM excepts the change description should have latest version of base-restclient as 0.5.2

sakama commented 7 years ago

Thanks. Fixed and merged.