CsvAggregateCmd deal with a large amount of data, the memory error.
2017-03-30 05:20:18,912 ERROR [main] engine.EngineUtil (EngineUtil.java:281) - 2017-03-30T10:20:18.907,smsc,0001865-170104195305044-oozie-dbad-W,etl.cmd.CsvAggregateCmd,"java.lang.OutOfMemoryError: Java heap space
at java.io.BufferedReader.(BufferedReader.java:105)
at java.io.BufferedReader.(BufferedReader.java:116)
at org.apache.commons.csv.ExtendedBufferedReader.(ExtendedBufferedReader.java:55)
at org.apache.commons.csv.CSVParser.(CSVParser.java:283)
at org.apache.commons.csv.CSVParser.(CSVParser.java:252)
at org.apache.commons.csv.CSVParser.parse(CSVParser.java:178)
at etl.cmd.CsvAggregateCmd.reduceByKey(CsvAggregateCmd.java:694)
at etl.engine.EngineUtil.processReduceCmd(EngineUtil.java:271)
at etl.engine.InvokeReducer.reduce(InvokeReducer.java:54)
at etl.engine.InvokeReducer.reduce(InvokeReducer.java:14)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
CsvAggregateCmd deal with a large amount of data, the memory error.
2017-03-30 05:20:18,912 ERROR [main] engine.EngineUtil (EngineUtil.java:281) - 2017-03-30T10:20:18.907,smsc,0001865-170104195305044-oozie-dbad-W,etl.cmd.CsvAggregateCmd,"java.lang.OutOfMemoryError: Java heap space at java.io.BufferedReader.(BufferedReader.java:105)
at java.io.BufferedReader.(BufferedReader.java:116)
at org.apache.commons.csv.ExtendedBufferedReader.(ExtendedBufferedReader.java:55)
at org.apache.commons.csv.CSVParser.(CSVParser.java:283)
at org.apache.commons.csv.CSVParser.(CSVParser.java:252)
at org.apache.commons.csv.CSVParser.parse(CSVParser.java:178)
at etl.cmd.CsvAggregateCmd.reduceByKey(CsvAggregateCmd.java:694)
at etl.engine.EngineUtil.processReduceCmd(EngineUtil.java:271)
at etl.engine.InvokeReducer.reduce(InvokeReducer.java:54)
at etl.engine.InvokeReducer.reduce(InvokeReducer.java:14)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)