Unidata / awips2

Weather forecasting display and analysis package developed by NWS/Raytheon, released as open source software by Unidata.
http://unidata.github.io/awips2/
Other
171 stars 67 forks source link

[Ingest.dmw-1] IndexSrv: Error occurred during persist #275

Open mjames-upc opened 5 years ago

mjames-upc commented 5 years ago
ERROR 2019-01-09 17:57:13,580 6509 [Ingest.dmw-1] IndexSrv: Error occurred during persist
org.hibernate.exception.GenericJDBCException: could not execute batch
    at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:54) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:124) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.engine.jdbc.batch.internal.BatchingBatch.performExecution(BatchingBatch.java:122) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.engine.jdbc.batch.internal.BatchingBatch.addToBatch(BatchingBatch.java:87) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.persister.entity.AbstractEntityPersister.insert(AbstractEntityPersister.java:3079) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.persister.entity.AbstractEntityPersister.insert(AbstractEntityPersister.java:3521) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.action.internal.EntityInsertAction.execute(EntityInsertAction.java:88) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.engine.spi.ActionQueue.execute(ActionQueue.java:395) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.engine.spi.ActionQueue.executeActions(ActionQueue.java:387) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.engine.spi.ActionQueue.executeActions(ActionQueue.java:303) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.event.internal.AbstractFlushingEventListener.performExecutions(AbstractFlushingEventListener.java:349) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.event.internal.DefaultFlushEventListener.onFlush(DefaultFlushEventListener.java:56) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.internal.SessionImpl.flush(SessionImpl.java:1159) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.internal.SessionImpl.managedFlush(SessionImpl.java:404) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.engine.transaction.internal.jdbc.JdbcTransaction.beforeTransactionCommit(JdbcTransaction.java:101) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at org.hibernate.engine.transaction.spi.AbstractTransactionImpl.commit(AbstractTransactionImpl.java:175) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    at com.raytheon.uf.edex.database.plugin.PluginDao.persistToDatabase(PluginDao.java:260) ~[com.raytheon.uf.edex.database.jar:na]
    at com.raytheon.uf.edex.ingest.IndexSrv.index(IndexSrv.java:116) ~[com.raytheon.uf.edex.ingest.jar:na]
    at sun.reflect.GeneratedMethodAccessor186.invoke(Unknown Source) ~[na:na]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_101]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_101]
    at org.apache.camel.component.bean.MethodInfo.invoke(MethodInfo.java:458) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.component.bean.MethodInfo$1.doProceed(MethodInfo.java:289) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.component.bean.MethodInfo$1.proceed(MethodInfo.java:262) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.component.bean.BeanProcessor.process(BeanProcessor.java:178) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:109) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.component.bean.BeanProcessor.process(BeanProcessor.java:69) [camel-core-2.18.3.jar:2.18.3]
    at com.raytheon.uf.edex.esb.camel.MessageProducer$2.process(MessageProducer.java:439) [com.raytheon.uf.edex.esb.camel.jar:na]
    at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:110) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:542) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:197) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.Pipeline.process(Pipeline.java:120) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.Pipeline.process(Pipeline.java:83) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:197) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.component.directvm.DirectVmProcessor.process(DirectVmProcessor.java:57) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.component.directvm.DirectVmProducer.process(DirectVmProducer.java:68) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:145) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:109) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:116) [camel-core-2.18.3.jar:2.18.3]
    at com.raytheon.uf.edex.esb.camel.MessageProducer$2.process(MessageProducer.java:439) [com.raytheon.uf.edex.esb.camel.jar:na]
    at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:110) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:197) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.Pipeline.process(Pipeline.java:120) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.Pipeline.process(Pipeline.java:83) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:109) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.Pipeline.process(Pipeline.java:63) [camel-core-2.18.3.jar:2.18.3]
    at com.raytheon.uf.edex.esb.camel.MessageProducer$2.process(MessageProducer.java:439) [com.raytheon.uf.edex.esb.camel.jar:na]
    at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:110) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:197) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.TryProcessor.process(TryProcessor.java:113) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.TryProcessor.process(TryProcessor.java:84) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.util.AsyncProcessorHelper.process(AsyncProcessorHelper.java:109) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.TryProcessor.process(TryProcessor.java:68) [camel-core-2.18.3.jar:2.18.3]
    at com.raytheon.uf.edex.esb.camel.MessageProducer$2.process(MessageProducer.java:439) [com.raytheon.uf.edex.esb.camel.jar:na]
    at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:110) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:197) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.Pipeline.process(Pipeline.java:120) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.Pipeline.process(Pipeline.java:83) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:197) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) [camel-core-2.18.3.jar:2.18.3]
    at org.apache.camel.component.jms.EndpointMessageListener.onMessage(EndpointMessageListener.java:112) [camel-jms-2.18.3.jar:2.18.3]
    at org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:721) [spring-jms-4.2.9.RELEASE.jar:4.2.9.RELEASE]
    at org.springframework.jms.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:681) [spring-jms-4.2.9.RELEASE.jar:4.2.9.RELEASE]
    at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:651) [spring-jms-4.2.9.RELEASE.jar:4.2.9.RELEASE]
    at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:313) [spring-jms-4.2.9.RELEASE.jar:4.2.9.RELEASE]
    at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:251) [spring-jms-4.2.9.RELEASE.jar:4.2.9.RELEASE]
    at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1164) [spring-jms-4.2.9.RELEASE.jar:4.2.9.RELEASE]
    at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1156) [spring-jms-4.2.9.RELEASE.jar:4.2.9.RELEASE]
    at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1053) [spring-jms-4.2.9.RELEASE.jar:4.2.9.RELEASE]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_101]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_101]
    at java.lang.Thread.run(Thread.java:745) [na:1.8.0_101]
Caused by: java.sql.BatchUpdateException: Batch entry 22 insert into dmw (forecastTime, refTime, utilityFlags, rangeEnd, rangeStart, insertTime, channel, filter, elevation, latitude, location, locationDefined, longitude, stationId, orbitalSlot, idx, scene, windDir, windSpd, id) values (0, '2019-01-09 17:47:18.000000 +00:00:00', '[]', '2019-01-09 17:47:18.000000 +00:00:00', '2019-01-09 17:47:18.000000 +00:00:00', '2019-01-09 17:57:13.510000 +00:00:00', 8, 289.69946, NULL, 29.014744, 'POINT(-67.08082580566406 29.01474380493164)', '0', -67.080826, '-67.081:29.015', 'GOES-East', 0, 'CONUS', 259.46835, 28.448364, 727555) was aborted.  Call getNextException to see the cause.
    at org.postgresql.jdbc2.AbstractJdbc2Statement$BatchResultHandler.handleError(AbstractJdbc2Statement.java:2744) ~[postgresql-9.3-1104.jdbc4.jar:na]
    at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:1928) ~[postgresql-9.3-1104.jdbc4.jar:na]
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:405) ~[postgresql-9.3-1104.jdbc4.jar:na]
    at org.postgresql.jdbc2.AbstractJdbc2Statement.executeBatch(AbstractJdbc2Statement.java:2893) ~[postgresql-9.3-1104.jdbc4.jar:na]
    at com.mchange.v2.c3p0.impl.NewProxyPreparedStatement.executeBatch(NewProxyPreparedStatement.java:1135) ~[c3p0-0.9.2.1.jar:0.9.2.1]
    at org.hibernate.engine.jdbc.batch.internal.BatchingBatch.performExecution(BatchingBatch.java:113) ~[hibernate-core-4.2.15.Final.jar:4.2.15.Final]
    ... 69 common frames omitted
mjames-upc commented 5 years ago

from pg_log:

2019-01-09 17:57:46 UTC 10344633 metadata : 127.0.0.1 : ERROR:  right sibling's left-link doesn't match: block 456 links to 227 instead of expected 863 in index "dmw_stationindex"
2019-01-09 17:57:46 UTC 10344633 metadata : 127.0.0.1 : STATEMENT:  insert into dmw (forecastTime, refTime, utilityFlags, rangeEnd, rangeStart, insertTime, channel, filter, elevation, latitude, location, locationDefined, longitude, stationId, orbitalSlot, idx, scene, windDir, windSpd, id) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)
2019-01-09 17:57:46 UTC 10344633 metadata : 127.0.0.1 : ERROR:  current transaction is aborted, commands ignored until end of transaction block
2019-01-09 17:57:46 UTC 10344633 metadata : 127.0.0.1 : STATEMENT:  SELECT NULL AS TABLE_CAT, n.nspname AS TABLE_SCHEM, c.relname AS TABLE_NAME,  CASE n.nspname ~ '^pg_' OR n.nspname = 'information_schema'  WHEN true THEN CASE  WHEN n.nspname = 'pg_catalog' OR n.nspname = 'information_schema' THEN CASE c.relkind   WHEN 'r' THEN 'SYSTEM TABLE'   WHEN 'v' THEN 'SYSTEM VIEW'   WHEN 'i' THEN 'SYSTEM INDEX'   ELSE NULL   END  WHEN n.nspname = 'pg_toast' THEN CASE c.relkind   WHEN 'r' THEN 'SYSTEM TOAST TABLE'   WHEN 'i' THEN 'SYSTEM TOAST INDEX'   ELSE NULL   END  ELSE CASE c.relkind   WHEN 'r' THEN 'TEMPORARY TABLE'   WHEN 'i' THEN 'TEMPORARY INDEX'   WHEN 'S' THEN 'TEMPORARY SEQUENCE'   WHEN 'v' THEN 'TEMPORARY VIEW'   ELSE NULL   END  END  WHEN false THEN CASE c.relkind  WHEN 'r' THEN 'TABLE'  WHEN 'i' THEN 'INDEX'  WHEN 'S' THEN 'SEQUENCE'  WHEN 'v' THEN 'VIEW'  WHEN 'c' THEN 'TYPE'  WHEN 'f' THEN 'FOREIGN TABLE'  WHEN 'm' THEN 'MATERIALIZED VIEW'  ELSE NULL  END  ELSE NULL  END  AS TABLE_TYPE, d.description AS REMARKS  FROM pg_catalog.pg_namespace n, pg_catalog.pg_class c  LEFT JOIN pg_catalog.pg_description d ON (c.oid = d.objoid AND d.objsubid = 0)  LEFT JOIN pg_catalog.pg_class dc ON (d.classoid=dc.oid AND dc.relname='pg_class')  LEFT JOIN pg_catalog.pg_namespace dn ON (dn.oid=dc.relnamespace AND dn.nspname='pg_catalog')  WHERE c.relnamespace = n.oid  AND c.relname LIKE 'PROBABLYNOT' AND (false  OR ( c.relkind = 'r' AND n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' ) )  ORDER BY TABLE_TYPE,TABLE_SCHEM,TABLE_NAME