Open jimduff-usds opened 3 years ago
Stack: org.jooq.exception.DataAccessException: SQL [update "public"."task" set "next_action" = ?::"public"."task_action", "next_action_at" = cast(? as timestamp(6) with time zone), "retry_token" = cast(? as json), "sent_at" = cast(? as timestamp(6) with time zone) where "public"."task"."report_id" = cast(? as uuid)]; An I/O error occurred while sending to the backend.
at org.jooq_3.15.1.POSTGRES.debug(Unknown Source)
at org.jooq.impl.Tools.translate(Tools.java:2979)
at org.jooq.impl.DefaultExecuteContext.sqlException(DefaultExecuteContext.java:643)
at org.jooq.impl.AbstractQuery.execute(AbstractQuery.java:349)
at org.jooq.impl.AbstractDelegatingQuery.execute(AbstractDelegatingQuery.java:111)
at gov.cdc.prime.router.azure.DatabaseAccess.updateTask(DatabaseAccess.kt:166)
at gov.cdc.prime.router.azure.WorkflowEngine.updateHeader(WorkflowEngine.kt:503)
at gov.cdc.prime.router.azure.WorkflowEngine.access$updateHeader(WorkflowEngine.kt:42)
at gov.cdc.prime.router.azure.WorkflowEngine$handleReportEvent$1.invoke(WorkflowEngine.kt:165)
at gov.cdc.prime.router.azure.WorkflowEngine$handleReportEvent$1.invoke(WorkflowEngine.kt:140)
at gov.cdc.prime.router.azure.DatabaseAccess.transact$lambda-0(DatabaseAccess.kt:72)
at org.jooq.impl.DefaultDSLContext.lambda$transaction$5(DefaultDSLContext.java:611)
at org.jooq.impl.DefaultDSLContext.lambda$transactionResult0$3(DefaultDSLContext.java:549)
at org.jooq.impl.Tools$4$1.block(Tools.java:5243)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(Unknown Source)
at org.jooq.impl.Tools$4.get(Tools.java:5240)
at org.jooq.impl.DefaultDSLContext.transactionResult0(DefaultDSLContext.java:597)
at org.jooq.impl.DefaultDSLContext.transactionResult(DefaultDSLContext.java:521)
at org.jooq.impl.DefaultDSLContext.transaction(DefaultDSLContext.java:610)
at gov.cdc.prime.router.azure.DatabaseAccess.transact(DatabaseAccess.kt:72)
at gov.cdc.prime.router.azure.WorkflowEngine.handleReportEvent(WorkflowEngine.kt:140)
at gov.cdc.prime.router.azure.SendFunction.run(SendFunction.kt:63)
at jdk.internal.reflect.GeneratedMethodAccessor425.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.base/java.lang.reflect.Method.invoke(Unknown Source)
at com.microsoft.azure.functions.worker.broker.JavaMethodInvokeInfo.invoke(JavaMethodInvokeInfo.java:22)
at com.microsoft.azure.functions.worker.broker.EnhancedJavaMethodExecutorImpl.execute(EnhancedJavaMethodExecutorImpl.java:55)
at com.microsoft.azure.functions.worker.broker.JavaFunctionBroker.invokeMethod(JavaFunctionBroker.java:57)
at com.microsoft.azure.functions.worker.handler.InvocationRequestHandler.execute(InvocationRequestHandler.java:33)
at com.microsoft.azure.functions.worker.handler.InvocationRequestHandler.execute(InvocationRequestHandler.java:10)
at com.microsoft.azure.functions.worker.handler.MessageHandler.handle(MessageHandler.java:45)
at com.microsoft.azure.functions.worker.JavaWorkerClient$StreamingMessagePeer.lambda$onNext$0(JavaWorkerClient.java:92)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
at java.base/java.util.concurrent.FutureTask.run(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Suppressed: org.jooq.exception.DataAccessException: Cannot set autoCommit
at org.jooq.impl.DefaultConnectionProvider.setAutoCommit(DefaultConnectionProvider.java:207)
at org.jooq.impl.DefaultTransactionProvider.brace(DefaultTransactionProvider.java:226)
at org.jooq.impl.DefaultTransactionProvider.rollback(DefaultTransactionProvider.java:209)
at org.jooq.impl.DefaultDSLContext.lambda$transactionResult0$3(DefaultDSLContext.java:573)
... 24 more
Caused by: java.sql.SQLException: Connection is closed
at com.zaxxer.hikari.pool.ProxyConnection$ClosedConnection.lambda$getClosedConnection$0(ProxyConnection.java:515)
at com.sun.proxy.$Proxy28.setAutoCommit(Unknown Source)
at com.zaxxer.hikari.pool.ProxyConnection.setAutoCommit(ProxyConnection.java:414)
at com.zaxxer.hikari.pool.HikariProxyConnection.setAutoCommit(HikariProxyConnection.java)
at org.jooq.impl.DefaultConnectionProvider.setAutoCommit(DefaultConnectionProvider.java:204)
... 27 more
Caused by: org.postgresql.util.PSQLException: An I/O error occurred while sending to the backend.
at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:349)
at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:481)
at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:401)
at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:164)
at org.postgresql.jdbc.PgPreparedStatement.execute(PgPreparedStatement.java:153)
at com.zaxxer.hikari.pool.ProxyPreparedStatement.execute(ProxyPreparedStatement.java:44)
at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.execute(HikariProxyPreparedStatement.java)
at org.jooq.tools.jdbc.DefaultPreparedStatement.execute(DefaultPreparedStatement.java:219)
at org.jooq.impl.AbstractQuery.execute(AbstractQuery.java:414)
at org.jooq.impl.AbstractDMLQuery.execute(AbstractDMLQuery.java:984)
at org.jooq.impl.AbstractQuery.execute(AbstractQuery.java:335)
... 33 more
Caused by: javax.net.ssl.SSLException: Connection timed out (Read failed)
at java.base/sun.security.ssl.Alert.createSSLException(Unknown Source)
at java.base/sun.security.ssl.TransportContext.fatal(Unknown Source)
at java.base/sun.security.ssl.TransportContext.fatal(Unknown Source)
at java.base/sun.security.ssl.TransportContext.fatal(Unknown Source)
at java.base/sun.security.ssl.SSLSocketImpl.handleException(Unknown Source)
at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(Unknown Source)
at org.postgresql.core.VisibleBufferedInputStream.readMore(VisibleBufferedInputStream.java:161)
at org.postgresql.core.VisibleBufferedInputStream.ensureBytes(VisibleBufferedInputStream.java:128)
at org.postgresql.core.VisibleBufferedInputStream.ensureBytes(VisibleBufferedInputStream.java:113)
at org.postgresql.core.VisibleBufferedInputStream.read(VisibleBufferedInputStream.java:73)
at org.postgresql.core.PGStream.receiveChar(PGStream.java:443)
at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2056)
at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:322)
... 43 more
Suppressed: java.net.SocketException: Broken pipe (Write failed)
at java.base/java.net.SocketOutputStream.socketWrite0(Native Method)
at java.base/java.net.SocketOutputStream.socketWrite(Unknown Source)
at java.base/java.net.SocketOutputStream.write(Unknown Source)
at java.base/sun.security.ssl.SSLSocketOutputRecord.encodeAlert(Unknown Source)
... 55 more
Caused by: java.net.SocketException: Connection timed out (Read failed)
at java.base/java.net.SocketInputStream.socketRead0(Native Method)
at java.base/java.net.SocketInputStream.socketRead(Unknown Source)
at java.base/java.net.SocketInputStream.read(Unknown Source)
at java.base/java.net.SocketInputStream.read(Unknown Source)
at java.base/sun.security.ssl.SSLSocketInputRecord.read(Unknown Source)
at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(Unknown Source)
at java.base/sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(Unknown Source)
at java.base/sun.security.ssl.SSLSocketImpl.readApplicationRecord(Unknown Source)
... 51 more
First time I've seen this timeout error, which occurred in the SendFunction when trying to write to the TASK table.
The Send was actually successful, then the error occurred when writing that info to the database.
An I/O error occurred while sending to the backend. SocketException. Full exception is in the comment.
The subsequent write to to the ACTION table succeeded, as it records this:
action_id 334314, send_error, Tue, Oct 26, 2021, 23:32, report&SEND&7ce33fc9-4e27-42e4-a96d-447a70b95222 Success: sftp upload of covid-19-7ce33fc9-4e27-42e4-a96d-447a70b95222-20211026231537.hl7 to SFTPTransportType(host=sftp-edts.hhs.texas.gov, port=22, filePath=./, credentialName=null), Send function unrecoverable exception for event: report&SEND&7ce33fc9-4e27-42e4-a96d-447a70b95222