search

treasure-data / digdag

Workload Automation System
https://www.digdag.io/
Apache License 2.0
1.3k stars 221 forks source link

Fixed to retry when loading plugin fails #1728

Closed hnarimiya closed 2 years ago

hnarimiya commented 2 years ago

I fixed it to retry when the plugin fails to load.

I sometimes get the following error.

java.lang.RuntimeException: org.eclipse.aether.resolution.DependencyResolutionException: Could not transfer artifact pro.civitaspo:digdag-operator-pg_lock:jar:0.0.3 from/to repository-1 (https://jitpack.io): Connection reset
    at com.google.common.base.Throwables.propagate(Throwables.java:241)
    at io.digdag.core.plugin.RemotePluginLoader.resolveArtifacts(RemotePluginLoader.java:167)
    at io.digdag.core.plugin.RemotePluginLoader.load(RemotePluginLoader.java:119)
    at io.digdag.core.plugin.DynamicPluginLoader.loadCache(DynamicPluginLoader.java:66)
    at io.digdag.core.plugin.DynamicPluginLoader.lambda$load$0(DynamicPluginLoader.java:54)
    at com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4718)
    at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3445)
    at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2194)
    at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2153)
    at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2043)
    at com.google.common.cache.LocalCache.get(LocalCache.java:3851)
    at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4713)
    at io.digdag.core.plugin.DynamicPluginLoader.load(DynamicPluginLoader.java:54)
    at io.digdag.core.agent.OperatorRegistry.get(OperatorRegistry.java:97)
    at io.digdag.core.agent.OperatorManager.callExecutor(OperatorManager.java:324)
    at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invokeMain(DigdagTimedMethodInterceptor.java:58)
    at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invoke(DigdagTimedMethodInterceptor.java:31)
    at io.digdag.core.agent.OperatorManager.runWithWorkspace(OperatorManager.java:298)
    at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invokeMain(DigdagTimedMethodInterceptor.java:58)
    at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invoke(DigdagTimedMethodInterceptor.java:31)
    at io.digdag.core.agent.OperatorManager.lambda$runWithHeartbeat$2(OperatorManager.java:151)
    at io.digdag.core.agent.ExtractArchiveWorkspaceManager.withExtractedArchive(ExtractArchiveWorkspaceManager.java:77)
    at io.digdag.core.agent.OperatorManager.runWithHeartbeat(OperatorManager.java:149)
    at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invokeMain(DigdagTimedMethodInterceptor.java:58)
    at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invoke(DigdagTimedMethodInterceptor.java:31)
    at io.digdag.core.agent.OperatorManager.run(OperatorManager.java:132)
    at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invokeMain(DigdagTimedMethodInterceptor.java:58)
    at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invoke(DigdagTimedMethodInterceptor.java:31)
    at io.digdag.core.agent.MultiThreadAgent.lambda$null$0(MultiThreadAgent.java:132)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)
Caused by: org.eclipse.aether.resolution.DependencyResolutionException: Could not transfer artifact pro.civitaspo:digdag-operator-pg_lock:jar:0.0.3 from/to repository-1 (https://jitpack.io): Connection reset
    at org.eclipse.aether.internal.impl.DefaultRepositorySystem.resolveDependencies(DefaultRepositorySystem.java:391)
    at io.digdag.core.plugin.RemotePluginLoader.resolveArtifacts(RemotePluginLoader.java:164)
    ... 32 common frames omitted
Caused by: org.eclipse.aether.resolution.ArtifactResolutionException: Could not transfer artifact pro.civitaspo:digdag-operator-pg_lock:jar:0.0.3 from/to repository-1 (https://jitpack.io): Connection reset
    at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolve(DefaultArtifactResolver.java:444)
    at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolveArtifacts(DefaultArtifactResolver.java:246)
    at org.eclipse.aether.internal.impl.DefaultRepositorySystem.resolveDependencies(DefaultRepositorySystem.java:373)
    ... 33 common frames omitted
Caused by: org.eclipse.aether.transfer.ArtifactTransferException: Could not transfer artifact pro.civitaspo:digdag-operator-pg_lock:jar:0.0.3 from/to repository-1 (https://jitpack.io): Connection reset
    at org.eclipse.aether.connector.basic.ArtifactTransportListener.transferFailed(ArtifactTransportListener.java:43)
    at org.eclipse.aether.connector.basic.BasicRepositoryConnector$TaskRunner.run(BasicRepositoryConnector.java:355)
    at org.eclipse.aether.util.concurrency.RunnableErrorForwarder$1.run(RunnableErrorForwarder.java:67)
    at org.eclipse.aether.connector.basic.BasicRepositoryConnector$DirectExecutor.execute(BasicRepositoryConnector.java:581)
    at org.eclipse.aether.connector.basic.BasicRepositoryConnector.get(BasicRepositoryConnector.java:249)
    at org.eclipse.aether.internal.impl.DefaultArtifactResolver.performDownloads(DefaultArtifactResolver.java:520)
    at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolve(DefaultArtifactResolver.java:421)
    ... 35 common frames omitted
Caused by: javax.net.ssl.SSLException: Connection reset
    at sun.security.ssl.Alert.createSSLException(Alert.java:127)
    at sun.security.ssl.TransportContext.fatal(TransportContext.java:324)
    at sun.security.ssl.TransportContext.fatal(TransportContext.java:267)
    at sun.security.ssl.TransportContext.fatal(TransportContext.java:262)
    at sun.security.ssl.SSLTransport.decode(SSLTransport.java:138)
    at sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1395)
    at sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1363)
    at sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73)
    at sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:957)
    at org.apache.http.impl.io.AbstractSessionInputBuffer.read(AbstractSessionInputBuffer.java:200)
    at org.apache.http.impl.io.ContentLengthInputStream.read(ContentLengthInputStream.java:176)
    at org.apache.http.conn.EofSensorInputStream.read(EofSensorInputStream.java:135)
    at org.apache.http.conn.EofSensorInputStream.read(EofSensorInputStream.java:148)
    at org.eclipse.aether.spi.connector.transport.AbstractTransporter.copy(AbstractTransporter.java:200)
    at org.eclipse.aether.spi.connector.transport.AbstractTransporter.utilGet(AbstractTransporter.java:96)
    at org.eclipse.aether.transport.http.HttpTransporter.access$100(HttpTransporter.java:72)
    at org.eclipse.aether.transport.http.HttpTransporter$EntityGetter.handle(HttpTransporter.java:569)
    at org.eclipse.aether.transport.http.HttpTransporter.execute(HttpTransporter.java:325)
    at org.eclipse.aether.transport.http.HttpTransporter.implGet(HttpTransporter.java:274)
    at org.eclipse.aether.spi.connector.transport.AbstractTransporter.get(AbstractTransporter.java:59)
    at org.eclipse.aether.connector.basic.BasicRepositoryConnector$GetTaskRunner.runTask(BasicRepositoryConnector.java:447)
    at org.eclipse.aether.connector.basic.BasicRepositoryConnector$TaskRunner.run(BasicRepositoryConnector.java:350)
    ... 40 common frames omitted
    Suppressed: java.net.SocketException: Broken pipe (Write failed)
        at java.net.SocketOutputStream.socketWrite0(Native Method)
        at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
        at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
        at sun.security.ssl.SSLSocketOutputRecord.encodeAlert(SSLSocketOutputRecord.java:81)
        at sun.security.ssl.TransportContext.fatal(TransportContext.java:355)
        at sun.security.ssl.TransportContext.fatal(TransportContext.java:267)
        at sun.security.ssl.TransportContext.fatal(TransportContext.java:262)
        at sun.security.ssl.SSLTransport.decode(SSLTransport.java:138)
        at sun.security.ssl.SSLSocketImpl.decode(SSLSocketImpl.java:1395)
        at sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1363)
        at sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73)
        at sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:957)
        at org.apache.http.impl.io.AbstractSessionInputBuffer.read(AbstractSessionInputBuffer.java:200)
        at org.apache.http.impl.io.ContentLengthInputStream.read(ContentLengthInputStream.java:176)
        at org.apache.http.conn.EofSensorInputStream.read(EofSensorInputStream.java:135)
        at org.apache.http.conn.EofSensorInputStream.read(EofSensorInputStream.java:148)
        at org.eclipse.aether.spi.connector.transport.AbstractTransporter.copy(AbstractTransporter.java:200)
        at org.eclipse.aether.spi.connector.transport.AbstractTransporter.utilGet(AbstractTransporter.java:96)
        at org.eclipse.aether.transport.http.HttpTransporter.access$100(HttpTransporter.java:72)
        at org.eclipse.aether.transport.http.HttpTransporter$EntityGetter.handle(HttpTransporter.java:569)
        at org.eclipse.aether.transport.http.HttpTransporter.execute(HttpTransporter.java:325)
        at org.eclipse.aether.transport.http.HttpTransporter.implGet(HttpTransporter.java:274)
        at org.eclipse.aether.spi.connector.transport.AbstractTransporter.get(AbstractTransporter.java:59)
        at org.eclipse.aether.connector.basic.BasicRepositoryConnector$GetTaskRunner.runTask(BasicRepositoryConnector.java:447)
        at org.eclipse.aether.connector.basic.BasicRepositoryConnector$TaskRunner.run(BasicRepositoryConnector.java:350)
        at org.eclipse.aether.util.concurrency.RunnableErrorForwarder$1.run(RunnableErrorForwarder.java:67)
        at org.eclipse.aether.connector.basic.BasicRepositoryConnector$DirectExecutor.execute(BasicRepositoryConnector.java:581)
        at org.eclipse.aether.connector.basic.BasicRepositoryConnector.get(BasicRepositoryConnector.java:249)
        at org.eclipse.aether.internal.impl.DefaultArtifactResolver.performDownloads(DefaultArtifactResolver.java:520)
        at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolve(DefaultArtifactResolver.java:421)
        at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolveArtifacts(DefaultArtifactResolver.java:246)
        at org.eclipse.aether.internal.impl.DefaultRepositorySystem.resolveDependencies(DefaultRepositorySystem.java:373)
        at io.digdag.core.plugin.RemotePluginLoader.resolveArtifacts(RemotePluginLoader.java:164)
        at io.digdag.core.plugin.RemotePluginLoader.load(RemotePluginLoader.java:119)
        at io.digdag.core.plugin.DynamicPluginLoader.loadCache(DynamicPluginLoader.java:66)
        at io.digdag.core.plugin.DynamicPluginLoader.lambda$load$0(DynamicPluginLoader.java:54)
        at com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4718)
        at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3445)
        at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2194)
        at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2153)
        at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2043)
        at com.google.common.cache.LocalCache.get(LocalCache.java:3851)
        at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4713)
        at io.digdag.core.plugin.DynamicPluginLoader.load(DynamicPluginLoader.java:54)
        at io.digdag.core.agent.OperatorRegistry.get(OperatorRegistry.java:97)
        at io.digdag.core.agent.OperatorManager.callExecutor(OperatorManager.java:324)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7.CGLIB$callExecutor$4(<generated>)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7$$FastClassByGuice$$f6c4ba5c.invoke(<generated>)
        at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228)
        at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76)
        at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invokeMain(DigdagTimedMethodInterceptor.java:58)
        at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invoke(DigdagTimedMethodInterceptor.java:31)
        at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:78)
        at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:54)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7.callExecutor(<generated>)
        at io.digdag.core.agent.OperatorManager.runWithWorkspace(OperatorManager.java:298)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7.CGLIB$runWithWorkspace$2(<generated>)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7$$FastClassByGuice$$f6c4ba5c.invoke(<generated>)
        at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228)
        at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76)
        at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invokeMain(DigdagTimedMethodInterceptor.java:58)
        at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invoke(DigdagTimedMethodInterceptor.java:31)
        at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:78)
        at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:54)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7.runWithWorkspace(<generated>)
        at io.digdag.core.agent.OperatorManager.lambda$runWithHeartbeat$2(OperatorManager.java:151)
        at io.digdag.core.agent.ExtractArchiveWorkspaceManager.withExtractedArchive(ExtractArchiveWorkspaceManager.java:77)
        at io.digdag.core.agent.OperatorManager.runWithHeartbeat(OperatorManager.java:149)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7.CGLIB$runWithHeartbeat$0(<generated>)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7$$FastClassByGuice$$f6c4ba5c.invoke(<generated>)
        at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228)
        at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76)
        at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invokeMain(DigdagTimedMethodInterceptor.java:58)
        at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invoke(DigdagTimedMethodInterceptor.java:31)
        at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:78)
        at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:54)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7.runWithHeartbeat(<generated>)
        at io.digdag.core.agent.OperatorManager.run(OperatorManager.java:132)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7.CGLIB$run$5(<generated>)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7$$FastClassByGuice$$f6c4ba5c.invoke(<generated>)
        at com.google.inject.internal.cglib.proxy.$MethodProxy.invokeSuper(MethodProxy.java:228)
        at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:76)
        at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invokeMain(DigdagTimedMethodInterceptor.java:58)
        at io.digdag.server.metrics.DigdagTimedMethodInterceptor.invoke(DigdagTimedMethodInterceptor.java:31)
        at com.google.inject.internal.InterceptorStackCallback$InterceptedMethodInvocation.proceed(InterceptorStackCallback.java:78)
        at com.google.inject.internal.InterceptorStackCallback.intercept(InterceptorStackCallback.java:54)
        at io.digdag.core.agent.OperatorManager$$EnhancerByGuice$$d80147f7.run(<generated>)
        ... 6 common frames omitted
Caused by: java.net.SocketException: Connection reset
    at java.net.SocketInputStream.read(SocketInputStream.java:210)
    at java.net.SocketInputStream.read(SocketInputStream.java:141)
    at sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:464)
    at sun.security.ssl.SSLSocketInputRecord.decodeInputRecord(SSLSocketInputRecord.java:237)
    at sun.security.ssl.SSLSocketInputRecord.decode(SSLSocketInputRecord.java:190)
    at sun.security.ssl.SSLTransport.decode(SSLTransport.java:109)
yoyama commented 2 years ago

@hnarimiya Thank you for your contribution. It seems LGTM. But it might be better to add tests for it.

hnarimiya commented 2 years ago

@yoyama Thank you for your review. I couldn't think of a good way to test, but I added a test to check the log in the integration test. Please review again.