search

jenkinsci / google-storage-plugin

This plugin provides the “Google Cloud Storage Uploader” post-build step for publishing build artifacts to Google Cloud Storage.
https://plugins.jenkins.io/google-storage-plugin/
Apache License 2.0
31 stars 57 forks source link

Concurrent modification exception from BuildGcsUploadReport #61

Open petehastwofeet opened 5 years ago

petehastwofeet commented 5 years ago

Using the plugin from a multi branch pipeline with a number of parallel stages, the following has been observed sporadically.

java.util.ConcurrentModificationException
    at java.util.HashMap$HashIterator.nextNode(HashMap.java:1445)
    at java.util.HashMap$KeyIterator.next(HashMap.java:1469)
    at com.thoughtworks.xstream.converters.collections.CollectionConverter.marshal(CollectionConverter.java:73)
    at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
    at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
    at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:84)
    at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:265)
    at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:252)
Caused: java.lang.RuntimeException: Failed to serialize com.google.jenkins.plugins.storage.reports.BuildGcsUploadReport#files for class com.google.jenkins.plugins.storage.reports.BuildGcsUploadReport
    at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:256)
    at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:224)
    at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:138)
    at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:209)
    at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:150)
    at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
    at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
    at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
    at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:88)
    at com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter.writeItem(AbstractCollectionConverter.java:64)
    at com.thoughtworks.xstream.converters.collections.CollectionConverter.marshal(CollectionConverter.java:74)
    at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
    at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
    at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:84)
    at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:265)
    at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:252)
Caused: java.lang.RuntimeException: Failed to serialize hudson.model.Actionable#actions for class org.jenkinsci.plugins.workflow.job.WorkflowRun
    at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:256)
    at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:224)
    at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:138)
    at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:209)
    at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:150)
    at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
    at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
    at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
    at com.thoughtworks.xstream.core.TreeMarshaller.start(TreeMarshaller.java:82)
    at com.thoughtworks.xstream.core.AbstractTreeMarshallingStrategy.marshal(AbstractTreeMarshallingStrategy.java:37)
    at com.thoughtworks.xstream.XStream.marshal(XStream.java:1026)
    at com.thoughtworks.xstream.XStream.marshal(XStream.java:1015)
    at com.thoughtworks.xstream.XStream.toXML(XStream.java:988)
    at hudson.util.XStream2.toXMLUTF8(XStream2.java:313)
    at org.jenkinsci.plugins.workflow.support.PipelineIOUtils.writeByXStream(PipelineIOUtils.java:34)
    at org.jenkinsci.plugins.workflow.job.WorkflowRun.save(WorkflowRun.java:1292)
    at hudson.BulkChange.commit(BulkChange.java:98)
    at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.notifyListeners(CpsFlowExecution.java:1446)
    at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$3.run(CpsThreadGroup.java:417)
    at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$1.run(CpsVmExecutorService.java:35)
    at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:131)
    at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
    at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)
Finished: FAILURE
v1v commented 4 years ago

I also got the same stacktrace:

[2020-10-23T12:06:05.234Z] java.util.ConcurrentModificationException
[2020-10-23T12:06:05.234Z]  at java.util.HashMap$HashIterator.nextNode(HashMap.java:1445)
[2020-10-23T12:06:05.234Z]  at java.util.HashMap$KeyIterator.next(HashMap.java:1469)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.converters.collections.CollectionConverter.marshal(CollectionConverter.java:73)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:84)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:264)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:251)
[2020-10-23T12:06:05.234Z] Caused: java.lang.RuntimeException: Failed to serialize com.google.jenkins.plugins.storage.reports.BuildGcsUploadReport#files for class com.google.jenkins.plugins.storage.reports.BuildGcsUploadReport
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:255)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:223)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:138)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:209)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:150)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:88)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter.writeItem(AbstractCollectionConverter.java:64)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.converters.collections.CollectionConverter.marshal(CollectionConverter.java:74)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:84)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:264)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:251)
[2020-10-23T12:06:05.234Z] Caused: java.lang.RuntimeException: Failed to serialize hudson.model.Actionable#actions for class org.jenkinsci.plugins.workflow.job.WorkflowRun
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:255)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:223)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:138)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:209)
[2020-10-23T12:06:05.234Z]  at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:150)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:69)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.TreeMarshaller.start(TreeMarshaller.java:82)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.core.AbstractTreeMarshallingStrategy.marshal(AbstractTreeMarshallingStrategy.java:37)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.XStream.marshal(XStream.java:1026)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.XStream.marshal(XStream.java:1015)
[2020-10-23T12:06:05.234Z]  at com.thoughtworks.xstream.XStream.toXML(XStream.java:988)
[2020-10-23T12:06:05.234Z]  at hudson.util.XStream2.toXMLUTF8(XStream2.java:313)
[2020-10-23T12:06:05.234Z]  at org.jenkinsci.plugins.workflow.support.PipelineIOUtils.writeByXStream(PipelineIOUtils.java:34)
[2020-10-23T12:06:05.234Z]  at org.jenkinsci.plugins.workflow.job.WorkflowRun.save(WorkflowRun.java:1199)
[2020-10-23T12:06:05.234Z]  at hudson.BulkChange.commit(BulkChange.java:98)
[2020-10-23T12:06:05.234Z]  at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.notifyListeners(CpsFlowExecution.java:1475)
[2020-10-23T12:06:05.234Z]  at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$3.run(CpsThreadGroup.java:489)
[2020-10-23T12:06:05.234Z]  at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$1.run(CpsVmExecutorService.java:38)
[2020-10-23T12:06:05.234Z]  at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:136)
[2020-10-23T12:06:05.234Z]  at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
[2020-10-23T12:06:05.234Z]  at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59)
[2020-10-23T12:06:05.234Z]  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[2020-10-23T12:06:05.234Z]  at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[2020-10-23T12:06:05.234Z]  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[2020-10-23T12:06:05.234Z]  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[2020-10-23T12:06:05.234Z]  at java.lang.Thread.run(Thread.java:748)
[2020-10-23T12:06:05.234Z] Finished: FAILURE

Logs -> here

In our case we use up to 70 parallel stages that interact with Google storage concurrently.

v1v commented 3 years ago

I'm afraid the proposal done in https://github.com/jenkinsci/google-storage-plugin/pull/120

didn't help

[2021-04-30T19:08:01.435Z] 
[2021-04-30T19:08:01.435Z] GitHub has been notified of this commit’s build result
[2021-04-30T19:08:01.435Z] 
[2021-04-30T19:08:01.435Z] java.util.ConcurrentModificationException
[2021-04-30T19:08:01.435Z]  at java.util.HashMap$HashIterator.nextNode(HashMap.java:1445)
[2021-04-30T19:08:01.435Z]  at java.util.HashMap$KeyIterator.next(HashMap.java:1469)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.converters.collections.CollectionConverter.marshal(CollectionConverter.java:73)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:68)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:83)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:275)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:262)
[2021-04-30T19:08:01.435Z] Caused: java.lang.RuntimeException: Failed to serialize com.google.jenkins.plugins.storage.reports.BuildGcsUploadReport#files for class com.google.jenkins.plugins.storage.reports.BuildGcsUploadReport
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:266)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:233)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:150)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:219)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:158)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:68)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:87)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter.writeBareItem(AbstractCollectionConverter.java:94)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter.writeItem(AbstractCollectionConverter.java:66)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter.writeCompleteItem(AbstractCollectionConverter.java:81)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.converters.collections.CollectionConverter.marshal(CollectionConverter.java:74)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:68)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller$1.convertAnother(AbstractReferenceMarshaller.java:83)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter.marshallField(RobustReflectionConverter.java:275)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:262)
[2021-04-30T19:08:01.435Z] Caused: java.lang.RuntimeException: Failed to serialize hudson.model.Actionable#actions for class org.jenkinsci.plugins.workflow.job.WorkflowRun
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter$2.writeField(RobustReflectionConverter.java:266)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter$2.visit(RobustReflectionConverter.java:233)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.converters.reflection.PureJavaReflectionProvider.visitSerializableFields(PureJavaReflectionProvider.java:150)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter.doMarshal(RobustReflectionConverter.java:219)
[2021-04-30T19:08:01.435Z]  at hudson.util.RobustReflectionConverter.marshal(RobustReflectionConverter.java:158)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.AbstractReferenceMarshaller.convert(AbstractReferenceMarshaller.java:68)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:58)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.TreeMarshaller.convertAnother(TreeMarshaller.java:43)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.TreeMarshaller.start(TreeMarshaller.java:82)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.core.AbstractTreeMarshallingStrategy.marshal(AbstractTreeMarshallingStrategy.java:37)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.XStream.marshal(XStream.java:1276)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.XStream.marshal(XStream.java:1265)
[2021-04-30T19:08:01.435Z]  at com.thoughtworks.xstream.XStream.toXML(XStream.java:1238)
[2021-04-30T19:08:01.435Z]  at hudson.util.XStream2.toXMLUTF8(XStream2.java:312)
[2021-04-30T19:08:01.435Z]  at org.jenkinsci.plugins.workflow.support.PipelineIOUtils.writeByXStream(PipelineIOUtils.java:34)
[2021-04-30T19:08:01.435Z]  at org.jenkinsci.plugins.workflow.job.WorkflowRun.save(WorkflowRun.java:1210)
[2021-04-30T19:08:01.435Z]  at hudson.BulkChange.commit(BulkChange.java:98)
[2021-04-30T19:08:01.435Z]  at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.notifyListeners(CpsFlowExecution.java:1485)
[2021-04-30T19:08:01.435Z]  at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$3.run(CpsThreadGroup.java:489)
[2021-04-30T19:08:01.435Z]  at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$1.run(CpsVmExecutorService.java:38)
[2021-04-30T19:08:01.435Z]  at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:139)
[2021-04-30T19:08:01.435Z]  at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
[2021-04-30T19:08:01.435Z]  at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68)
[2021-04-30T19:08:01.435Z]  at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[2021-04-30T19:08:01.435Z]  at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[2021-04-30T19:08:01.435Z]  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[2021-04-30T19:08:01.435Z]  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[2021-04-30T19:08:01.435Z]  at java.lang.Thread.run(Thread.java:748)
[2021-04-30T19:08:01.435Z] Finished: FAILURE

https://github.com/jenkinsci/workflow-scm-step-plugin/pull/27 seems what they did for another plugin though