Infinispan Server failed to start org.infinispan.manager.EmbeddedCacheManagerStartupException

[SSamyak]

Hi I deployed Infinispan with two instances on Kubernetes using this Helm chart. Initially, everything worked fine, but unexpectedly, the Infinispan pods started crashing yesterday evening. Despite my efforts to identify the root cause, I couldn't pinpoint the issue. As a result, I decided to redeploy the cluster, and it is functioning correctly now. Could someone please help what could be the reason for that. Here are the logs from crashing pod 2024-01-21 04:39:38,857 INFO (main) [org.infinispan.CONTAINER] ISPN000104: Using EmbeddedTransactionManager 2024-01-21 04:39:46,055 ERROR (main) [org.infinispan.CONFIG] ISPN000660: Cache offlineSessions start failed, stopping any running components org.infinispan.commons.CacheConfigurationException: ISPN000659: Component ComponentWrapper{name=org.infinispan.persistence.manager.PersistenceManager, instance=org.infinispan.persistence.manager.PersistenceManagerImpl@190bf8e4, status=FAILED} failed to start at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:584) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:379) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:252) at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:222) at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1009) at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:504) at org.infinispan.security.impl.SecureCacheImpl.start(SecureCacheImpl.java:129) at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:727) at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:673) at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:562) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:525) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14) at org.infinispan.security.Security.doPrivileged(Security.java:61) at org.infinispan.globalstate.impl.SecurityActions.doPrivileged(SecurityActions.java:30) at org.infinispan.globalstate.impl.SecurityActions.getCache(SecurityActions.java:39) at org.infinispan.globalstate.impl.GlobalConfigurationManagerImpl.start(GlobalConfigurationManagerImpl.java:114) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:61) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:48) at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:379) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:252) at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:779) at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:747) at org.infinispan.server.SecurityActions.lambda$startCacheManager$1(SecurityActions.java:68) at org.infinispan.security.Security.doPrivileged(Security.java:56) at org.infinispan.server.SecurityActions.doPrivileged(SecurityActions.java:40) at org.infinispan.server.SecurityActions.startCacheManager(SecurityActions.java:71) at org.infinispan.server.Server.run(Server.java:417) at org.infinispan.server.Bootstrap.runInternal(Bootstrap.java:173) at org.infinispan.server.tool.Main.run(Main.java:98) at org.infinispan.server.Bootstrap.main(Bootstrap.java:56) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.infinispan.server.loader.Loader.run(Loader.java:106) at org.infinispan.server.loader.Loader.main(Loader.java:51) Caused by: java.util.concurrent.CompletionException: java.lang.IllegalStateException: Error reading header from 2:784519 | 0 at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315) at java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:320) at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1807) at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35) at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1982) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1377) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.lang.IllegalStateException: Error reading header from 2:784519 | 0 at org.infinispan.persistence.sifs.IndexNode$LeafNode.getHeaderAndKey(IndexNode.java:1144) at org.infinispan.persistence.sifs.IndexNode$LeafNode.loadHeaderAndKey(IndexNode.java:1124) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:363) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:369) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:369) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:351) at org.infinispan.persistence.sifs.Index.getMaxSeqId(Index.java:398) at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.lambda$start$0(NonBlockingSoftIndexFileStore.java:237) at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) ... 5 more  2024-01-21 04:39:46,062 ERROR (main) [org.infinispan.CONFIG] ISPN000660: DefaultCacheManager start failed, stopping any running components org.infinispan.commons.CacheConfigurationException: ISPN000659: Component ComponentWrapper{name=org.infinispan.persistence.manager.PersistenceManager, instance=org.infinispan.persistence.manager.PersistenceManagerImpl@190bf8e4, status=FAILED} failed to start at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:584) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:379) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:252) at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:222) at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1009) at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:504) at org.infinispan.security.impl.SecureCacheImpl.start(SecureCacheImpl.java:129) at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:727) at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:673) at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:562) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:525) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14) at org.infinispan.security.Security.doPrivileged(Security.java:61) at org.infinispan.globalstate.impl.SecurityActions.doPrivileged(SecurityActions.java:30) at org.infinispan.globalstate.impl.SecurityActions.getCache(SecurityActions.java:39) at org.infinispan.globalstate.impl.GlobalConfigurationManagerImpl.start(GlobalConfigurationManagerImpl.java:114) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:61) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:48) at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:379) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:252) at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:779) at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:747) at org.infinispan.server.SecurityActions.lambda$startCacheManager$1(SecurityActions.java:68) at org.infinispan.security.Security.doPrivileged(Security.java:56) at org.infinispan.server.SecurityActions.doPrivileged(SecurityActions.java:40) at org.infinispan.server.SecurityActions.startCacheManager(SecurityActions.java:71) at org.infinispan.server.Server.run(Server.java:417) at org.infinispan.server.Bootstrap.runInternal(Bootstrap.java:173) at org.infinispan.server.tool.Main.run(Main.java:98) at org.infinispan.server.Bootstrap.main(Bootstrap.java:56) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.infinispan.server.loader.Loader.run(Loader.java:106) at org.infinispan.server.loader.Loader.main(Loader.java:51) Caused by: java.util.concurrent.CompletionException: java.lang.IllegalStateException: Error reading header from 2:784519 | 0 at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315) at java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:320) at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1807) at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35) at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1982) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1377) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.lang.IllegalStateException: Error reading header from 2:784519 | 0 at org.infinispan.persistence.sifs.IndexNode$LeafNode.getHeaderAndKey(IndexNode.java:1144) at org.infinispan.persistence.sifs.IndexNode$LeafNode.loadHeaderAndKey(IndexNode.java:1124) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:363) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:369) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:369) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:351) at org.infinispan.persistence.sifs.Index.getMaxSeqId(Index.java:398) at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.lambda$start$0(NonBlockingSoftIndexFileStore.java:237) at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) ... 5 more  2024-01-21 04:39:46,066 INFO (main) [org.infinispan.CLUSTER] ISPN000080: Disconnecting JGroups channelinfinispan 2024-01-21 04:39:46,075 INFO (main) [org.infinispan.CONTAINER] ISPN000390: Persisted state, version=14.0.21.Final timestamp=2024-01-21T04:39:46.074199586Z 2024-01-21 04:39:46,251 FATAL (main) [org.infinispan.SERVER] ISPN080028: Infinispan Server failed to start org.infinispan.manager.EmbeddedCacheManagerStartupException: ISPN000659: Component ComponentWrapper{name=org.infinispan.persistence.manager.PersistenceManager, instance=org.infinispan.persistence.manager.PersistenceManagerImpl@190bf8e4, status=FAILED} failed to start at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:782) at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:747) at org.infinispan.server.SecurityActions.lambda$startCacheManager$1(SecurityActions.java:68) at org.infinispan.security.Security.doPrivileged(Security.java:56) at org.infinispan.server.SecurityActions.doPrivileged(SecurityActions.java:40) at org.infinispan.server.SecurityActions.startCacheManager(SecurityActions.java:71) at org.infinispan.server.Server.run(Server.java:417) at org.infinispan.server.Bootstrap.runInternal(Bootstrap.java:173) at org.infinispan.server.tool.Main.run(Main.java:98) at org.infinispan.server.Bootstrap.main(Bootstrap.java:56) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.infinispan.server.loader.Loader.run(Loader.java:106) at org.infinispan.server.loader.Loader.main(Loader.java:51) Caused by: org.infinispan.commons.CacheConfigurationException: ISPN000659: Component ComponentWrapper{name=org.infinispan.persistence.manager.PersistenceManager, instance=org.infinispan.persistence.manager.PersistenceManagerImpl@190bf8e4, status=FAILED} failed to start at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:584) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:379) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:252) at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:222) at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1009) at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:504) at org.infinispan.security.impl.SecureCacheImpl.start(SecureCacheImpl.java:129) at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:727) at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:673) at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:562) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:525) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:26) at org.infinispan.security.actions.GetCacheAction.run(GetCacheAction.java:14) at org.infinispan.security.Security.doPrivileged(Security.java:61) at org.infinispan.globalstate.impl.SecurityActions.doPrivileged(SecurityActions.java:30) at org.infinispan.globalstate.impl.SecurityActions.getCache(SecurityActions.java:39) at org.infinispan.globalstate.impl.GlobalConfigurationManagerImpl.start(GlobalConfigurationManagerImpl.java:114) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:61) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:48) at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:379) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:252) at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:779) ... 15 more Caused by: java.util.concurrent.CompletionException: java.lang.IllegalStateException: Error reading header from 2:784519 | 0 at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315) at java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:320) at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1807) at org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35) at org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1982) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1377) at java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.lang.IllegalStateException: Error reading header from 2:784519 | 0 at org.infinispan.persistence.sifs.IndexNode$LeafNode.getHeaderAndKey(IndexNode.java:1144) at org.infinispan.persistence.sifs.IndexNode$LeafNode.loadHeaderAndKey(IndexNode.java:1124) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:363) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:369) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:369) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:351) at org.infinispan.persistence.sifs.Index.getMaxSeqId(Index.java:398) at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.lambda$start$0(NonBlockingSoftIndexFileStore.java:237) at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) ... 5 more 

[rigazilla]

java.base/java.lang.Thread.run(Thread.java:840) Caused by: java.lang.IllegalStateException: Error reading header from 2:784519 | 0 at org.infinispan.persistence.sifs.IndexNode$LeafNode.getHeaderAndKey(IndexNode.java:1144) at

Seems something related to sifs, maybe @wburns can help

[wburns]

What version? This can no longer happen on ISPN main and those changes haven't yet been backported to 14.0.

[SSamyak]

i am using infinispan version 14.0.

[wburns]

i am using infinispan version 14.0.

I would also suggest being on the latest minor? There were a lot of fixes recently for SIFS that reduces the chances of this happening btw.

[SSamyak]

i am using infinispan version 14.0.

I would also suggest being on the latest minor? There were a lot of fixes recently for SIFS that reduces the chances of this happening btw.

ok, any particular version you would recommend.? Asking this because i dont see any minor release for version 14.

[wburns]

i am using infinispan version 14.0.

I would also suggest being on the latest minor? There were a lot of fixes recently for SIFS that reduces the chances of this happening btw.

ok, any particular version you would recommend.?

Whatever the newest 14 is, which should be 14.0.22 at the current time.

[SSamyak]

ok cool, thanks man..!!!

[ryanemerson]

Whatever the newest 14 is, which should be 14.0.22 at the current time.

FYI the 14.0 docker tag, e.g. quay.io/infinispan/server:14.0, always pulls the latest patch release for the 14.0.x stream.

[SSamyak]

i got this error again on version 15.0.0.Final today morning, there was resource crunch on our cluster around that time, could that be causing this.?

Caused by: org.infinispan.commons.CacheConfigurationException: ISPN000659: Component ComponentWrapper{name=org.infinispan.persistence.manager.PersistenceManager, instance=org.infinispan.persistence.manager.PersistenceManagerImpl@7fb82f73, status=FAILED} failed to start at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:584) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startDependencies(BasicComponentRegistryImpl.java:634) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:598) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:370) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:243) at org.infinispan.factories.ComponentRegistry.start(ComponentRegistry.java:222) at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1019) at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:511) at org.infinispan.security.impl.SecureCacheImpl.start(SecureCacheImpl.java:130) at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:727) at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:673) at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:567) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:530) at org.infinispan.security.actions.GetCacheAction.get(GetCacheAction.java:26) at org.infinispan.security.actions.GetCacheAction.get(GetCacheAction.java:14) at org.infinispan.security.Security.doPrivileged(Security.java:54) at org.infinispan.security.actions.SecurityActions.getCache(SecurityActions.java:55) at org.infinispan.globalstate.impl.GlobalConfigurationManagerImpl.start(GlobalConfigurationManagerImpl.java:115) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:61) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:48) at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:370) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:243) at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:779) ... 12 more Caused by: java.util.concurrent.CompletionException: java.lang.IllegalStateException: Error reading header from 0:10715308 | 0 at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315) at java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:320) at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1807) at org.jboss.threads.ContextHandler$1.runWith(ContextHandler.java:18) at org.jboss.threads.EnhancedQueueExecutor$Task.doRunWith(EnhancedQueueExecutor.java:2516) at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2495) at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1521) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: java.lang.IllegalStateException: Error reading header from 0:10715308 | 0 at org.infinispan.persistence.sifs.IndexNode$LeafNode.getHeaderAndKey(IndexNode.java:1126) at org.infinispan.persistence.sifs.IndexNode$LeafNode.loadHeaderAndKey(IndexNode.java:1106) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:362) at org.infinispan.persistence.sifs.IndexNode.calculateMaxSeqId(IndexNode.java:350) at org.infinispan.persistence.sifs.Index.getMaxSeqId(Index.java:457) at org.infinispan.persistence.sifs.NonBlockingSoftIndexFileStore.lambda$start$0(NonBlockingSoftIndexFileStore.java:242) at java.base/java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1804) ... 5 more

[samyakjain224]

https://github.com/infinispan/infinispan-helm-charts/issues/103#issuecomment-1993697114 @wburns could you please help here.

[cherrycharan]

#103 (comment) @wburns could you please help here.

To add some context to the above error , there were two namespaces in the cluster one with infinispan and another with new-infinispan . new-infinispan was the namespace which application was using and the other was redundant.

Multi-Attach error for volume "pvc-359ebe0e-c4bf-47ec-a242-e7ef300fcb07" Volume is already exclusively attached to one node and can't be attached to another

above was the error observed.

[ryanemerson]

@cherrycharan That's strange that you're encountering this error, as Infinispan is deployed using a StatefulSet which should prevent this. Are you explicitly configuring deploy.container.storage.storageClassName?

[SSamyak]

@ryanemerson no we are not configuring this in values.yml or anywhere in out deployment

deploy.container.storage.storageClassName

[cherrycharan]

2024-03-26 04:49:14,380 INFO (main) [BOOT] JVM OpenJDK 64-Bit Server VM Red Hat, Inc. 21.0.2+13-LTS 2024-03-26 04:49:14,399 INFO (main) [BOOT] JVM arguments = [-server, --add-exports, java.naming/com.sun.jndi.ldap=ALL-UNNAMED, --add-opens, java.base/java.util=ALL-UNNAMED, --add-opens, java.base/java.util.concurrent=ALL-UNNAMED, -Xlog:gc*:file=/opt/infinispan/server/log/gc.log:time,uptimemillis:filecount=5,filesize=3M, -Djgroups.dns.query=infinispan-ping.new-infinispan.svc.cluster.local, -Xms4096m, -Xmx8192m, -XX:+ExitOnOutOfMemoryError, -XX:MetaspaceSize=32m, -XX:MaxMetaspaceSize=96m, -Djava.net.preferIPv4Stack=true, -Djava.awt.headless=true, -Dvisualvm.display.name=infinispan-server, -Djava.util.logging.manager=org.infinispan.server.loader.LogManager, -Dinfinispan.server.home.path=/opt/infinispan, -classpath, :/opt/infinispan/boot/infinispan-server-runtime-15.0.0.Final-loader.jar, org.infinispan.server.loader.Loader, org.infinispan.server.Bootstrap, --cluster-name=infinispan, --server-config=/etc/config/infinispan.yml, --logging-config=/etc/config/log4j2.xml, --bind-address=0.0.0.0] 2024-03-26 04:49:14,405 INFO (main) [BOOT] PID = 178 2024-03-26 04:49:14,495 INFO (main) [org.infinispan.SERVER] ISPN080000: Infinispan Server 15.0.0.Final starting 2024-03-26 04:49:14,496 INFO (main) [org.infinispan.SERVER] ISPN080017: Server configuration: /etc/config/infinispan.yml 2024-03-26 04:49:14,496 INFO (main) [org.infinispan.SERVER] ISPN080032: Logging configuration: /etc/config/log4j2.xml 2024-03-26 04:49:15,730 INFO (main) [org.infinispan.SERVER] ISPN080027: Loaded extension 'query-dsl-filter-converter-factory' 2024-03-26 04:49:15,731 INFO (main) [org.infinispan.SERVER] ISPN080027: Loaded extension 'continuous-query-filter-converter-factory' 2024-03-26 04:49:15,735 INFO (main) [org.infinispan.SERVER] ISPN080027: Loaded extension 'iteration-filter-converter-factory' 2024-03-26 04:49:15,737 WARN (main) [org.infinispan.SERVER] ISPN080059: No script engines are available 2024-03-26 04:49:16,756 INFO (main) [org.infinispan.CONTAINER] ISPN000556: Starting user marshaller 'org.infinispan.commons.marshall.ImmutableProtoStreamMarshaller' 2024-03-26 04:49:17,862 INFO (main) [org.infinispan.CLUSTER] ISPN000078: Starting JGroups channelinfinispanwith stackkubernetes 2024-03-26 04:49:17,868 INFO (main) [org.jgroups.JChannel] local_addr: 4bc45b9c-3b15-4fea-965c-74050f9aa615, name: infinispan-0-28717 2024-03-26 04:49:17,886 INFO (main) [org.jgroups.protocols.FD_SOCK2] server listening on *.57800 2024-03-26 04:49:18,019 INFO (main) [org.infinispan.CLUSTER] ISPN000094: Received new cluster view for channel infinispan: [infinispan-1-27026|59] (2) [infinispan-1-27026, infinispan-0-28717] 2024-03-26 04:49:18,149 INFO (main) [org.infinispan.CLUSTER] ISPN000079: Channelinfinispanlocal address isinfinispan-0-28717, physical addresses are[10.20.63.224:7800] 2024-03-26 04:49:18,155 ERROR (main) [org.infinispan.CONFIG] ISPN000660: DefaultCacheManager start failed, stopping any running components org.infinispan.commons.CacheConfigurationException: ISPN000693: Dangling lock file '/opt/infinispan/server/data/___global.lck' in persistent global state, probably left behind by an unclean shutdown. at org.infinispan.globalstate.impl.GlobalStateManagerImpl.acquireGlobalLock(GlobalStateManagerImpl.java:84) at org.infinispan.globalstate.impl.GlobalStateManagerImpl.start(GlobalStateManagerImpl.java:65) at org.infinispan.globalstate.impl.CorePackageImpl$1.start(CorePackageImpl.java:34) at org.infinispan.globalstate.impl.CorePackageImpl$1.start(CorePackageImpl.java:27) at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:370) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:243) at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:779) at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:747) at org.infinispan.server.SecurityActions.lambda$startCacheManager$0(SecurityActions.java:41) at org.infinispan.security.Security.doPrivileged(Security.java:62) at org.infinispan.server.SecurityActions.startCacheManager(SecurityActions.java:43) at org.infinispan.server.Server.run(Server.java:446) at org.infinispan.server.Bootstrap.runInternal(Bootstrap.java:184) at org.infinispan.server.tool.Main.run(Main.java:98) at org.infinispan.server.Bootstrap.main(Bootstrap.java:67) at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) at java.base/java.lang.reflect.Method.invoke(Method.java:580) at org.infinispan.server.loader.Loader.run(Loader.java:104) at org.infinispan.server.loader.Loader.main(Loader.java:49)  2024-03-26 04:49:18,157 WARN (main) [org.infinispan.CONTAINER] ISPN000574: Global state cannot persisted because it is incomplete (usually caused by errors at startup). 2024-03-26 04:49:18,158 INFO (main) [org.infinispan.CLUSTER] ISPN000080: Disconnecting JGroups channelinfinispan 2024-03-26 04:49:18,282 FATAL (main) [org.infinispan.SERVER] ISPN080028: Infinispan Server failed to start org.infinispan.manager.EmbeddedCacheManagerStartupException: ISPN000693: Dangling lock file '/opt/infinispan/server/data/___global.lck' in persistent global state, probably left behind by an unclean shutdown. at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:782) at org.infinispan.manager.DefaultCacheManager.start(DefaultCacheManager.java:747) at org.infinispan.server.SecurityActions.lambda$startCacheManager$0(SecurityActions.java:41) at org.infinispan.security.Security.doPrivileged(Security.java:62) at org.infinispan.server.SecurityActions.startCacheManager(SecurityActions.java:43) at org.infinispan.server.Server.run(Server.java:446) at org.infinispan.server.Bootstrap.runInternal(Bootstrap.java:184) at org.infinispan.server.tool.Main.run(Main.java:98) at org.infinispan.server.Bootstrap.main(Bootstrap.java:67) at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) at java.base/java.lang.reflect.Method.invoke(Method.java:580) at org.infinispan.server.loader.Loader.run(Loader.java:104) at org.infinispan.server.loader.Loader.main(Loader.java:49) Caused by: org.infinispan.commons.CacheConfigurationException: ISPN000693: Dangling lock file '/opt/infinispan/server/data/___global.lck' in persistent global state, probably left behind by an unclean shutdown. at org.infinispan.globalstate.impl.GlobalStateManagerImpl.acquireGlobalLock(GlobalStateManagerImpl.java:84) at org.infinispan.globalstate.impl.GlobalStateManagerImpl.start(GlobalStateManagerImpl.java:65) at org.infinispan.globalstate.impl.CorePackageImpl$1.start(CorePackageImpl.java:34) at org.infinispan.globalstate.impl.CorePackageImpl$1.start(CorePackageImpl.java:27) at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:370) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:243) at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:779) ... 12 more 

Above is the Pod logs , how to fix this , tried restrating the pod but still facing this error

[cherrycharan]

For the above error , We have deleted PVC manually and restarted the pod and it worked.

[ryanemerson]

@cherrycharan You can workaround this issue by adding the following to your values.yaml:

  infinispan:
    cacheContainer:
      name: default
      globalState:
        uncleanShutdownAction: IGNORE

This should be included in the values.yaml by default for Infinispan 15 servers. I've created #119 to address this.

[samyakjain224]

@ryanemerson getting this error now Caused by: org.infinispan.commons.TimeoutException: ISPN000658: Initial state transfer timed out for cache clientSessions on infinispan-0-42272 at org.infinispan.statetransfer.StateTransferManagerImpl.waitForInitialStateTransferToComplete(StateTransferManagerImpl.java:235) at org.infinispan.cache.impl.CacheImpl.start(CacheImpl.java:1024) at org.infinispan.cache.impl.AbstractDelegatingCache.start(AbstractDelegatingCache.java:511) at org.infinispan.security.impl.SecureCacheImpl.start(SecureCacheImpl.java:130) at org.infinispan.manager.DefaultCacheManager.wireAndStartCache(DefaultCacheManager.java:727) at org.infinispan.manager.DefaultCacheManager.createCache(DefaultCacheManager.java:673) at org.infinispan.manager.DefaultCacheManager.internalGetCache(DefaultCacheManager.java:567) at org.infinispan.manager.DefaultCacheManager.getCache(DefaultCacheManager.java:530) at org.infinispan.security.actions.GetCacheAction.get(GetCacheAction.java:26) at org.infinispan.security.actions.GetCacheAction.get(GetCacheAction.java:14) at org.infinispan.security.Security.doPrivileged(Security.java:54) at org.infinispan.security.actions.SecurityActions.getCache(SecurityActions.java:55) at org.infinispan.globalstate.impl.GlobalConfigurationManagerImpl.start(GlobalConfigurationManagerImpl.java:115) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:61) at org.infinispan.globalstate.impl.CorePackageImpl$2.start(CorePackageImpl.java:48) at org.infinispan.factories.impl.BasicComponentRegistryImpl.invokeStart(BasicComponentRegistryImpl.java:616) at org.infinispan.factories.impl.BasicComponentRegistryImpl.doStartWrapper(BasicComponentRegistryImpl.java:607) at org.infinispan.factories.impl.BasicComponentRegistryImpl.startWrapper(BasicComponentRegistryImpl.java:576) at org.infinispan.factories.impl.BasicComponentRegistryImpl$ComponentWrapper.running(BasicComponentRegistryImpl.java:807) at org.infinispan.factories.AbstractComponentRegistry.internalStart(AbstractComponentRegistry.java:370) at org.infinispan.factories.AbstractComponentRegistry.start(AbstractComponentRegistry.java:243) at org.infinispan.manager.DefaultCacheManager.internalStart(DefaultCacheManager.java:779) ... 12 more

[ryanemerson]

@samyakjain224 Can you post the full logs of your Infinispan pods?