search

dsukhoroslov / bagri

XML/Document DB on top of distributed cache
Apache License 2.0
41 stars 5 forks source link

[Core] Error on cache population #161

Open dsukhoroslov opened 6 years ago

dsukhoroslov commented 6 years ago

shouldn't check cache size as it is too expensive:

2018-06-18 13:52:58.609 [hz.Inventory-24.partition-operation.thread-51] WARN  com.hazelcast.spi.impl.operationservice.impl.Invocation - [172.20.3.20]:10709 [Inventory] [3.10.1] Retrying invocation: Invocation{op=com.hazelcast.map.impl.operation.MapSizeOperation{serviceName='hz:impl:mapService', identityHash=1671141207, partitionId=627, replicaIndex=0, callId=600467, invocationTime=1529319178609 (2018-06-18 13:52:58.609), waitTimeout=-1, callTimeout=60000, name=xdm-document}, tryCount=250, tryPauseMillis=500, invokeCount=210, callTimeoutMillis=60000, firstInvocationTimeMs=1529319072764, firstInvocationTime='2018-06-18 13:51:12.764', lastHeartbeatMillis=0, lastHeartbeatTime='1970-01-01 03:00:00.000', target=[172.20.3.20]:10709, pendingResponse={VOID}, backupsAcksExpected=0, backupsAcksReceived=0, connection=Connection[id=24, /172.20.3.20:10731->/172.20.3.13:10701, endpoint=[172.20.3.13]:10701, alive=true, type=MEMBER]}, Reason: com.hazelcast.spi.exception.RetryableHazelcastException: Map xdm-document is still loading data from external store
2018-06-18 13:53:03.612 [hz.Inventory-24.partition-operation.thread-51] WARN  com.hazelcast.spi.impl.operationservice.impl.Invocation - [172.20.3.20]:10709 [Inventory] [3.10.1] Retrying invocation: Invocation{op=com.hazelcast.map.impl.operation.MapSizeOperation{serviceName='hz:impl:mapService', identityHash=1671141207, partitionId=627, replicaIndex=0, callId=726569, invocationTime=1529319183611 (2018-06-18 13:53:03.611), waitTimeout=-1, callTimeout=60000, name=xdm-document}, tryCount=250, tryPauseMillis=500, invokeCount=220, callTimeoutMillis=60000, firstInvocationTimeMs=1529319072764, firstInvocationTime='2018-06-18 13:51:12.764', lastHeartbeatMillis=0, lastHeartbeatTime='1970-01-01 03:00:00.000', target=[172.20.3.20]:10709, pendingResponse={VOID}, backupsAcksExpected=0, backupsAcksReceived=0, connection=Connection[id=24, /172.20.3.20:10731->/172.20.3.13:10701, endpoint=[172.20.3.13]:10701, alive=true, type=MEMBER]}, Reason: com.hazelcast.spi.exception.RetryableHazelcastException: Map xdm-document is still loading data from external store
2018-06-18 13:53:07.616 [hz.Inventory-24.partition-operation.thread-51] WARN  com.hazelcast.spi.impl.operationservice.impl.Invocation - [172.20.3.20]:10709 [Inventory] [3.10.1] Retrying invocation: Invocation{op=com.hazelcast.map.impl.operation.MapSizeOperation{serviceName='hz:impl:mapService', identityHash=1671141207, partitionId=627, replicaIndex=0, callId=848823, invocationTime=1529319187617 (2018-06-18 13:53:07.617), waitTimeout=-1, callTimeout=60000, name=xdm-document}, tryCount=250, tryPauseMillis=500, invokeCount=230, callTimeoutMillis=60000, firstInvocationTimeMs=1529319072764, firstInvocationTime='2018-06-18 13:51:12.764', lastHeartbeatMillis=0, lastHeartbeatTime='1970-01-01 03:00:00.000', target=[172.20.3.20]:10709, pendingResponse={VOID}, backupsAcksExpected=0, backupsAcksReceived=0, connection=Connection[id=24, /172.20.3.20:10731->/172.20.3.13:10701, endpoint=[172.20.3.13]:10701, alive=true, type=MEMBER]}, Reason: com.hazelcast.spi.exception.RetryableHazelcastException: Map xdm-document is still loading data from external store
2018-06-18 13:53:09.129 [hz.Inventory-24.partition-operation.thread-51] WARN  com.hazelcast.spi.impl.operationservice.impl.Invocation - [172.20.3.20]:10709 [Inventory] [3.10.1] Retrying invocation: Invocation{op=com.hazelcast.map.impl.operation.MapSizeOperation{serviceName='hz:impl:mapService', identityHash=1671141207, partitionId=627, replicaIndex=0, callId=909623, invocationTime=1529319189130 (2018-06-18 13:53:09.130), waitTimeout=-1, callTimeout=60000, name=xdm-document}, tryCount=250, tryPauseMillis=500, invokeCount=240, callTimeoutMillis=60000, firstInvocationTimeMs=1529319072764, firstInvocationTime='2018-06-18 13:51:12.764', lastHeartbeatMillis=0, lastHeartbeatTime='1970-01-01 03:00:00.000', target=[172.20.3.20]:10709, pendingResponse={VOID}, backupsAcksExpected=0, backupsAcksReceived=0, connection=Connection[id=24, /172.20.3.20:10731->/172.20.3.13:10701, endpoint=[172.20.3.13]:10701, alive=true, type=MEMBER]}, Reason: com.hazelcast.spi.exception.RetryableHazelcastException: Map xdm-document is still loading data from external store
2018-06-18 13:53:14.418 [hz.hzInstance-24.cached.thread-15] ERROR com.bagri.server.hazelcast.task.schema.SchemaInitiator - initSchema.error; Map xdm-document is still loading data from external store
com.hazelcast.spi.exception.RetryableHazelcastException: Map xdm-document is still loading data from external store
    at com.hazelcast.map.impl.recordstore.DefaultRecordStore.checkIfLoaded(DefaultRecordStore.java:1106)
    at com.hazelcast.map.impl.operation.MapSizeOperation.run(MapSizeOperation.java:38)
    at com.hazelcast.spi.Operation.call(Operation.java:148)
    at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:202)
    at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:191)
    at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:120)
    at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.run(OperationThread.java:100)
    at ------ submitted from ------.(Unknown Source)
    at com.hazelcast.spi.impl.operationservice.impl.InvocationFuture.resolve(InvocationFuture.java:127)
    at com.hazelcast.spi.impl.operationservice.impl.InvocationFuture.resolveAndThrowIfException(InvocationFuture.java:79)
    at com.hazelcast.spi.impl.AbstractInvocationFuture.get(AbstractInvocationFuture.java:162)
    at com.hazelcast.spi.impl.operationservice.impl.InvokeOnPartitions.retryFailedPartitions(InvokeOnPartitions.java:144)
    at com.hazelcast.spi.impl.operationservice.impl.InvokeOnPartitions.invoke(InvokeOnPartitions.java:74)
    at com.hazelcast.spi.impl.operationservice.impl.OperationServiceImpl.invokeOnAllPartitions(OperationServiceImpl.java:377)
    at com.hazelcast.map.impl.proxy.MapProxySupport.size(MapProxySupport.java:692)
    at com.hazelcast.map.impl.proxy.MapProxyImpl.size(MapProxyImpl.java:105)
    at com.bagri.server.hazelcast.store.MemoryMappedStore.loadEntries(MemoryMappedStore.java:208)
    at com.bagri.server.hazelcast.store.DocumentMemoryStore.init(DocumentMemoryStore.java:87)
    at com.bagri.server.hazelcast.impl.PopulationManagementImpl.activateDocStore(PopulationManagementImpl.java:212)
    at com.bagri.server.hazelcast.impl.PopulationManagementImpl.checkPopulation(PopulationManagementImpl.java:149)
    at com.bagri.server.hazelcast.task.schema.SchemaInitiator.call(SchemaInitiator.java:77)
    at com.bagri.server.hazelcast.task.schema.SchemaInitiator.call(SchemaInitiator.java:32)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at com.hazelcast.executor.impl.DistributedExecutorService$CallableProcessor.run(DistributedExecutorService.java:285)
    at com.hazelcast.util.executor.CachedExecutorServiceDelegate$Worker.run(CachedExecutorServiceDelegate.java:227)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)
    at com.hazelcast.util.executor.HazelcastManagedThread.executeRun(HazelcastManagedThread.java:64)
    at com.hazelcast.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:80)
dsukhoroslov commented 6 years ago

another exception is:

2018-06-18 13:53:14.418 [hz.hzInstance-24.cached.thread-15] ERROR com.bagri.server.hazelcast.task.schema.SchemaInitiator - initSchema.error; Map xdm-document is still loading data from external store
com.hazelcast.spi.exception.RetryableHazelcastException: Map xdm-document is still loading data from external store
    at com.hazelcast.map.impl.recordstore.DefaultRecordStore.checkIfLoaded(DefaultRecordStore.java:1106)
    at com.hazelcast.map.impl.operation.MapSizeOperation.run(MapSizeOperation.java:38)
    at com.hazelcast.spi.Operation.call(Operation.java:148)
    at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.call(OperationRunnerImpl.java:202)
    at com.hazelcast.spi.impl.operationservice.impl.OperationRunnerImpl.run(OperationRunnerImpl.java:191)
    at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.process(OperationThread.java:120)
    at com.hazelcast.spi.impl.operationexecutor.impl.OperationThread.run(OperationThread.java:100)
    at ------ submitted from ------.(Unknown Source)
    at com.hazelcast.spi.impl.operationservice.impl.InvocationFuture.resolve(InvocationFuture.java:127)
    at com.hazelcast.spi.impl.operationservice.impl.InvocationFuture.resolveAndThrowIfException(InvocationFuture.java:79)
    at com.hazelcast.spi.impl.AbstractInvocationFuture.get(AbstractInvocationFuture.java:162)
    at com.hazelcast.spi.impl.operationservice.impl.InvokeOnPartitions.retryFailedPartitions(InvokeOnPartitions.java:144)
    at com.hazelcast.spi.impl.operationservice.impl.InvokeOnPartitions.invoke(InvokeOnPartitions.java:74)
    at com.hazelcast.spi.impl.operationservice.impl.OperationServiceImpl.invokeOnAllPartitions(OperationServiceImpl.java:377)
    at com.hazelcast.map.impl.proxy.MapProxySupport.size(MapProxySupport.java:692)
    at com.hazelcast.map.impl.proxy.MapProxyImpl.size(MapProxyImpl.java:105)
    at com.bagri.server.hazelcast.store.MemoryMappedStore.loadEntries(MemoryMappedStore.java:208)
    at com.bagri.server.hazelcast.store.DocumentMemoryStore.init(DocumentMemoryStore.java:87)
    at com.bagri.server.hazelcast.impl.PopulationManagementImpl.activateDocStore(PopulationManagementImpl.java:212)
    at com.bagri.server.hazelcast.impl.PopulationManagementImpl.checkPopulation(PopulationManagementImpl.java:149)
    at com.bagri.server.hazelcast.task.schema.SchemaInitiator.call(SchemaInitiator.java:77)
    at com.bagri.server.hazelcast.task.schema.SchemaInitiator.call(SchemaInitiator.java:32)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at com.hazelcast.executor.impl.DistributedExecutorService$CallableProcessor.run(DistributedExecutorService.java:285)
    at com.hazelcast.util.executor.CachedExecutorServiceDelegate$Worker.run(CachedExecutorServiceDelegate.java:227)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at java.lang.Thread.run(Thread.java:748)
    at com.hazelcast.util.executor.HazelcastManagedThread.executeRun(HazelcastManagedThread.java:64)
    at com.hazelcast.util.executor.HazelcastManagedThread.run(HazelcastManagedThread.java:80)