[JBoss JIRA] (ISPN-9044) In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
by Debashish Bharali (JIRA)
[ https://issues.jboss.org/browse/ISPN-9044?page=com.atlassian.jira.plugin.... ]
Debashish Bharali edited comment on ISPN-9044 at 4/9/18 3:58 AM:
-----------------------------------------------------------------
{color:red}*---ReplStateTransferCacheLoaderTest.java---*{color}
@Test(groups = "functional", testName = "statetransfer.ReplStateTransferCacheLoaderTest")
@CleanupAfterMethod
public class ReplStateTransferCacheLoaderTest extends MultipleCacheManagersTest implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(ReplStateTransferCacheLoaderTest.class);
private File tmpDir;
private ConfigurationBuilder builder;
@Override
protected void createCacheManagers() {
tmpDir = new File(TestingUtil.tmpDirectory(this.getClass()));
Util.recursiveFileRemove(tmpDir);
// reproduce the MODE-1754 config as closely as possible
builder = getDefaultClusteredCacheConfig(CacheMode.REPL_SYNC, true, true);
builder.transaction().*transactionMode(TransactionMode.NON_TRANSACTIONAL)*.*lockingMode(LockingMode.OPTIMISTIC)*
.transactionManagerLookup(new DummyTransactionManagerLookup())
.eviction().*maxEntries(100)*.*strategy(EvictionStrategy.LRU)*
.locking().lockAcquisitionTimeout(20000)
.*concurrencyLevel(1000)* // lowering this to 50 makes the test pass also on 5.2 but it's just a temporary workaround
.useLockStriping(false).writeSkewCheck(false).isolationLevel(IsolationLevel.READ_COMMITTED)
.dataContainer().storeAsBinary()
.clustering().sync().*replTimeout(120000)*
.*stateTransfer().timeout(480000)*.*fetchInMemoryState(true)*.chunkSize(10000).*awaitInitialTransfer(true)*
.persistence().passivation(false).addSingleFileStore().location(new File(tmpDir, "store0").getAbsolutePath()).shared(false).preload(false)
.*fetchPersistentState(true)*
.ignoreModifications(false)
.purgeOnStartup(false);
createCluster(builder, 1);
waitForClusterToForm();
}
@AfterClass
protected void clearTempDir() {
// Util.recursiveFileRemove(tmpDir);
}
public void *testStateTransfer*() throws Exception {
final Long *numKeys = 100000l*;
for (Long i = 0l; i < numKeys; i++) {
TestEntity testEntity = new TestEntity(i,"DEBA_"+i);
cache(0).put(i, testEntity);
}
log.info("Finished putting keys");
System.out.println("Debashish -- "+"Finished putting keys");
for (Long i = 0l; i < numKeys; i++) {
assertEquals(i, ((TestEntity)cache(0).get(i)).getId());
}
log.info("Adding a new node ..");
System.out.println("Debashish -- "+"Adding a new node ..");
builder.persistence().clearStores().addSingleFileStore().location(new File(tmpDir, "store1").getAbsolutePath()) // make sure this node writes in a different location
.fetchPersistentState(true)
.ignoreModifications(false)
.purgeOnStartup(false);
addClusterEnabledCacheManager(builder);
log.info("Added a new node");
System.out.println("Debashish -- "+"Added a new node");
for (Long i = 0l; i < numKeys; i++) {
assertEquals(i, ((TestEntity)cache(1).get(i)).getId());
// assertEquals(i, cache(1).get(i)); // some keys are lost in 5.2
}
}
}
was (Author: debashish.bharali):
{color:red}*---ReplStateTransferCacheLoaderTest.java---*{color}
@Test(groups = "functional", testName = "statetransfer.ReplStateTransferCacheLoaderTest")
@CleanupAfterMethod
public class ReplStateTransferCacheLoaderTest extends MultipleCacheManagersTest implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(ReplStateTransferCacheLoaderTest.class);
private File tmpDir;
private ConfigurationBuilder builder;
@Override
protected void createCacheManagers() {
tmpDir = new File(TestingUtil.tmpDirectory(this.getClass()));
Util.recursiveFileRemove(tmpDir);
// reproduce the MODE-1754 config as closely as possible
builder = getDefaultClusteredCacheConfig(CacheMode.REPL_SYNC, true, true);
builder.transaction().transactionMode(TransactionMode.NON_TRANSACTIONAL).lockingMode(LockingMode.OPTIMISTIC)
.transactionManagerLookup(new DummyTransactionManagerLookup())
.eviction().maxEntries(100).strategy(EvictionStrategy.LRU)
.locking().lockAcquisitionTimeout(20000)
.concurrencyLevel(1000) // lowering this to 50 makes the test pass also on 5.2 but it's just a temporary workaround
.useLockStriping(false).writeSkewCheck(false).isolationLevel(IsolationLevel.READ_COMMITTED)
.dataContainer().storeAsBinary()
.clustering().sync().replTimeout(120000)
.stateTransfer().timeout(480000).fetchInMemoryState(true).chunkSize(10000).awaitInitialTransfer(true)
.persistence().passivation(false).addSingleFileStore().location(new File(tmpDir, "store0").getAbsolutePath()).shared(false).preload(false)
.fetchPersistentState(true)
.ignoreModifications(false)
.purgeOnStartup(false);
createCluster(builder, 1);
waitForClusterToForm();
}
@AfterClass
protected void clearTempDir() {
// Util.recursiveFileRemove(tmpDir);
}
public void testStateTransfer() throws Exception {
final Long numKeys = 100000l;
for (Long i = 0l; i < numKeys; i++) {
TestEntity testEntity = new TestEntity(i,"DEBA_"+i);
cache(0).put(i, testEntity);
}
log.info("Finished putting keys");
System.out.println("Debashish -- "+"Finished putting keys");
for (Long i = 0l; i < numKeys; i++) {
assertEquals(i, ((TestEntity)cache(0).get(i)).getId());
}
log.info("Adding a new node ..");
System.out.println("Debashish -- "+"Adding a new node ..");
builder.persistence().clearStores().addSingleFileStore().location(new File(tmpDir, "store1").getAbsolutePath()) // make sure this node writes in a different location
.fetchPersistentState(true)
.ignoreModifications(false)
.purgeOnStartup(false);
addClusterEnabledCacheManager(builder);
log.info("Added a new node");
System.out.println("Debashish -- "+"Added a new node");
for (Long i = 0l; i < numKeys; i++) {
assertEquals(i, ((TestEntity)cache(1).get(i)).getId());
// assertEquals(i, cache(1).get(i)); // some keys are lost in 5.2
}
}
}
> In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
> -----------------------------------------------------------------------------------------------------------------------------
>
> Key: ISPN-9044
> URL: https://issues.jboss.org/browse/ISPN-9044
> Project: Infinispan
> Issue Type: Bug
> Components: Lucene Directory
> Affects Versions: 8.2.5.Final
> Reporter: Debashish Bharali
> Priority: Critical
> Attachments: neutrino-hibernatesearch-infinispan.xml
>
>
> Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node.
> Related to ISPN-8980 (https://issues.jboss.org/browse/ISPN-8980).
> We are using Hibernate Search Indexes - Lucene indexes being stored on Infinispan with SingleFileStore.
> In case of more than 1 node. For example 4 nodes. We are observing below behaviour.
> Below are the steps:
> # We startup the first node *'N1'* in maintenance mode - with MassIndexer - creating initial indexes.
> # Now after all the MassIndexer/EntityLoader threads ends (after 1-2 Hrs). I.e. MassIndexing has been completed. We startup all other 3 nodes *'N2' , 'N3' and 'N4'*. Without MassIndexer.
> # Now on moderate to heavy application usage (concurrency), we are again getting the same exception of *Exception occurred java.io.FileNotFoundException: Error loading metadata for index file. Which indicates, {color:red}Some entries are not present in cache.{color}*
> # *But this exception comes only on the other 3 nodes (N2, N3 and N4). Not on the first node N1.*
> # On checking the sizes of the Cache stores in all the Nodes, the 3 Nodes (N2,N3 and N4) are having almost equal size (600 MB), which is 50%-70% of the size of Cache Stores of N1 (1.2 GB).
> # We have repeated these steps multiple times. Even switched MassIndexing node to other 3 nodes too. We have even reduced the number of nodes to 2.
> # *But the behaviour is exactly same. I.e. Exception on all the nodes except the initial node doing MassIndexing.*
> # {color:red} It seems like, *'N1's* cache-store's persistent state is not getting fetched by *'N2' 'N3' and 'N4'*, when these node joins joins.{color}
> # This is indicated by the fact that, FileNotFoundException doesn't comes in 'N1'. It comes in other nodes only (who joined later -- like N2, N3 & N4). And size of cache store's *'.DAT'* files are smaller then *'N1's*.
> Require urgent support.
> Attaching the corresponding Infinispan config file (neutrino-hibernatesearch-infinispan.xml)
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
6 years, 9 months
[JBoss JIRA] (ISPN-9044) In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
by Debashish Bharali (JIRA)
[ https://issues.jboss.org/browse/ISPN-9044?page=com.atlassian.jira.plugin.... ]
Debashish Bharali commented on ISPN-9044:
-----------------------------------------
{color:red}*---TestEntity.java---*{color}
package org.infinispan.statetransfer;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
public class TestEntity implements Serializable {
private static final long serialVersionUID = 1L;
private long id;
private String name;
private Map<Long,String> dataMap;
// Inner class created for Creating heavy data
public TestEntity(long id, String name) {
super();
this.id = id;
this.name = name;
System.out.println("Debahsish : "+id);
dataMap = new HashMap<>();
for(Long i=0l;i<1000;i++) {
dataMap.put(i, name);
}
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (id ^ (id >>> 32));
result = prime * result + ((name == null) ? 0 : name.hashCode());
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
TestEntity other = (TestEntity) obj;
if (id != other.id)
return false;
if (name == null) {
if (other.name != null)
return false;
} else if (!name.equals(other.name))
return false;
return true;
}
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
> In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
> -----------------------------------------------------------------------------------------------------------------------------
>
> Key: ISPN-9044
> URL: https://issues.jboss.org/browse/ISPN-9044
> Project: Infinispan
> Issue Type: Bug
> Components: Lucene Directory
> Affects Versions: 8.2.5.Final
> Reporter: Debashish Bharali
> Priority: Critical
> Attachments: neutrino-hibernatesearch-infinispan.xml
>
>
> Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node.
> Related to ISPN-8980 (https://issues.jboss.org/browse/ISPN-8980).
> We are using Hibernate Search Indexes - Lucene indexes being stored on Infinispan with SingleFileStore.
> In case of more than 1 node. For example 4 nodes. We are observing below behaviour.
> Below are the steps:
> # We startup the first node *'N1'* in maintenance mode - with MassIndexer - creating initial indexes.
> # Now after all the MassIndexer/EntityLoader threads ends (after 1-2 Hrs). I.e. MassIndexing has been completed. We startup all other 3 nodes *'N2' , 'N3' and 'N4'*. Without MassIndexer.
> # Now on moderate to heavy application usage (concurrency), we are again getting the same exception of *Exception occurred java.io.FileNotFoundException: Error loading metadata for index file. Which indicates, {color:red}Some entries are not present in cache.{color}*
> # *But this exception comes only on the other 3 nodes (N2, N3 and N4). Not on the first node N1.*
> # On checking the sizes of the Cache stores in all the Nodes, the 3 Nodes (N2,N3 and N4) are having almost equal size (600 MB), which is 50%-70% of the size of Cache Stores of N1 (1.2 GB).
> # We have repeated these steps multiple times. Even switched MassIndexing node to other 3 nodes too. We have even reduced the number of nodes to 2.
> # *But the behaviour is exactly same. I.e. Exception on all the nodes except the initial node doing MassIndexing.*
> # {color:red} It seems like, *'N1's* cache-store's persistent state is not getting fetched by *'N2' 'N3' and 'N4'*, when these node joins joins.{color}
> # This is indicated by the fact that, FileNotFoundException doesn't comes in 'N1'. It comes in other nodes only (who joined later -- like N2, N3 & N4). And size of cache store's *'.DAT'* files are smaller then *'N1's*.
> Require urgent support.
> Attaching the corresponding Infinispan config file (neutrino-hibernatesearch-infinispan.xml)
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
6 years, 9 months
[JBoss JIRA] (ISPN-9044) In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
by Debashish Bharali (JIRA)
[ https://issues.jboss.org/browse/ISPN-9044?page=com.atlassian.jira.plugin.... ]
Debashish Bharali commented on ISPN-9044:
-----------------------------------------
{color:red}*---ReplStateTransferCacheLoaderTest.java---*{color}
@Test(groups = "functional", testName = "statetransfer.ReplStateTransferCacheLoaderTest")
@CleanupAfterMethod
public class ReplStateTransferCacheLoaderTest extends MultipleCacheManagersTest implements Serializable{
/**
*
*/
private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(ReplStateTransferCacheLoaderTest.class);
private File tmpDir;
private ConfigurationBuilder builder;
@Override
protected void createCacheManagers() {
tmpDir = new File(TestingUtil.tmpDirectory(this.getClass()));
Util.recursiveFileRemove(tmpDir);
// reproduce the MODE-1754 config as closely as possible
builder = getDefaultClusteredCacheConfig(CacheMode.REPL_SYNC, true, true);
builder.transaction().transactionMode(TransactionMode.NON_TRANSACTIONAL).lockingMode(LockingMode.OPTIMISTIC)
.transactionManagerLookup(new DummyTransactionManagerLookup())
.eviction().maxEntries(100).strategy(EvictionStrategy.LRU)
.locking().lockAcquisitionTimeout(20000)
.concurrencyLevel(1000) // lowering this to 50 makes the test pass also on 5.2 but it's just a temporary workaround
.useLockStriping(false).writeSkewCheck(false).isolationLevel(IsolationLevel.READ_COMMITTED)
.dataContainer().storeAsBinary()
.clustering().sync().replTimeout(120000)
.stateTransfer().timeout(480000).fetchInMemoryState(true).chunkSize(10000).awaitInitialTransfer(true)
.persistence().passivation(false).addSingleFileStore().location(new File(tmpDir, "store0").getAbsolutePath()).shared(false).preload(false)
.fetchPersistentState(true)
.ignoreModifications(false)
.purgeOnStartup(false);
createCluster(builder, 1);
waitForClusterToForm();
}
@AfterClass
protected void clearTempDir() {
// Util.recursiveFileRemove(tmpDir);
}
public void testStateTransfer() throws Exception {
final Long numKeys = 100000l;
for (Long i = 0l; i < numKeys; i++) {
TestEntity testEntity = new TestEntity(i,"DEBA_"+i);
cache(0).put(i, testEntity);
}
log.info("Finished putting keys");
System.out.println("Debashish -- "+"Finished putting keys");
for (Long i = 0l; i < numKeys; i++) {
assertEquals(i, ((TestEntity)cache(0).get(i)).getId());
}
log.info("Adding a new node ..");
System.out.println("Debashish -- "+"Adding a new node ..");
builder.persistence().clearStores().addSingleFileStore().location(new File(tmpDir, "store1").getAbsolutePath()) // make sure this node writes in a different location
.fetchPersistentState(true)
.ignoreModifications(false)
.purgeOnStartup(false);
addClusterEnabledCacheManager(builder);
log.info("Added a new node");
System.out.println("Debashish -- "+"Added a new node");
for (Long i = 0l; i < numKeys; i++) {
assertEquals(i, ((TestEntity)cache(1).get(i)).getId());
// assertEquals(i, cache(1).get(i)); // some keys are lost in 5.2
}
}
}
> In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
> -----------------------------------------------------------------------------------------------------------------------------
>
> Key: ISPN-9044
> URL: https://issues.jboss.org/browse/ISPN-9044
> Project: Infinispan
> Issue Type: Bug
> Components: Lucene Directory
> Affects Versions: 8.2.5.Final
> Reporter: Debashish Bharali
> Priority: Critical
> Attachments: neutrino-hibernatesearch-infinispan.xml
>
>
> Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node.
> Related to ISPN-8980 (https://issues.jboss.org/browse/ISPN-8980).
> We are using Hibernate Search Indexes - Lucene indexes being stored on Infinispan with SingleFileStore.
> In case of more than 1 node. For example 4 nodes. We are observing below behaviour.
> Below are the steps:
> # We startup the first node *'N1'* in maintenance mode - with MassIndexer - creating initial indexes.
> # Now after all the MassIndexer/EntityLoader threads ends (after 1-2 Hrs). I.e. MassIndexing has been completed. We startup all other 3 nodes *'N2' , 'N3' and 'N4'*. Without MassIndexer.
> # Now on moderate to heavy application usage (concurrency), we are again getting the same exception of *Exception occurred java.io.FileNotFoundException: Error loading metadata for index file. Which indicates, {color:red}Some entries are not present in cache.{color}*
> # *But this exception comes only on the other 3 nodes (N2, N3 and N4). Not on the first node N1.*
> # On checking the sizes of the Cache stores in all the Nodes, the 3 Nodes (N2,N3 and N4) are having almost equal size (600 MB), which is 50%-70% of the size of Cache Stores of N1 (1.2 GB).
> # We have repeated these steps multiple times. Even switched MassIndexing node to other 3 nodes too. We have even reduced the number of nodes to 2.
> # *But the behaviour is exactly same. I.e. Exception on all the nodes except the initial node doing MassIndexing.*
> # {color:red} It seems like, *'N1's* cache-store's persistent state is not getting fetched by *'N2' 'N3' and 'N4'*, when these node joins joins.{color}
> # This is indicated by the fact that, FileNotFoundException doesn't comes in 'N1'. It comes in other nodes only (who joined later -- like N2, N3 & N4). And size of cache store's *'.DAT'* files are smaller then *'N1's*.
> Require urgent support.
> Attaching the corresponding Infinispan config file (neutrino-hibernatesearch-infinispan.xml)
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
6 years, 9 months
[JBoss JIRA] (ISPN-9044) In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
by Debashish Bharali (JIRA)
[ https://issues.jboss.org/browse/ISPN-9044?page=com.atlassian.jira.plugin.... ]
Debashish Bharali commented on ISPN-9044:
-----------------------------------------
Due to network policies, I am unable to attach source files.
I am sharing the code in comment section.
> In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
> -----------------------------------------------------------------------------------------------------------------------------
>
> Key: ISPN-9044
> URL: https://issues.jboss.org/browse/ISPN-9044
> Project: Infinispan
> Issue Type: Bug
> Components: Lucene Directory
> Affects Versions: 8.2.5.Final
> Reporter: Debashish Bharali
> Priority: Critical
> Attachments: neutrino-hibernatesearch-infinispan.xml
>
>
> Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node.
> Related to ISPN-8980 (https://issues.jboss.org/browse/ISPN-8980).
> We are using Hibernate Search Indexes - Lucene indexes being stored on Infinispan with SingleFileStore.
> In case of more than 1 node. For example 4 nodes. We are observing below behaviour.
> Below are the steps:
> # We startup the first node *'N1'* in maintenance mode - with MassIndexer - creating initial indexes.
> # Now after all the MassIndexer/EntityLoader threads ends (after 1-2 Hrs). I.e. MassIndexing has been completed. We startup all other 3 nodes *'N2' , 'N3' and 'N4'*. Without MassIndexer.
> # Now on moderate to heavy application usage (concurrency), we are again getting the same exception of *Exception occurred java.io.FileNotFoundException: Error loading metadata for index file. Which indicates, {color:red}Some entries are not present in cache.{color}*
> # *But this exception comes only on the other 3 nodes (N2, N3 and N4). Not on the first node N1.*
> # On checking the sizes of the Cache stores in all the Nodes, the 3 Nodes (N2,N3 and N4) are having almost equal size (600 MB), which is 50%-70% of the size of Cache Stores of N1 (1.2 GB).
> # We have repeated these steps multiple times. Even switched MassIndexing node to other 3 nodes too. We have even reduced the number of nodes to 2.
> # *But the behaviour is exactly same. I.e. Exception on all the nodes except the initial node doing MassIndexing.*
> # {color:red} It seems like, *'N1's* cache-store's persistent state is not getting fetched by *'N2' 'N3' and 'N4'*, when these node joins joins.{color}
> # This is indicated by the fact that, FileNotFoundException doesn't comes in 'N1'. It comes in other nodes only (who joined later -- like N2, N3 & N4). And size of cache store's *'.DAT'* files are smaller then *'N1's*.
> Require urgent support.
> Attaching the corresponding Infinispan config file (neutrino-hibernatesearch-infinispan.xml)
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
6 years, 9 months
[JBoss JIRA] (ISPN-9044) In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
by Debashish Bharali (JIRA)
[ https://issues.jboss.org/browse/ISPN-9044?page=com.atlassian.jira.plugin.... ]
Debashish Bharali commented on ISPN-9044:
-----------------------------------------
[~dan.berindei] I have done multiple iterations of the suggested test *successfully*.
Corresponding files are attached.
We didn't face issue while doing state transfer.
We tried with 100000 objects with total size of CacheStoreFile approx 1.1 GB.
But we did face some issue when we tried to increase the individual object size, but it gave proper exception of StateTransferTimeOut.
>From these initial tests, it seems like, the issue is not related to StateTransfer and Fetching Persistent State of Cache Store for normal objects.
*But somehow, we are facing this issue for storing lucene indexes.*
Please comment,
> In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
> -----------------------------------------------------------------------------------------------------------------------------
>
> Key: ISPN-9044
> URL: https://issues.jboss.org/browse/ISPN-9044
> Project: Infinispan
> Issue Type: Bug
> Components: Lucene Directory
> Affects Versions: 8.2.5.Final
> Reporter: Debashish Bharali
> Priority: Critical
> Attachments: neutrino-hibernatesearch-infinispan.xml
>
>
> Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node.
> Related to ISPN-8980 (https://issues.jboss.org/browse/ISPN-8980).
> We are using Hibernate Search Indexes - Lucene indexes being stored on Infinispan with SingleFileStore.
> In case of more than 1 node. For example 4 nodes. We are observing below behaviour.
> Below are the steps:
> # We startup the first node *'N1'* in maintenance mode - with MassIndexer - creating initial indexes.
> # Now after all the MassIndexer/EntityLoader threads ends (after 1-2 Hrs). I.e. MassIndexing has been completed. We startup all other 3 nodes *'N2' , 'N3' and 'N4'*. Without MassIndexer.
> # Now on moderate to heavy application usage (concurrency), we are again getting the same exception of *Exception occurred java.io.FileNotFoundException: Error loading metadata for index file. Which indicates, {color:red}Some entries are not present in cache.{color}*
> # *But this exception comes only on the other 3 nodes (N2, N3 and N4). Not on the first node N1.*
> # On checking the sizes of the Cache stores in all the Nodes, the 3 Nodes (N2,N3 and N4) are having almost equal size (600 MB), which is 50%-70% of the size of Cache Stores of N1 (1.2 GB).
> # We have repeated these steps multiple times. Even switched MassIndexing node to other 3 nodes too. We have even reduced the number of nodes to 2.
> # *But the behaviour is exactly same. I.e. Exception on all the nodes except the initial node doing MassIndexing.*
> # {color:red} It seems like, *'N1's* cache-store's persistent state is not getting fetched by *'N2' 'N3' and 'N4'*, when these node joins joins.{color}
> # This is indicated by the fact that, FileNotFoundException doesn't comes in 'N1'. It comes in other nodes only (who joined later -- like N2, N3 & N4). And size of cache store's *'.DAT'* files are smaller then *'N1's*.
> Require urgent support.
> Attaching the corresponding Infinispan config file (neutrino-hibernatesearch-infinispan.xml)
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
6 years, 9 months
[JBoss JIRA] (ISPN-9044) In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
by Debashish Bharali (JIRA)
[ https://issues.jboss.org/browse/ISPN-9044?page=com.atlassian.jira.plugin.... ]
Debashish Bharali commented on ISPN-9044:
-----------------------------------------
Error mentioned in previous comment got resolved.
Reference used:
https://forum.hibernate.org/viewtopic.php?f=31&t=1041374
ISPN-1177
> In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
> -----------------------------------------------------------------------------------------------------------------------------
>
> Key: ISPN-9044
> URL: https://issues.jboss.org/browse/ISPN-9044
> Project: Infinispan
> Issue Type: Bug
> Components: Lucene Directory
> Affects Versions: 8.2.5.Final
> Reporter: Debashish Bharali
> Priority: Critical
> Attachments: neutrino-hibernatesearch-infinispan.xml
>
>
> Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node.
> Related to ISPN-8980 (https://issues.jboss.org/browse/ISPN-8980).
> We are using Hibernate Search Indexes - Lucene indexes being stored on Infinispan with SingleFileStore.
> In case of more than 1 node. For example 4 nodes. We are observing below behaviour.
> Below are the steps:
> # We startup the first node *'N1'* in maintenance mode - with MassIndexer - creating initial indexes.
> # Now after all the MassIndexer/EntityLoader threads ends (after 1-2 Hrs). I.e. MassIndexing has been completed. We startup all other 3 nodes *'N2' , 'N3' and 'N4'*. Without MassIndexer.
> # Now on moderate to heavy application usage (concurrency), we are again getting the same exception of *Exception occurred java.io.FileNotFoundException: Error loading metadata for index file. Which indicates, {color:red}Some entries are not present in cache.{color}*
> # *But this exception comes only on the other 3 nodes (N2, N3 and N4). Not on the first node N1.*
> # On checking the sizes of the Cache stores in all the Nodes, the 3 Nodes (N2,N3 and N4) are having almost equal size (600 MB), which is 50%-70% of the size of Cache Stores of N1 (1.2 GB).
> # We have repeated these steps multiple times. Even switched MassIndexing node to other 3 nodes too. We have even reduced the number of nodes to 2.
> # *But the behaviour is exactly same. I.e. Exception on all the nodes except the initial node doing MassIndexing.*
> # {color:red} It seems like, *'N1's* cache-store's persistent state is not getting fetched by *'N2' 'N3' and 'N4'*, when these node joins joins.{color}
> # This is indicated by the fact that, FileNotFoundException doesn't comes in 'N1'. It comes in other nodes only (who joined later -- like N2, N3 & N4). And size of cache store's *'.DAT'* files are smaller then *'N1's*.
> Require urgent support.
> Attaching the corresponding Infinispan config file (neutrino-hibernatesearch-infinispan.xml)
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
6 years, 9 months
[JBoss JIRA] (ISPN-9044) In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
by Debashish Bharali (JIRA)
[ https://issues.jboss.org/browse/ISPN-9044?page=com.atlassian.jira.plugin.... ]
Debashish Bharali commented on ISPN-9044:
-----------------------------------------
[~dan.berindei] I am getting below error on Running/Initializing the test. Please help in getting this resolved to move forward.
I am using Eclipse with TestNG plugin to execute this.
12:35:52,434 ERROR (testng-ReplStateTransferCacheLoaderTest) [ReplStateTransferCacheLoaderTest] Error in test setup:
java.lang.ExceptionInInitializerError
at org.infinispan.statetransfer.ReplStateTransferCacheLoaderTest.createCacheManagers(ReplStateTransferCacheLoaderTest.java:43)
at org.infinispan.test.MultipleCacheManagersTest.callCreateCacheManagers(MultipleCacheManagersTest.java:72)
at org.infinispan.test.MultipleCacheManagersTest.createBeforeMethod(MultipleCacheManagersTest.java:82)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84)
at org.testng.internal.Invoker.invokeConfigurationMethod(Invoker.java:564)FAILED CONFIGURATION: @BeforeMethod createBeforeMethod
java.lang.ExceptionInInitializerError
at org.infinispan.statetransfer.ReplStateTransferCacheLoaderTest.createCacheManagers(ReplStateTransferCacheLoaderTest.java:43)
at org.infinispan.test.MultipleCacheManagersTest.callCreateCacheManagers(MultipleCacheManagersTest.java:72)
at org.infinispan.test.MultipleCacheManagersTest.createBeforeMethod(MultipleCacheManagersTest.java:82)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:84)
at org.testng.internal.Invoker.invokeConfigurationMethod(Invoker.java:564)
at org.testng.internal.Invoker.invokeConfigurations(Invoker.java:213)
at org.testng.internal.Invoker.invokeMethod(Invoker.java:653)
at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:901)
at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1231)
at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:127)
at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:111)
at org.testng.TestRunner.privateRun(TestRunner.java:767)
at org.testng.TestRunner.run(TestRunner.java:617)
at org.testng.SuiteRunner.runTest(SuiteRunner.java:348)
at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:343)
at org.testng.SuiteRunner.privateRun(SuiteRunner.java:305)
at org.testng.SuiteRunner.run(SuiteRunner.java:254)
at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52)
at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:86)
at org.testng.TestNG.runSuitesSequentially(TestNG.java:1224)
at org.testng.TestNG.runSuitesLocally(TestNG.java:1149)
at org.testng.TestNG.run(TestNG.java:1057)
at org.testng.remote.AbstractRemoteTestNG.run(AbstractRemoteTestNG.java:132)
at org.testng.remote.RemoteTestNG.initAndRun(RemoteTestNG.java:236)
at org.testng.remote.RemoteTestNG.main(RemoteTestNG.java:81)
*{color:red}Caused by: java.lang.IllegalArgumentException: Invalid logger interface org.infinispan.commons.logging.Log (implementation not found in sun.misc.Launcher$AppClassLoader@18b4aac2){color}*
at org.jboss.logging.Logger$1.run(Logger.java:2556)
at java.security.AccessController.doPrivileged(Native Method)
at org.jboss.logging.Logger.getMessageLogger(Logger.java:2529)
at org.jboss.logging.Logger.getMessageLogger(Logger.java:2516)
at org.infinispan.commons.logging.LogFactory.getLog(LogFactory.java:15)
at org.infinispan.commons.util.Util.<clinit>(Util.java:60)
... 29 more
SKIPPED CONFIGURATION: @AfterMethod checkThreads
SKIPPED CONFIGURATION: @AfterClass clearTempDir
SKIPPED: testStateTransfer
===============================================
Default test
Tests run: 1, Failures: 0, Skips: 1
Configuration Failures: 1, Skips: 2
===============================================
> In Cluster - Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node
> -----------------------------------------------------------------------------------------------------------------------------
>
> Key: ISPN-9044
> URL: https://issues.jboss.org/browse/ISPN-9044
> Project: Infinispan
> Issue Type: Bug
> Components: Lucene Directory
> Affects Versions: 8.2.5.Final
> Reporter: Debashish Bharali
> Priority: Critical
> Attachments: neutrino-hibernatesearch-infinispan.xml
>
>
> Infinispan - SingleFileStore - fetchPersistentState/StateTransfer not transferring complete data to Joining Node.
> Related to ISPN-8980 (https://issues.jboss.org/browse/ISPN-8980).
> We are using Hibernate Search Indexes - Lucene indexes being stored on Infinispan with SingleFileStore.
> In case of more than 1 node. For example 4 nodes. We are observing below behaviour.
> Below are the steps:
> # We startup the first node *'N1'* in maintenance mode - with MassIndexer - creating initial indexes.
> # Now after all the MassIndexer/EntityLoader threads ends (after 1-2 Hrs). I.e. MassIndexing has been completed. We startup all other 3 nodes *'N2' , 'N3' and 'N4'*. Without MassIndexer.
> # Now on moderate to heavy application usage (concurrency), we are again getting the same exception of *Exception occurred java.io.FileNotFoundException: Error loading metadata for index file. Which indicates, {color:red}Some entries are not present in cache.{color}*
> # *But this exception comes only on the other 3 nodes (N2, N3 and N4). Not on the first node N1.*
> # On checking the sizes of the Cache stores in all the Nodes, the 3 Nodes (N2,N3 and N4) are having almost equal size (600 MB), which is 50%-70% of the size of Cache Stores of N1 (1.2 GB).
> # We have repeated these steps multiple times. Even switched MassIndexing node to other 3 nodes too. We have even reduced the number of nodes to 2.
> # *But the behaviour is exactly same. I.e. Exception on all the nodes except the initial node doing MassIndexing.*
> # {color:red} It seems like, *'N1's* cache-store's persistent state is not getting fetched by *'N2' 'N3' and 'N4'*, when these node joins joins.{color}
> # This is indicated by the fact that, FileNotFoundException doesn't comes in 'N1'. It comes in other nodes only (who joined later -- like N2, N3 & N4). And size of cache store's *'.DAT'* files are smaller then *'N1's*.
> Require urgent support.
> Attaching the corresponding Infinispan config file (neutrino-hibernatesearch-infinispan.xml)
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
6 years, 9 months
[JBoss JIRA] (ISPN-8719) KeySet.(iterator|spliterator|stream) not compatible with versions before 9.1
by William Burns (JIRA)
[ https://issues.jboss.org/browse/ISPN-8719?page=com.atlassian.jira.plugin.... ]
William Burns updated ISPN-8719:
--------------------------------
Git Pull Request: https://github.com/infinispan/infinispan/pull/5896, https://github.com/infinispan/infinispan/pull/5903 (was: https://github.com/infinispan/infinispan/pull/5896)
> KeySet.(iterator|spliterator|stream) not compatible with versions before 9.1
> ----------------------------------------------------------------------------
>
> Key: ISPN-8719
> URL: https://issues.jboss.org/browse/ISPN-8719
> Project: Infinispan
> Issue Type: Bug
> Affects Versions: 9.1.0.Final, 9.2.0.Final
> Reporter: Marek Posolda
> Assignee: William Burns
> Priority: Critical
> Fix For: 9.1.8.Final, 9.2.2.Final, 9.3.0.Alpha1
>
> Attachments: InfinispanRemote.java
>
>
> Steps to reproduce:
> 1) Use infinispan server version 8.2.6 (or JDG server 7.1.0) and start it.
> {code}
> cd JDG_HOME/bin
> ./standalone.sh
> {code}
> 2) Create sample java project having dependency on latest dependency 9.2.0.CR1 in pom.xml:
> {code}
> <dependencies>
> <dependency>
> <groupId>org.infinispan</groupId>
> <artifactId>infinispan-core</artifactId>
> <version>9.2.0.CR1</version>
> </dependency>
> <dependency>
> <groupId>org.infinispan</groupId>
> <artifactId>infinispan-cachestore-remote</artifactId>
> <version>9.2.0.CR1</version>
> </dependency>
> </dependencies>
> {code}
> 3) Add one simple java class based on the tutorial: http://infinispan.org/tutorials/simple/remote/ . The only difference is that I use hotRod protocolVersion 2.5 and calling:
> {code}
> remoteCache.keySet().iterator().hasNext()
> {code}. I am attaching the class in attachement.
> 4) Run the class. Seeing exception in both server log and on client-side.
> Server exception
> {code}
> 10:44:20,365 ERROR [org.infinispan.server.hotrod.CacheDecodeContext] (HotRodServerWorker-6-1) ISPN005003: Exception reported: java.lang.IllegalStateException: ISPN006016: Factory 'org.infinispan.server.hotrod.HotRodServer$ToEmptyBytesKeyValueFilterConverter' not found in server
> at org.infinispan.server.hotrod.iteration.DefaultIterationManager.getFactory(DefaultIterationManager.java:148)
> at org.infinispan.server.hotrod.iteration.DefaultIterationManager.start(DefaultIterationManager.java:131)
> at org.infinispan.server.hotrod.ContextHandler.realRead(ContextHandler.java:175)
> at org.infinispan.server.hotrod.ContextHandler.lambda$channelRead0$1(ContextHandler.java:57)
> at org.infinispan.server.hotrod.ContextHandler$$Lambda$86/1492247987.run(Unknown Source)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144)
> at java.lang.Thread.run(Thread.java:745)
> {code}
> Client exception:
> {code}
> Jan 24, 2018 10:44:20 AM org.infinispan.client.hotrod.impl.protocol.Codec20 checkForErrorsInResponseStatus
> WARN: ISPN004005: Error received from the server: java.lang.IllegalStateException: ISPN006016: Factory 'org.infinispan.server.hotrod.HotRodServer$ToEmptyBytesKeyValueFilterConverter' not found in server
> Exception in thread "main" org.infinispan.client.hotrod.exceptions.HotRodClientException:Request for messageId=5 returned server error (status=0x85): java.lang.IllegalStateException: ISPN006016: Factory 'org.infinispan.server.hotrod.HotRodServer$ToEmptyBytesKeyValueFilterConverter' not found in server
> at org.infinispan.client.hotrod.impl.protocol.Codec20.checkForErrorsInResponseStatus(Codec20.java:408)
> at org.infinispan.client.hotrod.impl.protocol.Codec20.readPartialHeader(Codec20.java:162)
> at org.infinispan.client.hotrod.impl.protocol.Codec20.readHeader(Codec20.java:148)
> at org.infinispan.client.hotrod.impl.operations.HotRodOperation.readHeaderAndValidate(HotRodOperation.java:60)
> at org.infinispan.client.hotrod.impl.operations.IterationStartOperation.executeOperation(IterationStartOperation.java:72)
> at org.infinispan.client.hotrod.impl.operations.IterationStartOperation.executeOperation(IterationStartOperation.java:21)
> at org.infinispan.client.hotrod.impl.operations.RetryOnFailureOperation.execute(RetryOnFailureOperation.java:56)
> at org.infinispan.client.hotrod.impl.iteration.RemoteCloseableIterator.startInternal(RemoteCloseableIterator.java:127)
> at org.infinispan.client.hotrod.impl.iteration.RemoteCloseableIterator.start(RemoteCloseableIterator.java:140)
> at org.infinispan.client.hotrod.impl.RemoteCacheImpl.retrieveEntries(RemoteCacheImpl.java:162)
> at org.infinispan.client.hotrod.impl.RemoteCacheImpl.retrieveEntries(RemoteCacheImpl.java:168)
> at org.infinispan.client.hotrod.impl.RemoteCacheImpl.retrieveEntries(RemoteCacheImpl.java:173)
> at org.infinispan.client.hotrod.impl.RemoteCacheImpl$KeySet.iterator(RemoteCacheImpl.java:553)
> at org.mposolda.ispn.InfinispanRemote.main(InfinispanRemote.java:34)
> {code}
> Indeed, When looking at this line of class RemoteCacheImpl, I see that it references the class, which seem that it was added in HotRodServer version 9. This looks like the cause of the error: https://github.com/infinispan/infinispan/blob/master/client/hotrod-client...
--
This message was sent by Atlassian JIRA
(v7.5.0#75005)
6 years, 9 months