[infinispan-commits] Infinispan SVN: r2463 - in trunk/lucene-directory/src: test/java/org/infinispan/lucene and 1 other directories.
infinispan-commits at lists.jboss.org
infinispan-commits at lists.jboss.org
Sat Oct 2 10:00:16 EDT 2010
Author: sannegrinovero
Date: 2010-10-02 10:00:15 -0400 (Sat, 02 Oct 2010)
New Revision: 2463
Modified:
trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexInput.java
trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/DatabaseStoredIndexTest.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/DynamicBufferSizeTest.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
trunk/lucene-directory/src/test/resources/log4j.xml
Log:
[ISPN-684] (Lucene: improve tests to make sure no empty buffers are stored) - merged from 4.2
Modified: trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexInput.java
===================================================================
--- trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexInput.java 2010-10-02 13:53:18 UTC (rev 2462)
+++ trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexInput.java 2010-10-02 14:00:15 UTC (rev 2463)
@@ -73,7 +73,7 @@
}
@Override
- public byte readByte() throws IOException {
+ public final byte readByte() throws IOException {
if (bufferPosition >= currentBufferSize) {
nextChunk();
bufferPosition = 0;
@@ -82,8 +82,7 @@
}
@Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- int bytesToRead = len;
+ public final void readBytes(final byte[] b, int offset, int bytesToRead) throws IOException {
if (buffer == null) {
nextChunk();
}
Modified: trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java
===================================================================
--- trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java 2010-10-02 13:53:18 UTC (rev 2462)
+++ trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java 2010-10-02 14:00:15 UTC (rev 2463)
@@ -97,7 +97,7 @@
}
private void newChunk() throws IOException {
- doFlush();// save data first
+ storeCurrentBuffer();// save data first
currentChunkNumber++;
// check if we have to create new chunk, or get already existing in cache for modification
buffer = getChunkById(cache, fileKey, currentChunkNumber, bufferSize);
@@ -105,7 +105,7 @@
}
@Override
- public void writeByte(byte b) throws IOException {
+ public final void writeByte(byte b) throws IOException {
if (isNewChunkNeeded()) {
newChunk();
}
@@ -114,17 +114,17 @@
}
@Override
- public void writeBytes(byte[] b, int offset, int length) throws IOException {
+ public final void writeBytes(final byte[] b, final int offset, int length) throws IOException {
int writtenBytes = 0;
while (writtenBytes < length) {
+ if (isNewChunkNeeded()) {
+ newChunk();
+ }
int pieceLength = Math.min(bufferSize - positionInBuffer, length - writtenBytes);
System.arraycopy(b, offset + writtenBytes, buffer, positionInBuffer, pieceLength);
positionInBuffer += pieceLength;
filePosition += pieceLength;
writtenBytes += pieceLength;
- if (isNewChunkNeeded()) {
- newChunk();
- }
}
}
@@ -139,12 +139,11 @@
if ( ! transactionRunning) {
transactionRunning = cache.startBatch();
}
- doFlush();
+ storeCurrentBuffer();
}
- public void doFlush() throws IOException {
+ protected void storeCurrentBuffer() throws IOException {
// size changed, apply change to file header
- file.touch();
resizeFileIfNeeded();
byte[] bufferToFlush = buffer;
boolean writingOnLastChunk = isWritingOnLastChunk();
@@ -161,7 +160,7 @@
microbatch = cache.startBatch();
}
// add chunk to cache
- if (!writingOnLastChunk || this.positionInBuffer!=0) {
+ if ( ! writingOnLastChunk || this.positionInBuffer != 0) {
// create key for the current chunk
ChunkCacheKey key = new ChunkCacheKey(fileKey.getIndexName(), fileKey.getFileName(), currentChunkNumber);
cache.withFlags(Flag.SKIP_REMOTE_LOOKUP, Flag.SKIP_LOCKING).put(key, bufferToFlush);
@@ -187,7 +186,7 @@
@Override
public void close() throws IOException {
- doFlush();
+ storeCurrentBuffer();
if (transactionRunning) {
//commit
cache.endBatch(true);
@@ -212,10 +211,10 @@
if (pos > file.getSize()) {
resizeFileIfNeeded();
if (pos > file.getSize()) // check again, might be fixed by the resize
- throw new IOException(fileKey.getFileName() + ": seeking past of the file");
+ throw new IOException(fileKey.getFileName() + ": seeking past end of file");
}
if (requestedChunkNumber != currentChunkNumber) {
- doFlush();
+ storeCurrentBuffer();
buffer = getChunkById(cache, fileKey, requestedChunkNumber, bufferSize);
currentChunkNumber = requestedChunkNumber;
}
@@ -229,8 +228,8 @@
}
private boolean isWritingOnLastChunk() {
- int lastChunkNumber = (int) (file.getSize() / bufferSize);
- return currentChunkNumber == lastChunkNumber;
+ int lastChunkNumber = file.getNumberOfChunks() - 1;
+ return currentChunkNumber >= lastChunkNumber;
}
}
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/DatabaseStoredIndexTest.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/DatabaseStoredIndexTest.java 2010-10-02 13:53:18 UTC (rev 2462)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/DatabaseStoredIndexTest.java 2010-10-02 14:00:15 UTC (rev 2463)
@@ -25,8 +25,6 @@
import java.sql.SQLException;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Set;
import static org.infinispan.lucene.CacheTestSupport.assertTextIsFoundInIds;
import static org.infinispan.lucene.CacheTestSupport.removeByTerm;
@@ -105,15 +103,11 @@
public void indexWasStored() throws IOException {
cache = cacheManager.getCache();
assert cache.isEmpty();
- Directory dir = new InfinispanDirectory(cache, INDEX_NAME);
- assertTextIsFoundInIds(dir, "index", 1);
- dir.close();
boolean failed = false;
- Set<FileReadLockKey> keysToRemove = new HashSet<FileReadLockKey>();
for (Object key : cacheCopy.keySet()) {
if (key instanceof FileReadLockKey) {
- // ignore until ISPN-581 defines a behaviour
- keysToRemove.add((FileReadLockKey) key);
+ System.out.println("Key found in store, shouldn't have persisted this or should have cleaned up all readlocks on directory close:" + key);
+ failed = true;
}
else {
Object expected = cacheCopy.get(key);
@@ -124,17 +118,17 @@
expected = Arrays.toString((byte[]) expected);
actual = Arrays.toString((byte[]) actual);
}
- if (!expected.equals(actual)) {
- System.out.println("Failure on key["+key.toString()+"] expected value:\n"+expected+"\nactual value:\n"+actual);
+ if (expected == null || ! expected.equals(actual)) {
+ System.out.println("Failure on key["+key.toString()+"] expected value:\n\t"+expected+"\tactual value:\n\t"+actual);
failed = true;
}
}
}
Assert.assertFalse(failed);
- for (FileReadLockKey key : keysToRemove){
- Assert.assertNotNull(cacheCopy.remove(key));
- }
Assert.assertEquals(cacheCopy.keySet().size(), cache.keySet().size(), "have a different number of keys");
+ Directory dir = new InfinispanDirectory(cache, INDEX_NAME);
+ assertTextIsFoundInIds(dir, "index", 1);
+ dir.close();
}
}
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java 2010-10-02 13:53:18 UTC (rev 2462)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java 2010-10-02 14:00:15 UTC (rev 2463)
@@ -135,8 +135,10 @@
ChunkCacheKey chunkKey = new ChunkCacheKey(indexName, fileName, i);
byte[] buffer = (byte[]) cache.get(chunkKey);
if (buffer == null) {
+ assert cache.containsKey(chunkKey)==false;
return accumulator;
} else {
+ assert buffer.length > 0; //check we don't store useless data
accumulator += buffer.length;
}
}
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/DynamicBufferSizeTest.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/DynamicBufferSizeTest.java 2010-10-02 13:53:18 UTC (rev 2462)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/DynamicBufferSizeTest.java 2010-10-02 14:00:15 UTC (rev 2463)
@@ -57,6 +57,7 @@
public void roundingTest() {
FileMetadata m = new FileMetadata();
m.setBufferSize(10);
+ Assert.assertEquals(0, m.getNumberOfChunks());
m.setSize(10);
Assert.assertEquals(1, m.getNumberOfChunks());
m.setSize(11);
@@ -65,10 +66,12 @@
Assert.assertEquals(1, m.getNumberOfChunks());
m.setSize(22);
Assert.assertEquals(2, m.getNumberOfChunks());
+ m.setSize(31);
+ m.setBufferSize(10);
+ Assert.assertEquals(4, m.getNumberOfChunks());
}
@Test
- @SuppressWarnings("unchecked")
public void testReadingFromDifferentlySizedBuffers() throws IOException {
cache = cacheManager.getCache();
Directory dirA = new InfinispanDirectory(cache, "indexName", 7);
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2010-10-02 13:53:18 UTC (rev 2462)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2010-10-02 14:00:15 UTC (rev 2463)
@@ -90,6 +90,9 @@
io.writeByte(bytesGenerator.nextByte());
}
io.flush();
+ assert io.length() == REPEATABLE_BUFFER_SIZE;
+ long deepCountFileSize = DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME,fileName), cache);
+ assert io.length() == deepCountFileSize;
//Text to write on file with repeatable text
final String someText = "This is some text";
@@ -107,6 +110,7 @@
bytesGenerator.reset();
final long finalSize = REPEATABLE_BUFFER_SIZE + someTextAsBytes.length;
assert io.length() == finalSize;
+ assert io.length() == DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME,fileName), cache);
int indexPointer = 0;
Arrays.sort(pointers);
@@ -309,8 +313,7 @@
* The size content file to create
* @throws IOException
*/
- private void createFileWithRepeatableContent(InfinispanDirectory dir, String fileName,
- final int contentFileSize) throws IOException {
+ private void createFileWithRepeatableContent(InfinispanDirectory dir, String fileName, final int contentFileSize) throws IOException {
IndexOutput indexOutput = dir.createOutput(fileName);
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
for (int i = 0; i < contentFileSize; i++) {
@@ -553,14 +556,14 @@
Cache cache = cacheManager.getCache();
cache.clear();
InfinispanDirectory dir = new InfinispanDirectory(cache, INDEXNAME, 13);
- testChunkBorders(dir);
+ testChunkBorders(dir, cache);
cache.clear();
}
@Test
public void testChunkBordersOnRAMDirectory() throws IOException {
RAMDirectory dir = new RAMDirectory();
- testChunkBorders(dir);
+ testChunkBorders(dir, null);
}
@Test
@@ -568,57 +571,61 @@
boolean directoriesCreated = indexDir.mkdirs();
assert directoriesCreated : "couldn't create directory for FSDirectory test";
FSDirectory dir = FSDirectory.open(indexDir);
- testChunkBorders(dir);
+ testChunkBorders(dir, null);
}
/**
* Useful to verify the Infinispan Directory has similar behaviour
* to standard Lucene implementations regarding reads out of ranges.
*/
- private void testChunkBorders(Directory dir) throws IOException {
+ private void testChunkBorders(Directory dir, Cache cache) throws IOException {
//numbers are chosen to be multiples of the chunksize set for the InfinispanDirectory
//so that we test the borders of it.
- testOn(dir, 0 ,0);
- testOn(dir, 0 ,1);
- testOn(dir, 1 ,1);
- testOn(dir, 1 ,0);
+ testOn(dir, 0 ,0, cache);
+ testOn(dir, 0 ,1, cache);
+ testOn(dir, 1 ,1, cache);
+ testOn(dir, 1 ,0, cache);
// all equal:
- testOn(dir, 13 ,13);
+ testOn(dir, 13 ,13, cache);
// one less:
- testOn(dir, 12 ,13);
- testOn(dir, 13 ,12);
- testOn(dir, 12 ,12);
+ testOn(dir, 12 ,13, cache);
+ testOn(dir, 13 ,12, cache);
+ testOn(dir, 12 ,12, cache);
// one higher
- testOn(dir, 13 ,14);
- testOn(dir, 14 ,13);
- testOn(dir, 14 ,14);
+ testOn(dir, 13 ,14, cache);
+ testOn(dir, 14 ,13, cache);
+ testOn(dir, 14 ,14, cache);
// now repeat in multi-chunk scenario:
// all equal:
- testOn(dir, 39 ,39);
+ testOn(dir, 39 ,39, cache);
// one less:
- testOn(dir, 38 ,38);
- testOn(dir, 38 ,39);
- testOn(dir, 39 ,38);
+ testOn(dir, 38 ,38, cache);
+ testOn(dir, 38 ,39, cache);
+ testOn(dir, 39 ,38, cache);
// one higher
- testOn(dir, 40 ,40);
- testOn(dir, 40 ,39);
- testOn(dir, 39 ,40);
+ testOn(dir, 40 ,40, cache);
+ testOn(dir, 40 ,39, cache);
+ testOn(dir, 39 ,40, cache);
}
- private void testOn(Directory dir, int writeSize, int readSize) throws IOException {
- cacheManager.getCache().clear();//needed to make sure no chunks are left over in case of Infinispan implementation
+ private void testOn(Directory dir, int writeSize, int readSize, Cache cache) throws IOException {
+ if (cache != null) cache.clear();//needed to make sure no chunks are left over in case of Infinispan implementation
final String filename = "chunkTest";
IndexOutput indexOutput = dir.createOutput(filename);
byte[] toWrite = fillBytes(writeSize);
indexOutput.writeBytes(toWrite, writeSize);
indexOutput.close();
+ if (cache != null) {
+ assert writeSize == DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME,filename), cache);
+ }
+ assert indexOutput.length() == writeSize;
byte[] results = new byte[readSize];
IndexInput openInput = dir.openInput(filename);
try {
Modified: trunk/lucene-directory/src/test/resources/log4j.xml
===================================================================
--- trunk/lucene-directory/src/test/resources/log4j.xml 2010-10-02 13:53:18 UTC (rev 2462)
+++ trunk/lucene-directory/src/test/resources/log4j.xml 2010-10-02 14:00:15 UTC (rev 2463)
@@ -48,6 +48,14 @@
<category name="org.infinispan.factories">
<priority value="WARN"/>
</category>
+
+ <category name="org.jgroups">
+ <priority value="WARN"/>
+ </category>
+
+ <category name="com.mchange">
+ <priority value="WARN"/>
+ </category>
<!-- ======================= -->
<!-- Setup the Root category -->
More information about the infinispan-commits
mailing list