[infinispan-commits] Infinispan SVN: r2435 - in trunk/lucene-directory/src: test/java/org/infinispan/lucene and 1 other directory.
infinispan-commits at lists.jboss.org
infinispan-commits at lists.jboss.org
Sun Sep 26 12:26:48 EDT 2010
Author: sannegrinovero
Date: 2010-09-26 12:26:47 -0400 (Sun, 26 Sep 2010)
New Revision: 2435
Modified:
trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexInput.java
trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java
trunk/lucene-directory/src/main/java/org/infinispan/lucene/SingleChunkIndexInput.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
Log:
[ISPN-671] (Lucene Directory doesn't appropriately throw an IOException in case read-past-eof happens at chunk size switch) - merged from 4.2
Modified: trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexInput.java
===================================================================
--- trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexInput.java 2010-09-26 16:20:30 UTC (rev 2434)
+++ trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexInput.java 2010-09-26 16:26:47 UTC (rev 2435)
@@ -136,7 +136,7 @@
ChunkCacheKey key = new ChunkCacheKey(fileKey.getIndexName(), filename, currentLoadedChunk);
buffer = (byte[]) cache.withFlags(Flag.SKIP_LOCKING).get(key);
if (buffer == null) {
- throw new IOException("Chunk value could not be found for key " + key);
+ throw new IOException("Read past EOF: Chunk value could not be found for key " + key);
}
currentBufferSize = buffer.length;
}
Modified: trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java
===================================================================
--- trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java 2010-09-26 16:20:30 UTC (rev 2434)
+++ trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java 2010-09-26 16:26:47 UTC (rev 2435)
@@ -143,13 +143,12 @@
}
public void doFlush() throws IOException {
- // create key for the current chunk
- ChunkCacheKey key = new ChunkCacheKey(fileKey.getIndexName(), fileKey.getFileName(), currentChunkNumber);
// size changed, apply change to file header
file.touch();
resizeFileIfNeeded();
byte[] bufferToFlush = buffer;
- if (isWritingOnLastChunk()) {
+ boolean writingOnLastChunk = isWritingOnLastChunk();
+ if (writingOnLastChunk) {
int newBufferSize = (int) (file.getSize() % bufferSize);
if (newBufferSize != 0) {
bufferToFlush = new byte[newBufferSize];
@@ -162,8 +161,12 @@
microbatch = cache.startBatch();
}
// add chunk to cache
- cache.withFlags(Flag.SKIP_REMOTE_LOOKUP, Flag.SKIP_LOCKING).put(key, bufferToFlush);
- // override existing file header with new size and last time access
+ if (!writingOnLastChunk || this.positionInBuffer!=0) {
+ // create key for the current chunk
+ ChunkCacheKey key = new ChunkCacheKey(fileKey.getIndexName(), fileKey.getFileName(), currentChunkNumber);
+ cache.withFlags(Flag.SKIP_REMOTE_LOOKUP, Flag.SKIP_LOCKING).put(key, bufferToFlush);
+ }
+ // override existing file header with new size and updated accesstime
cache.withFlags(Flag.SKIP_REMOTE_LOOKUP, Flag.SKIP_LOCKING).put(fileKey, file);
registerToFileListIfNeeded();
if (microbatch) cache.endBatch(true);
Modified: trunk/lucene-directory/src/main/java/org/infinispan/lucene/SingleChunkIndexInput.java
===================================================================
--- trunk/lucene-directory/src/main/java/org/infinispan/lucene/SingleChunkIndexInput.java 2010-09-26 16:20:30 UTC (rev 2434)
+++ trunk/lucene-directory/src/main/java/org/infinispan/lucene/SingleChunkIndexInput.java 2010-09-26 16:26:47 UTC (rev 2435)
@@ -45,10 +45,13 @@
public SingleChunkIndexInput(AdvancedCache cache, FileCacheKey fileKey, FileMetadata fileMetadata) throws FileNotFoundException {
ChunkCacheKey key = new ChunkCacheKey(fileKey.getIndexName(), fileKey.getFileName(), 0);
- buffer = (byte[]) cache.withFlags(Flag.SKIP_LOCKING).get(key);
- if (buffer == null) {
- throw new FileNotFoundException("Chunk value could not be found for key " + key);
+ byte[] b = (byte[]) cache.withFlags(Flag.SKIP_LOCKING).get(key);
+ if (b == null) {
+ buffer = new byte[0];
}
+ else {
+ buffer = b;
+ }
bufferPosition = 0;
}
@@ -77,7 +80,9 @@
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
- len = Math.min(len, buffer.length - bufferPosition);
+ if (buffer.length - bufferPosition < len) {
+ throw new IOException("Read past EOF");
+ }
System.arraycopy(buffer, bufferPosition, b, offset, len);
bufferPosition+=len;
}
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2010-09-26 16:20:30 UTC (rev 2434)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2010-09-26 16:26:47 UTC (rev 2435)
@@ -22,17 +22,22 @@
package org.infinispan.lucene;
import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMDirectory;
import org.infinispan.Cache;
import org.infinispan.lucene.testutils.RepeatableLongByteSequence;
import org.infinispan.manager.CacheContainer;
+import org.infinispan.test.TestingUtil;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.AfterTest;
@@ -51,6 +56,7 @@
private static final String INDEXNAME = "index";
private CacheContainer cacheManager;
+ private File indexDir = new File(new File("."), INDEXNAME);
@BeforeTest
public void prepareCacheManager() {
@@ -62,9 +68,10 @@
cacheManager.stop();
}
- @AfterMethod
+ @AfterMethod(alwaysRun=true)
public void clearCache() {
cacheManager.getCache().clear();
+ TestingUtil.recursiveFileRemove(indexDir);
}
@Test
@@ -129,7 +136,6 @@
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
-
@Test
public void testReadWholeFile() throws IOException {
@@ -541,5 +547,99 @@
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
+
+ @Test
+ public void testChunkBordersOnInfinispan() throws IOException {
+ Cache cache = cacheManager.getCache();
+ cache.clear();
+ InfinispanDirectory dir = new InfinispanDirectory(cache, INDEXNAME, 13);
+ testChunkBorders(dir);
+ cache.clear();
+ }
+
+ @Test
+ public void testChunkBordersOnRAMDirectory() throws IOException {
+ RAMDirectory dir = new RAMDirectory();
+ testChunkBorders(dir);
+ }
+
+ @Test
+ public void testChunkBordersOnFSDirectory() throws IOException {
+ boolean directoriesCreated = indexDir.mkdirs();
+ assert directoriesCreated : "couldn't create directory for FSDirectory test";
+ FSDirectory dir = FSDirectory.open(indexDir);
+ testChunkBorders(dir);
+ }
+
+ /**
+ * Useful to verify the Infinispan Directory has similar behaviour
+ * to standard Lucene implementations regarding reads out of ranges.
+ */
+ private void testChunkBorders(Directory dir) throws IOException {
+ //numbers are chosen to be multiples of the chunksize set for the InfinispanDirectory
+ //so that we test the borders of it.
+
+ testOn(dir, 0 ,0);
+ testOn(dir, 0 ,1);
+ testOn(dir, 1 ,1);
+ testOn(dir, 1 ,0);
+
+ // all equal:
+ testOn(dir, 13 ,13);
+
+ // one less:
+ testOn(dir, 12 ,13);
+ testOn(dir, 13 ,12);
+ testOn(dir, 12 ,12);
+
+ // one higher
+ testOn(dir, 13 ,14);
+ testOn(dir, 14 ,13);
+ testOn(dir, 14 ,14);
+
+ // now repeat in multi-chunk scenario:
+ // all equal:
+ testOn(dir, 39 ,39);
+
+ // one less:
+ testOn(dir, 38 ,38);
+ testOn(dir, 38 ,39);
+ testOn(dir, 39 ,38);
+
+ // one higher
+ testOn(dir, 40 ,40);
+ testOn(dir, 40 ,39);
+ testOn(dir, 39 ,40);
+ }
+ private void testOn(Directory dir, int writeSize, int readSize) throws IOException {
+ cacheManager.getCache().clear();//needed to make sure no chunks are left over in case of Infinispan implementation
+ final String filename = "chunkTest";
+ IndexOutput indexOutput = dir.createOutput(filename);
+ byte[] toWrite = fillBytes(writeSize);
+ indexOutput.writeBytes(toWrite, writeSize);
+ indexOutput.close();
+ byte[] results = new byte[readSize];
+ IndexInput openInput = dir.openInput(filename);
+ try {
+ openInput.readBytes(results, 0, readSize);
+ for (int i = 0; i < writeSize && i < readSize; i++) {
+ assert toWrite[i] == results[i];
+ }
+ if (readSize > writeSize)
+ assert false : "should have thrown an IOException for reading past EOF";
+ } catch (IOException ioe) {
+ if (readSize <= writeSize)
+ assert false :"should not have thrown an IOException" + ioe.getMessage();
+ }
+ }
+
+ private byte[] fillBytes(int size) {
+ byte[] b = new byte[size];
+ for (int i=0; i<size; i++) {
+ b[i]=(byte)i;
+ }
+ return b;
+ }
+
}
More information about the infinispan-commits
mailing list